diff -ruN --no-dereference -x .git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt --- a/Documentation/admin-guide/kernel-parameters.txt 2026-10-11 15:27:50.000000000 +0200 +++ b/Documentation/admin-guide/kernel-parameters.txt 2026-10-13 14:33:30.000000000 +0200 @@ -2487,6 +2487,9 @@ disable Do not enable intel_pstate as the default scaling driver for the supported processors + enable + Enable intel_pstate in-case "disable" was passed + previously in the kernel boot parameters active Use intel_pstate driver to bypass the scaling governors layer of cpufreq and provides it own @@ -5026,6 +5029,15 @@ nomsi [MSI] If the PCI_MSI kernel config parameter is enabled, this kernel boot option can be used to disable the use of MSI interrupts system-wide. + pcie_acs_override = + [PCIE] Override missing PCIe ACS support for: + downstream + All downstream ports - full ACS capabilities + multfunction + All multifunction devices - multifunction ACS subset + id:nnnn:nnnn + Specfic device - full ACS capabilities + Specified as vid:did (vendor/device ID) in hex noioapicquirk [APIC] Disable all boot interrupt quirks. Safety option to keep boot IRQs enabled. This should never be necessary. @@ -7906,6 +7918,8 @@ obtained from HW or FW using either an MSR or CPUID(0x15). Warn if the difference is more than 500 ppm. [x86] watchdog: Enforce the clocksource watchdog on TSC + [x86] directsync: attempt to sync the tsc via direct + writes if MSR_IA32_TSC_ADJUST isn't available tsc_early_khz= [X86,EARLY] Skip early TSC calibration and use the given value instead. Useful when the early TSC frequency discovery diff -ruN --no-dereference -x .git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst --- a/Documentation/admin-guide/pm/amd-pstate.rst 2026-10-11 15:27:50.000000000 +0200 +++ b/Documentation/admin-guide/pm/amd-pstate.rst 2026-10-13 14:33:30.000000000 +0200 @@ -372,6 +372,22 @@ Attempting to manually write to the ``energy_performance_preference`` sysfs file will fail when ``dynamic_epp`` is enabled. +Energy performance preference boosting +====================================== +On MSR-based systems running in the ``active mode``, the driver can optionally +bias the energy performance preference of a recently-busy CPU toward +performance while leaving all other CPUs at the policy value. C0 residency is +sampled at most every 10 ms; while a CPU has been at least 50% busy within the +last 300 ms, the EPP field of its CPPC request is set to ``performance`` (0). +The policy value is restored once 300 ms pass without a busy sample. + +This behavior is disabled by default. It can be enabled with the module +parameter ``amd_pstate.epp_boost=1`` on the kernel command line, or at runtime +by writing ``1`` to ``/sys/module/amd_pstate/parameters/epp_boost``. The +mechanism only runs while the parameter is enabled and has no overhead when +disabled. It has no effect in the ``passive`` and ``guided`` modes or on +shared-memory (non-MSR) systems. + ``amd-pstate`` vs ``acpi-cpufreq`` ====================================== diff -ruN --no-dereference -x .git a/Makefile b/Makefile --- a/Makefile 2026-10-11 15:27:50.000000000 +0200 +++ b/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -932,11 +932,20 @@ ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE KBUILD_CFLAGS += -O2 KBUILD_RUSTFLAGS += -Copt-level=2 +else ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3 +KBUILD_CFLAGS += -O3 +KBUILD_RUSTFLAGS += -Copt-level=3 else ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE KBUILD_CFLAGS += -Os KBUILD_RUSTFLAGS += -Copt-level=s endif +# Perform swing modulo scheduling immediately before the first scheduling pass. +# This pass looks at innermost loops and reorders their instructions by +# overlapping different iterations. +KBUILD_CFLAGS += $(call cc-option,-fmodulo-sched -fmodulo-sched-allow-regmoves -fivopts) +KBUILD_CFLAGS += $(call cc-option,-mllvm -enable-pipeliner) + # Always set `debug-assertions` and `overflow-checks` because their default # depends on `opt-level` and `debug-assertions`, respectively. KBUILD_RUSTFLAGS += -Cdebug-assertions=$(if $(CONFIG_RUST_DEBUG_ASSERTIONS),y,n) diff -ruN --no-dereference -x .git a/arch/arm/include/asm/mmu_context.h b/arch/arm/include/asm/mmu_context.h --- a/arch/arm/include/asm/mmu_context.h 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/arm/include/asm/mmu_context.h 2026-10-13 14:33:30.000000000 +0200 @@ -80,7 +80,7 @@ #ifndef MODULE #define finish_arch_post_lock_switch \ finish_arch_post_lock_switch -static inline void finish_arch_post_lock_switch(void) +static __always_inline void finish_arch_post_lock_switch(void) { struct mm_struct *mm = current->mm; diff -ruN --no-dereference -x .git a/arch/riscv/include/asm/sync_core.h b/arch/riscv/include/asm/sync_core.h --- a/arch/riscv/include/asm/sync_core.h 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/riscv/include/asm/sync_core.h 2026-10-13 14:33:30.000000000 +0200 @@ -6,7 +6,7 @@ * RISC-V implements return to user-space through an xRET instruction, * which is not core serializing. */ -static inline void sync_core_before_usermode(void) +static __always_inline void sync_core_before_usermode(void) { asm volatile ("fence.i" ::: "memory"); } diff -ruN --no-dereference -x .git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h --- a/arch/s390/include/asm/mmu_context.h 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/s390/include/asm/mmu_context.h 2026-10-13 14:33:30.000000000 +0200 @@ -93,7 +93,7 @@ } #define finish_arch_post_lock_switch finish_arch_post_lock_switch -static inline void finish_arch_post_lock_switch(void) +static __always_inline void finish_arch_post_lock_switch(void) { struct task_struct *tsk = current; struct mm_struct *mm = tsk->mm; diff -ruN --no-dereference -x .git a/arch/sparc/include/asm/mmu_context_64.h b/arch/sparc/include/asm/mmu_context_64.h --- a/arch/sparc/include/asm/mmu_context_64.h 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/sparc/include/asm/mmu_context_64.h 2026-10-13 14:33:30.000000000 +0200 @@ -160,7 +160,7 @@ } #define finish_arch_post_lock_switch finish_arch_post_lock_switch -static inline void finish_arch_post_lock_switch(void) +static __always_inline void finish_arch_post_lock_switch(void) { /* Restore the state of MCDPER register for the new process * just switched to. diff -ruN --no-dereference -x .git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu --- a/arch/x86/Kconfig.cpu 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/x86/Kconfig.cpu 2026-10-13 14:33:30.000000000 +0200 @@ -214,6 +214,10 @@ def_bool (RUSTC_VERSION >= 108800 && RUSTC_LLVM_MAJOR_VERSION >= 23) || \ RUSTC_VERSION >= 109300 +choice + prompt "x86_64 Compiler Build Optimization" + default GENERIC_CPU + config X86_NATIVE_CPU bool "Build and optimize for local/native CPU" depends on X86_64 @@ -229,6 +233,47 @@ If unsure, say N. +config GENERIC_CPU + bool "Generic-x86-64" + depends on X86_64 + help + Generic x86-64 CPU. + Runs equally well on all x86-64 CPUs. + +config MZEN4 + bool "AMD Ryzen 4" + depends on (CC_IS_GCC && GCC_VERSION >= 130000) || (CC_IS_CLANG && CLANG_VERSION >= 160000) + help + Select this for AMD Family 19h Zen 4 processors. + + Enables -march=znver4 + +endchoice + +config X86_64_VERSION + int "x86-64 compiler ISA level" + range 1 4 + depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) + depends on X86_64 && GENERIC_CPU + help + Specify a specific x86-64 compiler ISA level. + + There are three x86-64 ISA levels that work on top of + the x86-64 baseline, namely: x86-64-v2 and x86-64-v3. + + x86-64-v2 brings support for vector instructions up to Streaming SIMD + Extensions 4.2 (SSE4.2) and Supplemental Streaming SIMD Extensions 3 + (SSSE3), the POPCNT instruction, and CMPXCHG16B. + + x86-64-v3 adds vector instructions up to AVX2, MOVBE, and additional + bit-manipulation instructions. + + x86-64-v4 is not included since the kernel does not use AVX512 instructions + + You can find the best version for your CPU by running one of the following: + /lib/ld-linux-x86-64.so.2 --help | grep supported + /lib64/ld-linux-x86-64.so.2 --help | grep supported + config X86_GENERIC bool "Generic x86 support" depends on X86_32 diff -ruN --no-dereference -x .git a/arch/x86/Makefile b/arch/x86/Makefile --- a/arch/x86/Makefile 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/x86/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -167,10 +167,22 @@ # generate_rust_target.rs handles Rust APX gating. KBUILD_RUSTFLAGS += -Ctarget-cpu=native -else +endif + +ifdef CONFIG_MZEN4 + KBUILD_CFLAGS += -march=znver4 + KBUILD_RUSTFLAGS += -Ctarget-cpu=znver4 +endif + +ifdef CONFIG_GENERIC_CPU +ifeq ($(CONFIG_X86_64_VERSION),1) KBUILD_CFLAGS += -march=x86-64 -mtune=generic KBUILD_RUSTFLAGS += -Ctarget-cpu=x86-64 -Ztune-cpu=generic -endif +else + KBUILD_CFLAGS +=-march=x86-64-v$(CONFIG_X86_64_VERSION) + KBUILD_RUSTFLAGS += -Ctarget-cpu=x86-64-v$(CONFIG_X86_64_VERSION) +endif # CONFIG_X86_64_VERSION +endif # CONFIG_GENERIC_CPU KBUILD_CFLAGS += -mno-red-zone KBUILD_CFLAGS += -mcmodel=kernel diff -ruN --no-dereference -x .git a/arch/x86/include/asm/pci.h b/arch/x86/include/asm/pci.h --- a/arch/x86/include/asm/pci.h 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/x86/include/asm/pci.h 2026-10-13 14:33:30.000000000 +0200 @@ -26,6 +26,7 @@ #if IS_ENABLED(CONFIG_VMD) struct pci_dev *vmd_dev; /* VMD Device if in Intel VMD domain */ #endif + struct pci_dev *nvme_remap_dev; /* AHCI Device if NVME remapped bus */ }; extern int pci_routeirq; @@ -69,6 +70,11 @@ #define is_vmd(bus) false #endif /* CONFIG_VMD */ +static inline bool is_nvme_remap(struct pci_bus *bus) +{ + return to_pci_sysdata(bus)->nvme_remap_dev != NULL; +} + /* Can be used to override the logic in pci_scan_bus for skipping already-configured bus numbers - to be used for buggy BIOSes or architectures with incomplete PCI setup by the loader */ diff -ruN --no-dereference -x .git a/arch/x86/include/asm/sync_core.h b/arch/x86/include/asm/sync_core.h --- a/arch/x86/include/asm/sync_core.h 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/x86/include/asm/sync_core.h 2026-10-13 14:33:30.000000000 +0200 @@ -93,7 +93,7 @@ * to user-mode. x86 implements return to user-space through sysexit, * sysrel, and sysretq, which are not core serializing. */ -static inline void sync_core_before_usermode(void) +static __always_inline void sync_core_before_usermode(void) { /* With PTI, we unconditionally serialize before running user code. */ if (static_cpu_has(X86_FEATURE_PTI)) diff -ruN --no-dereference -x .git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h --- a/arch/x86/include/asm/tlbflush.h 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/x86/include/asm/tlbflush.h 2026-10-13 14:33:30.000000000 +0200 @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -212,6 +213,12 @@ extern void initialize_tlbstate_and_flush(void); /* + * Keep stack-allocated flush_tlb_info cacheline aligned, but cap the + * alignment to avoid excessive stack usage on large-cacheline systems. + */ +#define FLUSH_TLB_INFO_ALIGN MIN(SMP_CACHE_BYTES, 64) + +/* * TLB flushing: * * - flush_tlb_all() flushes all processes TLBs @@ -249,7 +256,7 @@ u8 stride_shift; u8 freed_tables; u8 trim_cpumask; -}; +} __aligned(FLUSH_TLB_INFO_ALIGN); void flush_tlb_local(void); void flush_tlb_one_user(unsigned long addr); diff -ruN --no-dereference -x .git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h --- a/arch/x86/include/asm/tsc.h 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/x86/include/asm/tsc.h 2026-10-13 14:33:30.000000000 +0200 @@ -93,6 +93,7 @@ extern unsigned long long native_sched_clock_from_tsc(u64 tsc); extern int tsc_clocksource_reliable; +extern int tsc_allow_direct_sync; extern bool tsc_async_resets; /* diff -ruN --no-dereference -x .git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c --- a/arch/x86/kernel/acpi/sleep.c 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/x86/kernel/acpi/sleep.c 2026-10-13 14:33:30.000000000 +0200 @@ -104,6 +104,15 @@ header->pmode_misc_en_high)) header->pmode_behavior |= (1 << WAKEUP_BEHAVIOR_RESTORE_MISC_ENABLE); + + if ((boot_cpu_has(X86_FEATURE_TSC_ADJUST) || + tsc_allow_direct_sync) && + !rdmsr_safe(MSR_IA32_TSC, + &header->pmode_tsc_low, + &header->pmode_tsc_high)) + header->pmode_behavior |= + (1 << WAKEUP_BEHAVIOR_RESTORE_TSC); + header->realmode_flags = acpi_realmode_flags; header->real_magic = 0x12345678; diff -ruN --no-dereference -x .git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c --- a/arch/x86/kernel/cpu/amd.c 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/x86/kernel/cpu/amd.c 2026-10-13 14:33:30.000000000 +0200 @@ -1042,11 +1042,8 @@ static void init_amd_zen5(struct cpuinfo_x86 *c) { - if (!x86_match_min_microcode_rev(zen5_rdseed_microcode)) { - clear_cpu_cap(c, X86_FEATURE_RDSEED); - msr_clear_bit(MSR_AMD64_CPUID_FN_7, 18); - pr_emerg_once("RDSEED32 is broken. Disabling the corresponding CPUID bit.\n"); - } + if (!x86_match_min_microcode_rev(zen5_rdseed_microcode)) + pr_emerg_once("RDSEED32 is broken. Please update your firmware.\n"); } static void init_amd(struct cpuinfo_x86 *c) diff -ruN --no-dereference -x .git a/arch/x86/kernel/cpu/bus_lock.c b/arch/x86/kernel/cpu/bus_lock.c --- a/arch/x86/kernel/cpu/bus_lock.c 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/x86/kernel/cpu/bus_lock.c 2026-10-13 14:33:30.000000000 +0200 @@ -47,7 +47,11 @@ static struct ratelimit_state bld_ratelimit; +#ifdef CONFIG_CACHY +static unsigned int sysctl_sld_mitigate = 0; +#else static unsigned int sysctl_sld_mitigate = 1; +#endif /* CONFIG_CACHY */ static DEFINE_SEMAPHORE(buslock_sem, 1); #ifdef CONFIG_PROC_SYSCTL diff -ruN --no-dereference -x .git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c --- a/arch/x86/kernel/kvm.c 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/x86/kernel/kvm.c 2026-10-13 14:33:30.000000000 +0200 @@ -663,8 +663,10 @@ u8 state; int cpu; struct kvm_steal_time *src; - struct cpumask *flushmask = this_cpu_cpumask_var_ptr(__pv_cpu_mask); + struct cpumask *flushmask; + guard(preempt)(); + flushmask = this_cpu_cpumask_var_ptr(__pv_cpu_mask); cpumask_copy(flushmask, cpumask); /* * We have to call flush only on online vCPUs. And diff -ruN --no-dereference -x .git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c --- a/arch/x86/kernel/smpboot.c 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/x86/kernel/smpboot.c 2026-10-13 14:33:30.000000000 +0200 @@ -61,6 +61,7 @@ #include #include #include +#include #include #include @@ -1261,6 +1262,7 @@ void arch_thaw_secondary_cpus_end(void) { + clocksource_touch_watchdog(); cache_aps_init(); } @@ -1282,6 +1284,8 @@ { pr_debug("Boot done\n"); + clocksource_touch_watchdog(); + build_sched_topology(); nmi_selftest(); impress_friends(); diff -ruN --no-dereference -x .git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c --- a/arch/x86/kernel/tsc.c 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/x86/kernel/tsc.c 2026-10-13 14:33:30.000000000 +0200 @@ -52,6 +52,7 @@ static DEFINE_STATIC_KEY_FALSE_RO(__use_tsc); int tsc_clocksource_reliable; +int tsc_allow_direct_sync; static int __read_mostly tsc_force_recalibrate; @@ -334,6 +335,8 @@ tsc_force_recalibrate = 1; if (!strcmp(str, "watchdog")) tsc_watchdog = TSC_WATCHDOG_ON; + if (!strcmp(str, "directsync")) + tsc_allow_direct_sync = 1; return 1; } __setup("tsc=", tsc_setup); diff -ruN --no-dereference -x .git a/arch/x86/kernel/tsc_sync.c b/arch/x86/kernel/tsc_sync.c --- a/arch/x86/kernel/tsc_sync.c 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/x86/kernel/tsc_sync.c 2026-10-13 14:33:30.000000000 +0200 @@ -34,6 +34,8 @@ static DEFINE_PER_CPU(struct tsc_adjust, tsc_adjust); static struct timer_list tsc_sync_check_timer; +extern int tsc_allow_direct_sync; + /* * TSC's on different sockets may be reset asynchronously. * This may cause the TSC ADJUST value on socket 0 to be NOT 0. @@ -359,13 +361,16 @@ /* * Set the maximum number of test runs to - * 1 if the CPU does not provide the TSC_ADJUST MSR - * 3 if the MSR is available, so the target can try to adjust + * 5 if we can write TSC_ADJUST to compensate + * 1000 if we are allowed to write to the TSC MSR to compensate + * 1 if we cannot write MSRs to synchronize TSCs */ - if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) - atomic_set(&test_runs, 1); - else + if (boot_cpu_has(X86_FEATURE_TSC_ADJUST)) atomic_set(&test_runs, 3); + else if (tsc_allow_direct_sync) + atomic_set(&test_runs, 1000); + else + atomic_set(&test_runs, 1); retry: /* Wait for the target to start. */ while (atomic_read(&start_count) != cpus - 1) @@ -392,7 +397,7 @@ pr_debug("TSC synchronization [CPU#%d -> CPU#%u]: passed\n", smp_processor_id(), cpu); - } else if (atomic_dec_and_test(&test_runs) || random_warps) { + } else if (atomic_dec_and_test(&test_runs) || (random_warps && !tsc_allow_direct_sync)) { /* Force it to 0 if random warps brought us here */ atomic_set(&test_runs, 0); @@ -426,6 +431,21 @@ goto retry; } +static inline cycles_t write_tsc_adjustment(s64 adjustment) +{ + cycles_t adjval, nextval; + + rdmsrq(MSR_IA32_TSC, adjval); + adjval += adjustment; + wrmsrq(MSR_IA32_TSC, adjval); + rdmsrq(MSR_IA32_TSC, nextval); + + /* + * Estimated clock cycle overhead for wrmsr + rdmsr + */ + return nextval - adjval; +} + /* * Freshly booted CPUs call into this: */ @@ -433,7 +453,7 @@ { struct tsc_adjust *cur = this_cpu_ptr(&tsc_adjust); unsigned int cpu = smp_processor_id(); - cycles_t cur_max_warp, gbl_max_warp; + cycles_t cur_max_warp, gbl_max_warp, est_overhead = 0; int cpus = 2; /* Also aborts if there is no TSC. */ @@ -514,12 +534,18 @@ * value is used. In the worst case the adjustment needs to go * through a 3rd run for fine tuning. */ - cur->adjusted += cur_max_warp; + if (boot_cpu_has(X86_FEATURE_TSC_ADJUST)) { + cur->adjusted += cur_max_warp + est_overhead; - pr_warn("TSC ADJUST compensate: CPU%u observed %lld warp. Adjust: %lld\n", - cpu, cur_max_warp, cur->adjusted); + pr_warn("TSC ADJUST compensate: CPU%u observed %lld warp. Adjust: %lld\n", + cpu, cur_max_warp, cur->adjusted); - wrmsrq(MSR_IA32_TSC_ADJUST, cur->adjusted); + wrmsrq(MSR_IA32_TSC_ADJUST, cur->adjusted); + } else { + pr_debug("TSC direct sync: CPU%u observed %lld warp. Overhead: %lld\n", + cpu, cur_max_warp, est_overhead); + est_overhead = write_tsc_adjustment(cur_max_warp + est_overhead); + } goto retry; } diff -ruN --no-dereference -x .git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c --- a/arch/x86/mm/tlb.c 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/x86/mm/tlb.c 2026-10-13 14:33:30.000000000 +0200 @@ -1373,35 +1373,19 @@ */ unsigned long tlb_single_page_flush_ceiling __read_mostly = 33; -static DEFINE_PER_CPU_SHARED_ALIGNED(struct flush_tlb_info, flush_tlb_info); - -#ifdef CONFIG_DEBUG_VM -static DEFINE_PER_CPU(unsigned int, flush_tlb_info_idx); -#endif - -static struct flush_tlb_info *get_flush_tlb_info(struct mm_struct *mm, - unsigned long start, unsigned long end, - unsigned int stride_shift, bool freed_tables, - u64 new_tlb_gen) +static void init_flush_tlb_info(struct flush_tlb_info *info, + struct mm_struct *mm, + unsigned long start, unsigned long end, + unsigned int stride_shift, bool freed_tables, + u64 new_tlb_gen) { - struct flush_tlb_info *info = this_cpu_ptr(&flush_tlb_info); - -#ifdef CONFIG_DEBUG_VM - /* - * Ensure that the following code is non-reentrant and flush_tlb_info - * is not overwritten. This means no TLB flushing is initiated by - * interrupt handlers and machine-check exception handlers. - */ - BUG_ON(this_cpu_inc_return(flush_tlb_info_idx) != 1); -#endif - /* * If the number of flushes is so large that a full flush * would be faster, do a full flush. */ if ((end - start) >> stride_shift > tlb_single_page_flush_ceiling) { - start = 0; - end = TLB_FLUSH_ALL; + start = 0; + end = TLB_FLUSH_ALL; } info->start = start; @@ -1412,32 +1396,21 @@ info->new_tlb_gen = new_tlb_gen; info->initiating_cpu = smp_processor_id(); info->trim_cpumask = 0; - - return info; -} - -static void put_flush_tlb_info(void) -{ -#ifdef CONFIG_DEBUG_VM - /* Complete reentrancy prevention checks */ - barrier(); - this_cpu_dec(flush_tlb_info_idx); -#endif } void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, unsigned long end, unsigned int stride_shift, bool freed_tables) { - struct flush_tlb_info *info; + struct flush_tlb_info info; + bool remote_flush = false; int cpu = get_cpu(); u64 new_tlb_gen; /* This is also a barrier that synchronizes with switch_mm(). */ new_tlb_gen = inc_mm_tlb_gen(mm); - info = get_flush_tlb_info(mm, start, end, stride_shift, freed_tables, - new_tlb_gen); + init_flush_tlb_info(&info, mm, start, end, stride_shift, freed_tables, new_tlb_gen); /* * flush_tlb_multi() is not optimized for the common case in which only @@ -1445,20 +1418,24 @@ * flush_tlb_func_local() directly in this case. */ if (mm_global_asid(mm)) { - broadcast_tlb_flush(info); + broadcast_tlb_flush(&info); } else if (cpumask_any_but(mm_cpumask(mm), cpu) < nr_cpu_ids) { - info->trim_cpumask = should_trim_cpumask(mm); - flush_tlb_multi(mm_cpumask(mm), info); - consider_global_asid(mm); + remote_flush = true; } else if (mm == this_cpu_read(cpu_tlbstate.loaded_mm)) { lockdep_assert_irqs_enabled(); local_irq_disable(); - flush_tlb_func(info); + flush_tlb_func(&info); local_irq_enable(); } - put_flush_tlb_info(); put_cpu(); + + if (remote_flush) { + info.trim_cpumask = should_trim_cpumask(mm); + flush_tlb_multi(mm_cpumask(mm), &info); + consider_global_asid(mm); + } + mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); } @@ -1527,19 +1504,16 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end) { - struct flush_tlb_info *info; + struct flush_tlb_info info; guard(preempt)(); + init_flush_tlb_info(&info, NULL, start, end, PAGE_SHIFT, false, + TLB_GENERATION_INVALID); - info = get_flush_tlb_info(NULL, start, end, PAGE_SHIFT, false, - TLB_GENERATION_INVALID); - - if (info->end == TLB_FLUSH_ALL) - kernel_tlb_flush_all(info); + if (info.end == TLB_FLUSH_ALL) + kernel_tlb_flush_all(&info); else - kernel_tlb_flush_range(info); - - put_flush_tlb_info(); + kernel_tlb_flush_range(&info); } /* @@ -1707,12 +1681,12 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { - struct flush_tlb_info *info; - + struct flush_tlb_info info; + bool remote_flush = false; int cpu = get_cpu(); - info = get_flush_tlb_info(NULL, 0, TLB_FLUSH_ALL, 0, false, - TLB_GENERATION_INVALID); + init_flush_tlb_info(&info, NULL, 0, TLB_FLUSH_ALL, 0, false, + TLB_GENERATION_INVALID); /* * flush_tlb_multi() is not optimized for the common case in which only * a local TLB flush is needed. Optimize this use-case by calling @@ -1722,18 +1696,20 @@ invlpgb_flush_all_nonglobals(); batch->unmapped_pages = false; } else if (cpumask_any_but(&batch->cpumask, cpu) < nr_cpu_ids) { - flush_tlb_multi(&batch->cpumask, info); + remote_flush = true; } else if (cpumask_test_cpu(cpu, &batch->cpumask)) { lockdep_assert_irqs_enabled(); local_irq_disable(); - flush_tlb_func(info); + flush_tlb_func(&info); local_irq_enable(); } - cpumask_clear(&batch->cpumask); - - put_flush_tlb_info(); put_cpu(); + + if (remote_flush) + flush_tlb_multi(&batch->cpumask, &info); + + cpumask_clear(&batch->cpumask); } /* diff -ruN --no-dereference -x .git a/arch/x86/pci/common.c b/arch/x86/pci/common.c --- a/arch/x86/pci/common.c 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/x86/pci/common.c 2026-10-13 14:33:30.000000000 +0200 @@ -723,12 +723,15 @@ return 0; } -#if IS_ENABLED(CONFIG_VMD) struct pci_dev *pci_real_dma_dev(struct pci_dev *dev) { +#if IS_ENABLED(CONFIG_VMD) if (is_vmd(dev->bus)) return to_pci_sysdata(dev->bus)->vmd_dev; +#endif + + if (is_nvme_remap(dev->bus)) + return to_pci_sysdata(dev->bus)->nvme_remap_dev; return dev; } -#endif diff -ruN --no-dereference -x .git a/arch/x86/realmode/rm/wakeup.h b/arch/x86/realmode/rm/wakeup.h --- a/arch/x86/realmode/rm/wakeup.h 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/x86/realmode/rm/wakeup.h 2026-10-13 14:33:30.000000000 +0200 @@ -23,6 +23,8 @@ u64 pmode_gdt; u32 pmode_misc_en_low; /* Protected mode MISC_ENABLE */ u32 pmode_misc_en_high; + u32 pmode_tsc_low; + u32 pmode_tsc_high; u32 pmode_behavior; /* Wakeup routine behavior flags */ u32 realmode_flags; u32 real_magic; @@ -39,5 +41,6 @@ #define WAKEUP_BEHAVIOR_RESTORE_MISC_ENABLE 0 #define WAKEUP_BEHAVIOR_RESTORE_CR4 1 #define WAKEUP_BEHAVIOR_RESTORE_EFER 2 +#define WAKEUP_BEHAVIOR_RESTORE_TSC 3 #endif /* ARCH_X86_KERNEL_ACPI_RM_WAKEUP_H */ diff -ruN --no-dereference -x .git a/arch/x86/realmode/rm/wakeup_asm.S b/arch/x86/realmode/rm/wakeup_asm.S --- a/arch/x86/realmode/rm/wakeup_asm.S 2026-10-11 15:27:50.000000000 +0200 +++ b/arch/x86/realmode/rm/wakeup_asm.S 2026-10-13 14:33:30.000000000 +0200 @@ -27,6 +27,7 @@ pmode_efer: .quad 0 /* Saved EFER */ pmode_gdt: .quad 0 pmode_misc_en: .quad 0 /* Saved MISC_ENABLE MSR */ + pmode_tsc: .quad 0 /* Saved TSC MSR */ pmode_behavior: .long 0 /* Wakeup behavior flags */ realmode_flags: .long 0 real_magic: .long 0 @@ -104,6 +105,17 @@ wrmsr 1: + /* Restore TSC */ + movl pmode_behavior, %edi + btl $WAKEUP_BEHAVIOR_RESTORE_TSC, %edi + jnc 1f + + movl pmode_tsc, %eax + movl pmode_tsc + 4, %edx + movl $MSR_IA32_TSC, %ecx + wrmsr +1: + /* Do any other stuff... */ #ifndef CONFIG_64BIT diff -ruN --no-dereference -x .git a/block/Kconfig.iosched b/block/Kconfig.iosched --- a/block/Kconfig.iosched 2026-10-11 15:27:50.000000000 +0200 +++ b/block/Kconfig.iosched 2026-10-13 14:33:30.000000000 +0200 @@ -16,6 +16,20 @@ synchronous writes, it will self-tune queue depths to achieve that goal. +config MQ_IOSCHED_ADIOS + tristate "Adaptive Deadline I/O scheduler" + default m + help + The Adaptive Deadline I/O Scheduler (ADIOS) is a multi-queue I/O + scheduler with learning-based adaptive latency control. + +config MQ_IOSCHED_DEFAULT_ADIOS + bool "Enable ADIOS I/O scheduler as default MQ I/O scheduler" + depends on MQ_IOSCHED_ADIOS=y + default n + help + Enable the ADIOS I/O scheduler as the default scheduler for MQ I/O. + config IOSCHED_BFQ tristate "BFQ I/O scheduler" select BLK_ICQ diff -ruN --no-dereference -x .git a/block/Makefile b/block/Makefile --- a/block/Makefile 2026-10-11 15:27:50.000000000 +0200 +++ b/block/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -25,6 +25,7 @@ obj-$(CONFIG_BLK_CGROUP_IOCOST) += blk-iocost.o obj-$(CONFIG_MQ_IOSCHED_DEADLINE) += mq-deadline.o obj-$(CONFIG_MQ_IOSCHED_KYBER) += kyber-iosched.o +obj-$(CONFIG_MQ_IOSCHED_ADIOS) += adios.o bfq-y := bfq-iosched.o bfq-wf2q.o bfq-cgroup.o obj-$(CONFIG_IOSCHED_BFQ) += bfq.o @@ -39,3 +40,10 @@ blk-crypto-sysfs.o obj-$(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK) += blk-crypto-fallback.o obj-$(CONFIG_BLOCK_HOLDER_DEPRECATED) += holder.o + +all: + make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules + +clean: + make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean + diff -ruN --no-dereference -x .git a/block/adios.c b/block/adios.c --- a/block/adios.c 1970-01-01 01:00:00.000000000 +0100 +++ b/block/adios.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,1981 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Adaptive Deadline I/O Scheduler (ADIOS) + * Copyright (C) 2025 Masahito Suzuki + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "elevator.h" +#include "blk.h" +#include "blk-mq.h" +#include "blk-mq-sched.h" + +#define ADIOS_VERSION "3.3.0" + +/* Request Types: + * + * Tier 0 (Highest Priority): Emergency & System Integrity Requests + * ----------------------------------------------------------------- + * - Target: Requests with the BLK_MQ_INSERT_AT_HEAD flag. + * - Purpose: For critical, non-negotiable operations such as device error + * recovery or flush sequences that must bypass all other scheduling logic. + * - Implementation: Placed in a dedicated, high-priority FIFO queue + * (`prio_queue[0]`) for immediate dispatch. + * + * Tier 1 (Medium Priority): Application Responsiveness + * ---------------------------------------------------- + * - Target: Normal synchronous requests (e.g., from standard file reads). + * - Purpose: To ensure correct application behavior for operations that + * depend on sequential I/O completion (e.g., file system mounts) and to + * provide low latency for interactive applications. + * - Implementation: The deadline for these requests is set to their start + * time (`rq->start_time_ns`). This effectively enforces FIFO-like behavior + * within the deadline-sorted red-black tree, preventing out-of-order + * execution of dependent synchronous operations. + * + * Tier 2 (Normal Priority): Background Throughput + * ----------------------------------------------- + * - Target: Asynchronous requests. + * - Purpose: To maximize disk throughput for background tasks where latency + * is not critical. + * - Implementation: These are the only requests where ADIOS's adaptive + * latency prediction model is used. A dynamic deadline is calculated based + * on the predicted I/O latency, allowing for aggressive reordering to + * optimize I/O efficiency. + * + * I/O barrier guarantees (REQ_PREFLUSH / REQ_FUA) are provided entirely by + * the block layer's flush state machine (block/blk-flush.c), which is the + * mechanism that replaced whole-queue barriers upstream in 2.6.37. No + * queue-wide barrier is implemented or needed here: + * - blk_insert_flush() always strips REQ_PREFLUSH (and REQ_FUA unless the + * device supports it in hardware) before any request ever reaches this + * elevator's ->insert_requests(), and always sets REQ_SYNC on it, so such + * a request lands in Tier 1 (immediate deadline) like any other + * synchronous request -- never delayed behind Tier 2 reordering. + * - The data portion of a request that needed an actual PREFLUSH is + * resubmitted via BLK_MQ_INSERT_AT_HEAD once the flush completes, which + * lands in Tier 0 here, ahead of everything else. + * - REQ_PREFLUSH/REQ_FUA and RQF_FLUSH_SEQ requests are already excluded + * from bio/request merging by the generic block layer (rq_mergeable(), + * blk_rq_merge_ok()) before this elevator's merge callbacks ever run. + * - The synthesized pure REQ_OP_FLUSH command itself always bypasses this + * elevator entirely (blk_mq_insert_request() routes it straight to + * hctx->dispatch), so it never needs to be recognized or handled here. + * + * Dispatch Logic: + * The scheduler always dispatches requests in strict priority order: + * 1. prio_queue[0] (Tier 0) + * 2. The deadline-sorted batch queue (which naturally prioritizes Tier 1 + * over Tier 2 due to their calculated deadlines). + */ + +// Global variable to control the latency +static u64 default_global_latency_window = 16000000ULL; +static u64 default_global_latency_window_rotational = 22000000ULL; +// Ratio below which batch queues should be refilled +static u8 default_bq_refill_below_ratio = 20; +// Maximum latency sample to input +static u64 default_lat_model_latency_limit = 500 * NSEC_PER_MSEC; +// Batch ordering strategy +static u64 default_batch_order = 0; + +/* Compliance Flags: + * 0x1: Async requests will not be reordered based on the predicted latency + */ +enum adios_compliance_flags { + ADIOS_CF_FIXORDER = 1U << 0, +}; + +// Flags to control compliance with block layer constraints +static u64 default_compliance_flags = 0x0; + +// Dynamic thresholds for shrinkage +static u32 default_lm_shrink_at_kreqs = 5000; +static u32 default_lm_shrink_at_gbytes = 50; +static u32 default_lm_shrink_resist = 2; + +enum adios_optype { + ADIOS_READ = 0, + ADIOS_WRITE = 1, + ADIOS_DISCARD = 2, + ADIOS_OTHER = 3, + ADIOS_OPTYPES = 4, +}; + +// Latency targets for each operation type +static u64 default_latency_target[ADIOS_OPTYPES] = { + [ADIOS_READ] = 2ULL * NSEC_PER_MSEC, + [ADIOS_WRITE] = 2000ULL * NSEC_PER_MSEC, + [ADIOS_DISCARD] = 8000ULL * NSEC_PER_MSEC, + [ADIOS_OTHER] = 0ULL * NSEC_PER_MSEC, +}; + +// Maximum batch size limits for each operation type +static u32 default_batch_limit[ADIOS_OPTYPES] = { + [ADIOS_READ] = 36, + [ADIOS_WRITE] = 72, + [ADIOS_DISCARD] = 1, + [ADIOS_OTHER] = 1, +}; + +enum adios_batch_order { + ADIOS_BO_OPTYPE = 0, + ADIOS_BO_ELEVATOR = 1, +}; + +// Thresholds for latency model control +#define LM_BLOCK_SIZE_THRESHOLD 4096 +#define LM_SAMPLES_THRESHOLD 1024 +#define LM_INTERVAL_THRESHOLD 1500 +#define LM_OUTLIER_PERCENTILE 99 +#define LM_LAT_BUCKET_COUNT 64 + +#define ADIOS_PQ_LEVELS 2 +#define ADIOS_DL_TYPES 2 +#define ADIOS_BQ_PAGES 2 + +static u32 default_dl_prio[ADIOS_DL_TYPES] = {8, 0}; + +// Bit flags for the atomic state variable, indicating which queues have requests. +enum adios_state_flags { + ADIOS_STATE_PQ_0 = 1U << 0, + ADIOS_STATE_PQ_1 = 1U << 1, + ADIOS_STATE_DL_0 = 1U << 2, + ADIOS_STATE_DL_1 = 1U << 3, + ADIOS_STATE_BQ_PAGE_0 = 1U << 4, + ADIOS_STATE_BQ_PAGE_1 = 1U << 5, +}; +#define ADIOS_STATE_PQ 0 +#define ADIOS_STATE_DL 2 +#define ADIOS_STATE_BQ 4 + +// Temporal granularity of the deadline tree node (dl_group) +#define ADIOS_QUANTUM_SHIFT 20 + +#define ADIOS_MAX_INSERTS_PER_LOCK 72 +#define ADIOS_MAX_DELETES_PER_LOCK 24 + +// Structure to hold latency bucket data for small requests +struct latency_bucket_small { + u64 weighted_sum_latency; + u64 sum_of_weights; +}; + +// Structure to hold latency bucket data for large requests +struct latency_bucket_large { + u64 weighted_sum_latency; + u64 weighted_sum_block_size; + u64 sum_of_weights; +}; + +// Structure to hold per-cpu buckets, improving data locality and code clarity. +struct lm_buckets { + struct latency_bucket_small small_bucket[LM_LAT_BUCKET_COUNT]; + struct latency_bucket_large large_bucket[LM_LAT_BUCKET_COUNT]; +}; + +// Structure to hold RCU-protected latency model parameters +struct latency_model_params { + u64 base; + u64 slope; + u64 small_sum_delay; + u64 small_count; + u64 large_sum_delay; + u64 large_sum_bsize; + u64 last_update_jiffies; + struct rcu_head rcu; +}; + +// Structure to hold the latency model context data +struct latency_model { + spinlock_t update_lock; + struct latency_model_params __rcu *params; + + // Per-CPU buckets to avoid lock contention on the completion path + struct lm_buckets __percpu *pcpu_buckets; + // Per-CPU snapshots for delta-based aggregation (accessed under update_lock) + struct lm_buckets __percpu *pcpu_snapshot; + + u32 lm_shrink_at_kreqs; + u32 lm_shrink_at_gbytes; + u8 lm_shrink_resist; +}; + +struct adios_pcpu_completion { + u64 last_completed_time; + sector_t last_completed_pos; +}; + +union adios_in_flight_rqs { + atomic64_t atomic; + u64 scalar; + struct { + u64 count: 16; + u64 total_pred_lat: 48; + }; +}; + +// Adios scheduler data +struct adios_data { + spinlock_t pq_lock; + struct list_head prio_queue[2]; + + struct rb_root_cached dl_tree[2]; + spinlock_t lock; + s64 dl_bias; + s32 dl_prio[2]; + + atomic_t state; + u8 bq_state[ADIOS_BQ_PAGES]; + + bool models_stable; + + u64 global_latency_window; + u64 compliance_flags; + u64 latency_target[ADIOS_OPTYPES]; + u32 batch_limit[ADIOS_OPTYPES]; + u32 batch_actual_max_size[ADIOS_OPTYPES]; + u32 batch_actual_max_total; + u32 async_depth; + u32 lat_model_latency_limit; + u8 bq_refill_below_ratio; + u8 is_rotational; + u8 batch_order; + u8 elv_direction; + sector_t head_pos; + + bool bq_page; + struct list_head batch_queue[ADIOS_BQ_PAGES][ADIOS_OPTYPES]; + u32 batch_count[ADIOS_BQ_PAGES][ADIOS_OPTYPES]; + u8 bq_batch_order[ADIOS_BQ_PAGES]; + spinlock_t bq_lock; + + struct lm_buckets *aggr_buckets; + + struct latency_model latency_model[ADIOS_OPTYPES]; + struct timer_list update_timer; + + union adios_in_flight_rqs in_flight_rqs; + atomic64_t total_pred_lat; + + struct adios_pcpu_completion __percpu *pcpu_completion; + + struct kmem_cache *rq_data_cache; + mempool_t *rq_data_pool; + struct kmem_cache *dl_group_pool; + + struct request_queue *queue; +}; + +// List of requests with the same deadline in the deadline-sorted tree +struct dl_group { + struct rb_node node; + struct list_head rqs; + u64 deadline; +} __attribute__((aligned(64))); + +// Structure to hold scheduler-specific data for each request +struct adios_rq_data { + struct list_head *dl_group; + struct list_head dl_node; + + struct request *rq; + u64 deadline; + u64 pred_lat; + u32 block_size; + bool managed; +} __attribute__((aligned(64))); + +static const int adios_prio_to_wmult[40] = { + /* -20 */ 88761, 71755, 56483, 46273, 36291, + /* -15 */ 29154, 23254, 18705, 14949, 11916, + /* -10 */ 9548, 7620, 6100, 4904, 3906, + /* -5 */ 3121, 2501, 1991, 1586, 1277, + /* 0 */ 1024, 820, 655, 526, 423, + /* 5 */ 335, 272, 215, 172, 137, + /* 10 */ 110, 87, 70, 56, 45, + /* 15 */ 36, 29, 23, 18, 15, +}; + +static inline bool compliant(struct adios_data *ad, u32 flag) { + return ad->compliance_flags & flag; +} + +// Count the number of entries in aggregated small buckets +static u64 lm_count_small_entries(struct latency_bucket_small *buckets) { + u64 total_weight = 0; + for (u8 i = 0; i < LM_LAT_BUCKET_COUNT; i++) + total_weight += buckets[i].sum_of_weights; + return total_weight; +} + +// Update the small buckets in the latency model from aggregated data +static bool lm_update_small_buckets(struct latency_model *model, + struct latency_model_params *params, + struct latency_bucket_small *buckets, + u64 total_weight, bool count_all) { + u64 sum_latency = 0; + u64 sum_weight = 0; + u64 cumulative_weight = 0, threshold_weight = 0; + u8 outlier_threshold_bucket = 0; + u8 outlier_percentile = LM_OUTLIER_PERCENTILE; + u8 reduction; + + if (count_all) + outlier_percentile = 100; + + // Calculate the threshold weight for outlier detection + threshold_weight = (total_weight * outlier_percentile) / 100; + + // Identify the bucket that corresponds to the outlier threshold + for (u8 i = 0; i < LM_LAT_BUCKET_COUNT; i++) { + cumulative_weight += buckets[i].sum_of_weights; + if (cumulative_weight >= threshold_weight) { + outlier_threshold_bucket = i; + break; + } + } + + // Calculate the average latency, excluding outliers + for (u8 i = 0; i <= outlier_threshold_bucket; i++) { + struct latency_bucket_small *bucket = &buckets[i]; + if (i < outlier_threshold_bucket) { + sum_latency += bucket->weighted_sum_latency; + sum_weight += bucket->sum_of_weights; + } else { + // The threshold bucket's contribution is proportional + u64 remaining_weight = + threshold_weight - (cumulative_weight - bucket->sum_of_weights); + if (bucket->sum_of_weights > 0) { + sum_latency += div_u64(bucket->weighted_sum_latency * + remaining_weight, bucket->sum_of_weights); + sum_weight += remaining_weight; + } + } + } + + // Shrink the model if it reaches at the readjustment threshold + if (params->small_count >= 1000ULL * model->lm_shrink_at_kreqs) { + reduction = model->lm_shrink_resist; + if (params->small_count >> reduction) { + params->small_sum_delay -= params->small_sum_delay >> reduction; + params->small_count -= params->small_count >> reduction; + } + } + + if (!sum_weight) + return false; + + // Accumulate the average latency into the statistics + params->small_sum_delay += sum_latency; + params->small_count += sum_weight; + + return true; +} + +// Count the number of entries in aggregated large buckets +static u64 lm_count_large_entries(struct latency_bucket_large *buckets) { + u64 total_weight = 0; + for (u8 i = 0; i < LM_LAT_BUCKET_COUNT; i++) + total_weight += buckets[i].sum_of_weights; + return total_weight; +} + +// Update the large buckets in the latency model from aggregated data +static bool lm_update_large_buckets(struct latency_model *model, + struct latency_model_params *params, + struct latency_bucket_large *buckets, + u64 total_weight, bool count_all) { + s64 sum_latency = 0; + u64 sum_block_size = 0, intercept; + u64 cumulative_weight = 0, threshold_weight = 0; + u64 sum_weight = 0; + u8 outlier_threshold_bucket = 0; + u8 outlier_percentile = LM_OUTLIER_PERCENTILE; + u8 reduction; + + if (count_all) + outlier_percentile = 100; + + // Calculate the threshold weight for outlier detection + threshold_weight = (total_weight * outlier_percentile) / 100; + + // Identify the bucket that corresponds to the outlier threshold + for (u8 i = 0; i < LM_LAT_BUCKET_COUNT; i++) { + cumulative_weight += buckets[i].sum_of_weights; + if (cumulative_weight >= threshold_weight) { + outlier_threshold_bucket = i; + break; + } + } + + // Calculate the average latency and block size, excluding outliers + for (u8 i = 0; i <= outlier_threshold_bucket; i++) { + struct latency_bucket_large *bucket = &buckets[i]; + if (i < outlier_threshold_bucket) { + sum_latency += bucket->weighted_sum_latency; + sum_block_size += bucket->weighted_sum_block_size; + sum_weight += bucket->sum_of_weights; + } else { + // The threshold bucket's contribution is proportional + u64 remaining_weight = + threshold_weight - (cumulative_weight - bucket->sum_of_weights); + if (bucket->sum_of_weights > 0) { + sum_latency += div_u64(bucket->weighted_sum_latency * + remaining_weight, bucket->sum_of_weights); + sum_block_size += div_u64(bucket->weighted_sum_block_size * + remaining_weight, bucket->sum_of_weights); + sum_weight += remaining_weight; + } + } + } + + if (!sum_weight) + return false; + + // Shrink the model if it reaches at the readjustment threshold + if (params->large_sum_bsize >= 0x40000000ULL * model->lm_shrink_at_gbytes) { + reduction = model->lm_shrink_resist; + if (params->large_sum_bsize >> reduction) { + params->large_sum_delay -= params->large_sum_delay >> reduction; + params->large_sum_bsize -= params->large_sum_bsize >> reduction; + } + } + + // Accumulate the average delay into the statistics + intercept = params->base; + if (sum_latency > intercept) + sum_latency -= intercept; + + params->large_sum_delay += sum_latency; + params->large_sum_bsize += sum_block_size; + + return true; +} + +static void reset_buckets(struct lm_buckets *buckets) +{ memset(buckets, 0, sizeof(*buckets)); } + +static void lm_reset_pcpu_buckets(struct latency_model *model) { + int cpu; + for_each_possible_cpu(cpu) { + reset_buckets(per_cpu_ptr(model->pcpu_buckets, cpu)); + reset_buckets(per_cpu_ptr(model->pcpu_snapshot, cpu)); + } +} + +// Update the latency model parameters and statistics +static void latency_model_update( + struct adios_data *ad, struct latency_model *model) { + u64 now; + u64 small_weight, large_weight; + bool time_elapsed; + bool small_processed = false, large_processed = false; + struct lm_buckets *aggr = ad->aggr_buckets; + struct latency_bucket_small *asb; + struct latency_bucket_large *alb; + struct lm_buckets *pcpu_b, *snap; + unsigned long flags; + int cpu; + struct latency_model_params *old_params, *new_params; + + spin_lock_irqsave(&model->update_lock, flags); + + old_params = rcu_dereference_protected(model->params, + lockdep_is_held(&model->update_lock)); + new_params = kmemdup(old_params, sizeof(*new_params), GFP_ATOMIC); + if (!new_params) { + spin_unlock_irqrestore(&model->update_lock, flags); + return; + } + + // Aggregate deltas from all CPUs using snapshot-delta method. + // Per-CPU counters increase monotonically; we compute delta = current - snapshot. + for_each_possible_cpu(cpu) { + pcpu_b = per_cpu_ptr(model->pcpu_buckets, cpu); + snap = per_cpu_ptr(model->pcpu_snapshot, cpu); + + for (u8 i = 0; i < LM_LAT_BUCKET_COUNT; i++) { + u64 sw = pcpu_b->small_bucket[i].sum_of_weights; + u64 sl = pcpu_b->small_bucket[i].weighted_sum_latency; + u64 dsw = sw - snap->small_bucket[i].sum_of_weights; + u64 dsl = sl - snap->small_bucket[i].weighted_sum_latency; + snap->small_bucket[i].sum_of_weights = sw; + snap->small_bucket[i].weighted_sum_latency = sl; + if (dsw) { + asb = &aggr->small_bucket[i]; + asb->sum_of_weights += dsw; + asb->weighted_sum_latency += dsl; + } + + u64 lw = pcpu_b->large_bucket[i].sum_of_weights; + u64 ll = pcpu_b->large_bucket[i].weighted_sum_latency; + u64 lb = pcpu_b->large_bucket[i].weighted_sum_block_size; + u64 dlw = lw - snap->large_bucket[i].sum_of_weights; + u64 dll = ll - snap->large_bucket[i].weighted_sum_latency; + u64 dlb = lb - snap->large_bucket[i].weighted_sum_block_size; + snap->large_bucket[i].sum_of_weights = lw; + snap->large_bucket[i].weighted_sum_latency = ll; + snap->large_bucket[i].weighted_sum_block_size = lb; + if (dlw) { + alb = &aggr->large_bucket[i]; + alb->sum_of_weights += dlw; + alb->weighted_sum_latency += dll; + alb->weighted_sum_block_size += dlb; + } + } + } + + // Count the number of entries in aggregated buckets + small_weight = lm_count_small_entries(aggr->small_bucket); + large_weight = lm_count_large_entries(aggr->large_bucket); + + // Whether enough time has elapsed since the last update + now = jiffies; + time_elapsed = unlikely(!new_params->base) || + new_params->last_update_jiffies + + msecs_to_jiffies(LM_INTERVAL_THRESHOLD) <= now; + + // Update small buckets + if (small_weight && (time_elapsed || + LM_SAMPLES_THRESHOLD <= small_weight || !new_params->base)) { + small_processed = lm_update_small_buckets(model, new_params, + aggr->small_bucket, small_weight, !new_params->base); + memset(&aggr->small_bucket[0], 0, sizeof(aggr->small_bucket)); + } + // Update large buckets + if (large_weight && (time_elapsed || + LM_SAMPLES_THRESHOLD <= large_weight || !new_params->slope)) { + large_processed = lm_update_large_buckets(model, new_params, + aggr->large_bucket, large_weight, !new_params->slope); + memset(&aggr->large_bucket[0], 0, sizeof(aggr->large_bucket)); + } + + // Update the base parameter if small bucket was processed + if (small_processed && likely(new_params->small_count)) + new_params->base = div_u64(new_params->small_sum_delay, + new_params->small_count); + + // Update the slope parameter if large bucket was processed + if (large_processed && likely(new_params->large_sum_bsize)) + new_params->slope = div_u64(new_params->large_sum_delay, + DIV_ROUND_UP_ULL(new_params->large_sum_bsize, 1024)); + + // Update last updated jiffies if update happened or time has elapsed + if (small_processed || large_processed || time_elapsed) + new_params->last_update_jiffies = now; + + rcu_assign_pointer(model->params, new_params); + spin_unlock_irqrestore(&model->update_lock, flags); + + kfree_rcu(old_params, rcu); +} + +// Determine the bucket index for a given measured and predicted latency +static u8 lm_input_bucket_index(u64 measured, u64 predicted) { + u32 bucket_index; + + if (measured < predicted * 2) + bucket_index = div_u64((measured * 20), predicted); + else if (measured < predicted * 5) + bucket_index = div_u64((measured * 10), predicted) + 20; + else + bucket_index = div_u64((measured * 3), predicted) + 40; + + if (bucket_index >= LM_LAT_BUCKET_COUNT) + bucket_index = LM_LAT_BUCKET_COUNT - 1; + + return (u8)bucket_index; +} + +// Input latency data into the latency model +static void latency_model_input(struct adios_data *ad, + struct latency_model *model, + u32 block_size, u64 latency, u64 pred_lat, u32 weight) { + unsigned long flags; + u8 bucket_index; + struct lm_buckets *buckets; + u64 current_base; + struct latency_model_params *params; + + local_irq_save(flags); + buckets = per_cpu_ptr(model->pcpu_buckets, __smp_processor_id()); + + rcu_read_lock(); + params = rcu_dereference(model->params); + current_base = params->base; + rcu_read_unlock(); + + if (block_size <= LM_BLOCK_SIZE_THRESHOLD) { + // Handle small requests + bucket_index = lm_input_bucket_index(latency, current_base ?: 1); + + buckets->small_bucket[bucket_index].sum_of_weights += weight; + buckets->small_bucket[bucket_index].weighted_sum_latency += + latency * weight; + + local_irq_restore(flags); + + if (unlikely(!current_base)) { + latency_model_update(ad, model); + return; + } + } else { + // Handle large requests + if (!current_base || !pred_lat) { + local_irq_restore(flags); + return; + } + + bucket_index = lm_input_bucket_index(latency, pred_lat); + + buckets->large_bucket[bucket_index].sum_of_weights += weight; + buckets->large_bucket[bucket_index].weighted_sum_latency += + latency * weight; + buckets->large_bucket[bucket_index].weighted_sum_block_size += + block_size * weight; + + local_irq_restore(flags); + } +} + +// Predict the latency for a given block size using the latency model +static u64 latency_model_predict(struct latency_model *model, u32 block_size) { + u64 result; + struct latency_model_params *params; + + rcu_read_lock(); + params = rcu_dereference(model->params); + + result = params->base; + if (block_size > LM_BLOCK_SIZE_THRESHOLD) + result += params->slope * + DIV_ROUND_UP_ULL(block_size - LM_BLOCK_SIZE_THRESHOLD, 1024); + + rcu_read_unlock(); + + return result; +} + +// Determine the type of operation based on request flags +static u8 adios_optype(struct request *rq) { + switch (rq->cmd_flags & REQ_OP_MASK) { + case REQ_OP_READ: + return ADIOS_READ; + case REQ_OP_WRITE: + return ADIOS_WRITE; + case REQ_OP_DISCARD: + return ADIOS_DISCARD; + default: + return ADIOS_OTHER; + } +} + +static inline u8 adios_optype_not_read(struct request *rq) { + return (rq->cmd_flags & REQ_OP_MASK) != REQ_OP_READ; +} + +// Helper function to retrieve adios_rq_data from a request +static inline struct adios_rq_data *get_rq_data(struct request *rq) { + return rq->elv.priv[0]; +} + +static inline +void set_adios_state(struct adios_data *ad, u32 shift, u32 idx, bool flag) { + if (flag) + atomic_or(1U << (idx + shift), &ad->state); + else + atomic_andnot(1U << (idx + shift), &ad->state); +} + +static inline u32 get_adios_state(struct adios_data *ad) +{ return atomic_read(&ad->state); } + +static inline u32 eval_this_adios_state(u32 state, u32 shift) +{ return (state >> shift) & 0x3; } + +static inline u32 eval_adios_state(struct adios_data *ad, u32 shift) +{ return eval_this_adios_state(get_adios_state(ad), shift); } + +// Add a request to the deadline-sorted red-black tree +static void add_to_dl_tree( + struct adios_data *ad, bool dl_idx, struct request *rq) { + struct rb_root_cached *root = &ad->dl_tree[dl_idx]; + struct rb_node **link = &(root->rb_root.rb_node), *parent = NULL; + bool leftmost = true; + struct adios_rq_data *rd = get_rq_data(rq); + struct dl_group *dlg; + u64 deadline; + bool was_empty = RB_EMPTY_ROOT(&root->rb_root); + + /* Tier-2: Synchronous Requests + * - Needs to be FIFO within a same optype + * - Relaxed order between different optypes + * - basically needs to be processed in early time */ + rd->deadline = rq->start_time_ns; + + /* Tier-3: Aynchronous Requests + * - Can be reordered and delayed freely */ + if (!(rq->cmd_flags & REQ_SYNC)) { + rd->deadline += ad->latency_target[adios_optype(rq)]; + if (!compliant(ad, ADIOS_CF_FIXORDER)) + rd->deadline += rd->pred_lat; + } + + // Now quantize the deadline (-> dlg->deadline == RB-Tree key) + deadline = rd->deadline & ~((1ULL << ADIOS_QUANTUM_SHIFT) - 1); + + while (*link) { + dlg = rb_entry(*link, struct dl_group, node); + s64 diff = deadline - dlg->deadline; + + parent = *link; + if (diff < 0) { + link = &((*link)->rb_left); + } else if (diff > 0) { + link = &((*link)->rb_right); + leftmost = false; + } else { // diff == 0 + goto found; + } + } + + dlg = rb_entry_safe(parent, struct dl_group, node); + if (!dlg || dlg->deadline != deadline) { + dlg = kmem_cache_zalloc(ad->dl_group_pool, GFP_ATOMIC); + if (!dlg) + return; + dlg->deadline = deadline; + INIT_LIST_HEAD(&dlg->rqs); + rb_link_node(&dlg->node, parent, link); + rb_insert_color_cached(&dlg->node, root, leftmost); + } +found: + list_add_tail(&rd->dl_node, &dlg->rqs); + rd->dl_group = &dlg->rqs; + + if (was_empty) + set_adios_state(ad, ADIOS_STATE_DL, dl_idx, true); +} + +// Remove a request from the deadline-sorted red-black tree +static void del_from_dl_tree( + struct adios_data *ad, bool dl_idx, struct request *rq) { + struct rb_root_cached *root = &ad->dl_tree[dl_idx]; + struct adios_rq_data *rd = get_rq_data(rq); + struct dl_group *dlg = container_of(rd->dl_group, struct dl_group, rqs); + + list_del_init(&rd->dl_node); + if (list_empty(&dlg->rqs)) { + rb_erase_cached(&dlg->node, root); + kmem_cache_free(ad->dl_group_pool, dlg); + } + rd->dl_group = NULL; + + if (RB_EMPTY_ROOT(&ad->dl_tree[dl_idx].rb_root)) + set_adios_state(ad, ADIOS_STATE_DL, dl_idx, false); +} + +// Remove a request from the scheduler +static void remove_request(struct adios_data *ad, struct request *rq) { + bool dl_idx = adios_optype_not_read(rq); + struct request_queue *q = rq->q; + struct adios_rq_data *rd = get_rq_data(rq); + + list_del_init(&rq->queuelist); + + // We might not be on the rbtree, if we are doing an insert merge + if (rd->dl_group) + del_from_dl_tree(ad, dl_idx, rq); + + elv_rqhash_del(q, rq); + if (q->last_merge == rq) + q->last_merge = NULL; +} + +// We limit the depth of request allocation for asynchronous and write requests +static void adios_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) { + struct adios_data *ad = data->q->elevator->elevator_data; + + // Do not throttle synchronous reads + if (op_is_sync(opf) && !op_is_write(opf)) + return; + + /* + * sbitmap's __map_depth_with_shallow() has taken shallow_depth in + * whole-queue units since 6.12 (the same release mq-deadline's + * dd_to_word_depth() was dropped) and does the per-word division + * itself. Passing an already per-word value here (the old + * to_word_depth() helper) got it squared away: word_depth^2 / + * sb->depth, silently capping async/write allocation at <= 64 tags + * regardless of async_depth. Pass the raw queue-wide depth, matching + * mq-deadline/kyber. + */ + data->shallow_depth = ad->async_depth; +} + +// The number of requests in the queue was notified from the block layer +static void adios_depth_updated(struct request_queue *q) { + struct adios_data *ad = q->elevator->elevator_data; + + ad->async_depth = q->nr_requests; + blk_mq_set_min_shallow_depth(q, ad->async_depth); +} + +// Handle request merging after a merge operation +static void adios_request_merged(struct request_queue *q, struct request *req, + enum elv_merge type) { + bool dl_idx = adios_optype_not_read(req); + struct adios_data *ad = q->elevator->elevator_data; + + // Reposition request in the deadline-sorted tree + del_from_dl_tree(ad, dl_idx, req); + add_to_dl_tree(ad, dl_idx, req); +} + +// Handle merging of requests after one has been merged into another +static void adios_merged_requests(struct request_queue *q, struct request *req, + struct request *next) { + struct adios_data *ad = q->elevator->elevator_data; + + lockdep_assert_held(&ad->lock); + + // kill knowledge of next, this one is a goner + remove_request(ad, next); +} + +// Try to merge a bio into an existing rq before associating it with an rq +static bool adios_bio_merge(struct request_queue *q, struct bio *bio, + unsigned int nr_segs) { + unsigned long flags; + struct adios_data *ad = q->elevator->elevator_data; + struct request *free = NULL; + bool ret; + + if (!spin_trylock_irqsave(&ad->lock, flags)) + return false; + + ret = blk_mq_sched_try_merge(q, bio, nr_segs, &free); + spin_unlock_irqrestore(&ad->lock, flags); + + if (free) + blk_mq_free_request(free); + + return ret; +} + +static bool merge_or_insert_to_dl_tree(struct adios_data *ad, + struct request *rq, struct request_queue *q, struct list_head *free) { + if (blk_mq_sched_try_insert_merge(q, rq, free)) + return true; + + bool dl_idx = adios_optype_not_read(rq); + add_to_dl_tree(ad, dl_idx, rq); + + if (rq_mergeable(rq)) { + elv_rqhash_add(q, rq); + if (!q->last_merge) + q->last_merge = rq; + } + + return false; +} + +static void insert_to_prio_queue(struct adios_data *ad, + struct request *rq, bool pq_idx) { + struct adios_rq_data *rd = get_rq_data(rq); + + /* We're sure that rd->managed == true */ + union adios_in_flight_rqs ifr = { + .count = 1, + .total_pred_lat = rd->pred_lat, + }; + atomic64_add(ifr.scalar, &ad->in_flight_rqs.atomic); + + scoped_guard(spinlock_irqsave, &ad->pq_lock) { + bool was_empty = list_empty(&ad->prio_queue[pq_idx]); + list_add_tail(&rq->queuelist, &ad->prio_queue[pq_idx]); + if (was_empty) + set_adios_state(ad, ADIOS_STATE_PQ, pq_idx, true); + } +} + +// Insert a request into the scheduler (after Read & Write models stabilized) +static void insert_request_post_stability(struct blk_mq_hw_ctx *hctx, + struct request *rq, blk_insert_t insert_flags, struct list_head *free) { + struct request_queue *q = hctx->queue; + struct adios_data *ad = q->elevator->elevator_data; + struct adios_rq_data *rd = get_rq_data(rq); + u8 optype = adios_optype(rq); + + rd->managed = true; + rd->block_size = blk_rq_bytes(rq); + rd->pred_lat = + latency_model_predict(&ad->latency_model[optype], rd->block_size); + if (unlikely(rd->pred_lat > ad->lat_model_latency_limit)) + rd->pred_lat = ad->lat_model_latency_limit; + + /* Tier-0: BLK_MQ_INSERT_AT_HEAD Requests */ + if (insert_flags & BLK_MQ_INSERT_AT_HEAD) { + insert_to_prio_queue(ad, rq, 0); + return; + } + + if (merge_or_insert_to_dl_tree(ad, rq, q, free)) + return; +} + +// Insert a request into the scheduler (before Read & Write models stabilizes) +static void insert_request_pre_stability(struct blk_mq_hw_ctx *hctx, + struct request *rq, blk_insert_t insert_flags, struct list_head *free) { + struct adios_data *ad = hctx->queue->elevator->elevator_data; + struct adios_rq_data *rd = get_rq_data(rq); + u8 optype = adios_optype(rq); + u8 pq_idx = !(insert_flags & BLK_MQ_INSERT_AT_HEAD); + bool stable = false; + + rd->managed = true; + rd->block_size = blk_rq_bytes(rq); + rd->pred_lat = + latency_model_predict(&ad->latency_model[optype], rd->block_size); + if (unlikely(rd->pred_lat > ad->lat_model_latency_limit)) + rd->pred_lat = ad->lat_model_latency_limit; + + insert_to_prio_queue(ad, rq, pq_idx); + + rcu_read_lock(); + if (rcu_dereference(ad->latency_model[ADIOS_READ].params)->base > 0 && + rcu_dereference(ad->latency_model[ADIOS_WRITE].params)->base > 0) + stable = true; + rcu_read_unlock(); + + if (stable) + ad->models_stable = true; +} + +// Insert multiple requests into the scheduler +static void adios_insert_requests(struct blk_mq_hw_ctx *hctx, + struct list_head *list, + blk_insert_t insert_flags) { + struct request_queue *q = hctx->queue; + struct adios_data *ad = q->elevator->elevator_data; + struct request *rq; + bool stop = false; + LIST_HEAD(free); + + do { + scoped_guard(spinlock_irqsave, &ad->lock) + for (int i = 0; i < ADIOS_MAX_INSERTS_PER_LOCK; i++) { + if (list_empty(list)) { + stop = true; + break; + } + rq = list_first_entry(list, struct request, queuelist); + list_del_init(&rq->queuelist); + if (likely(ad->models_stable)) + insert_request_post_stability(hctx, rq, insert_flags, &free); + else + insert_request_pre_stability(hctx, rq, insert_flags, &free); + }} while (!stop); + + blk_mq_free_requests(&free); +} + +// Prepare a request before it is inserted into the scheduler +static void adios_prepare_request(struct request *rq) { + struct adios_data *ad = rq->q->elevator->elevator_data; + struct adios_rq_data *rd; + + rd = mempool_alloc(ad->rq_data_pool, GFP_ATOMIC); + memset(rd, 0, sizeof(*rd)); + rd->rq = rq; + rq->elv.priv[0] = rd; +} + +static struct adios_rq_data *get_dl_first_rd(struct adios_data *ad, bool idx) { + struct rb_root_cached *root = &ad->dl_tree[idx]; + struct rb_node *first = rb_first_cached(root); + struct dl_group *dl_group = rb_entry(first, struct dl_group, node); + + return list_first_entry(&dl_group->rqs, struct adios_rq_data, dl_node); +} + +// Comparison function for sorting requests by block address +static int cmp_rq_pos(void *priv, + const struct list_head *a, const struct list_head *b) { + struct request *rq_a = list_entry(a, struct request, queuelist); + struct request *rq_b = list_entry(b, struct request, queuelist); + u64 pos_a = blk_rq_pos(rq_a); + u64 pos_b = blk_rq_pos(rq_b); + + return (int)(pos_a > pos_b) - (int)(pos_a < pos_b); +} + +#ifndef list_last_entry_or_null +#define list_last_entry_or_null(ptr, type, member) \ + (!list_empty(ptr) ? list_last_entry(ptr, type, member) : NULL) +#endif + +// Update the elevator direction +static void update_elv_direction(struct adios_data *ad) { + if (!ad->is_rotational) + return; + + bool page = ad->bq_page; + struct list_head *q = &ad->batch_queue[page][1]; + if (ad->bq_batch_order[page] < ADIOS_BO_ELEVATOR || list_empty(q)) { + ad->elv_direction = 0; + return; + } + + // Get first and last request positions in the queue + struct request *rq_a = list_first_entry(q, struct request, queuelist); + struct request *rq_b = list_last_entry (q, struct request, queuelist); + u64 pos_a = blk_rq_pos(rq_a); + u64 pos_b = blk_rq_pos(rq_b); + u64 avg_rq_pos = (pos_a + pos_b) >> 1; + + ad->elv_direction = !!(ad->head_pos > avg_rq_pos); +} + +// Fill the batch queues with requests from the deadline-sorted red-black tree +static bool fill_batch_queues(struct adios_data *ad, u64 tpl) { + struct adios_rq_data *rd; + struct request *rq; + struct list_head *dest_q; + u8 dest_idx; + u64 added_lat = 0; + u32 optype_count[ADIOS_OPTYPES] = {0}; + u32 count = 0; + u8 optype; + bool page = !ad->bq_page, dl_idx, bias_idx, update_bias; + u32 dl_queued; + u8 bq_batch_order; + bool stop = false; + + // Reset batch queue counts for the back page + memset(&ad->batch_count[page], 0, sizeof(ad->batch_count[page])); + + ad->bq_batch_order[page] = + bq_batch_order = ad->batch_order; + + do { + scoped_guard(spinlock_irqsave, &ad->lock) + for (int i = 0; i < ADIOS_MAX_DELETES_PER_LOCK; i++) { + bool has_base = false; + + dl_queued = eval_adios_state(ad, ADIOS_STATE_DL); + // Check if there are any requests queued in the deadline tree + if (!dl_queued) { + stop = true; + break; + } + + // Reads if both queues have requests, otherwise pick the non-empty. + dl_idx = dl_queued >> 1; + + // Get the first request from the deadline-sorted tree + rd = get_dl_first_rd(ad, dl_idx); + + bias_idx = ad->dl_bias < 0; + // If read and write requests are queued, choose one based on bias + if (dl_queued == 0x3) { + struct adios_rq_data *trd[2] = {get_dl_first_rd(ad, 0), rd}; + rd = trd[bias_idx]; + + update_bias = (trd[bias_idx]->deadline > trd[!bias_idx]->deadline); + } else + update_bias = (bias_idx == dl_idx); + + rq = rd->rq; + optype = adios_optype(rq); + + rcu_read_lock(); + has_base = + !!rcu_dereference(ad->latency_model[optype].params)->base; + rcu_read_unlock(); + + // Check batch size and total predicted latency + if (count && (!has_base || + ad->batch_count[page][optype] >= ad->batch_limit[optype] || + (tpl + added_lat + rd->pred_lat) > ad->global_latency_window)) { + stop = true; + break; + } + + if (update_bias) { + s64 sign = ((s64)bias_idx << 1) - 1; + if (unlikely(!rd->pred_lat)) + ad->dl_bias = sign; + else + // Adjust the bias based on the predicted latency + ad->dl_bias += sign * (s64)((rd->pred_lat * + adios_prio_to_wmult[ad->dl_prio[bias_idx] + 20]) >> 10); + } + + remove_request(ad, rq); + + // Add request to the corresponding batch queue + dest_idx = (bq_batch_order == ADIOS_BO_OPTYPE || optype == ADIOS_OTHER)? + optype : !!(rd->deadline != rq->start_time_ns); + dest_q = &ad->batch_queue[page][dest_idx]; + list_add_tail(&rq->queuelist, dest_q); + ad->bq_state[page] |= 1U << dest_idx; + ad->batch_count[page][optype]++; + optype_count[optype]++; + added_lat += rd->pred_lat; + count++; + }} while (!stop); + + if (bq_batch_order == ADIOS_BO_ELEVATOR && ad->batch_count[page][1] > 1) + list_sort(NULL, &ad->batch_queue[page][1], cmp_rq_pos); + + if (count) { + /* We're sure that every request's rd->managed == true */ + union adios_in_flight_rqs ifr = { + .count = count, + .total_pred_lat = added_lat, + }; + atomic64_add(ifr.scalar, &ad->in_flight_rqs.atomic); + + set_adios_state(ad, ADIOS_STATE_BQ, page, true); + + for (optype = 0; optype < ADIOS_OPTYPES; optype++) + if (ad->batch_actual_max_size[optype] < optype_count[optype]) + ad->batch_actual_max_size[optype] = optype_count[optype]; + if (ad->batch_actual_max_total < count) + ad->batch_actual_max_total = count; + } + return count; +} + +// Flip to the next batch queue page +static void flip_bq_page(struct adios_data *ad) { + ad->bq_page = !ad->bq_page; + update_elv_direction(ad); +} + +// Pop a request from the specified index (optype or elevator tier) +static inline struct request *pop_bq_request( + struct adios_data *ad, u8 idx, bool direction) { + bool page = ad->bq_page; + struct list_head *q = &ad->batch_queue[page][idx]; + struct request *rq = direction ? + list_last_entry_or_null (q, struct request, queuelist): + list_first_entry_or_null(q, struct request, queuelist); + if (rq) { + list_del_init(&rq->queuelist); + if (list_empty(q)) + ad->bq_state[page] &= ~(1U << idx); + } + return rq; +} + +static struct request *pop_next_bq_request_optype(struct adios_data *ad) { + u32 bq_state = ad->bq_state[ad->bq_page]; + if (!bq_state) return NULL; + + struct request *rq; + u32 bq_idx = __builtin_ctz(bq_state); + + // Dispatch based on optype (FIFO within each) or single-queue elevator + rq = pop_bq_request(ad, bq_idx, false); + return rq; +} + +static struct request *pop_next_bq_request_elevator(struct adios_data *ad) { + u32 bq_state = ad->bq_state[ad->bq_page]; + if (!bq_state) return NULL; + + struct request *rq; + u32 bq_idx = __builtin_ctz(bq_state); + bool direction = (bq_idx == 1) & ad->elv_direction; + + // Tier-2 (sync) is always high priority + // Tier-3 (async) uses the pre-calculated elevator direction + rq = pop_bq_request(ad, bq_idx, direction); + + /* If batch queue for the sync requests just became empty */ + if (bq_idx == 0 && rq && !(bq_state & 0x1)) + update_elv_direction(ad); + + return rq; +} + +// Returns the state of the batch queue page +static inline bool bq_page_has_rq(u32 bq_state, bool page) { + return bq_state & (1U << page); +} + +// Dispatch a request from the batch queues +static struct request *dispatch_from_bq(struct adios_data *ad) { + struct request *rq; + + guard(spinlock_irqsave)(&ad->bq_lock); + + u32 state = get_adios_state(ad); + u32 bq_state = eval_this_adios_state(state, ADIOS_STATE_BQ); + u32 bq_curr_page_has_rq = bq_page_has_rq(bq_state, ad->bq_page); + union adios_in_flight_rqs ifr; + ifr.scalar = atomic64_read(&ad->in_flight_rqs.atomic); + u64 tpl = ifr.total_pred_lat; + + // Refill the batch queues if the back page is empty, dl_tree has work, and + // current page is empty or the total ongoing latency is below the threshold + if (!bq_page_has_rq(bq_state, !ad->bq_page) && + (!bq_curr_page_has_rq || (!tpl || tpl < div_u64( + ad->global_latency_window * ad->bq_refill_below_ratio, 100))) && + eval_this_adios_state(state, ADIOS_STATE_DL)) + fill_batch_queues(ad, tpl); + + // If current batch queue page is empty, and the other page has work, flip + if (!bq_curr_page_has_rq && + bq_page_has_rq(eval_adios_state(ad, ADIOS_STATE_BQ), !ad->bq_page)) + flip_bq_page(ad); + + // Use the per-page state to decide the dispatch logic, ensuring correctness + rq = (ad->bq_batch_order[ad->bq_page] == ADIOS_BO_ELEVATOR) ? + pop_next_bq_request_elevator(ad): + pop_next_bq_request_optype(ad); + + if (rq) { + bool page = ad->bq_page; + bool is_empty = !ad->bq_state[page]; + if (is_empty) + set_adios_state(ad, ADIOS_STATE_BQ, page, false); + return rq; + } + + return NULL; +} + +// Dispatch a request from the priority queue +static struct request *dispatch_from_pq(struct adios_data *ad) { + struct request *rq = NULL; + + guard(spinlock_irqsave)(&ad->pq_lock); + u32 pq_state = eval_adios_state(ad, ADIOS_STATE_PQ); + u8 pq_idx = pq_state >> 1; + struct list_head *q = &ad->prio_queue[pq_idx]; + + if (unlikely(list_empty(q))) return NULL; + + rq = list_first_entry(q, struct request, queuelist); + list_del_init(&rq->queuelist); + if (list_empty(q)) { + set_adios_state(ad, ADIOS_STATE_PQ, pq_idx, false); + update_elv_direction(ad); + } + return rq; +} + + +// Dispatch a request to the hardware queue +static struct request *adios_dispatch_request(struct blk_mq_hw_ctx *hctx) { + struct adios_data *ad = hctx->queue->elevator->elevator_data; + struct request *rq; + + rq = dispatch_from_pq(ad); + if (rq) + goto found; + + rq = dispatch_from_bq(ad); + if (rq) + goto found; + + return NULL; +found: + if (ad->is_rotational) + ad->head_pos = blk_rq_pos(rq) + blk_rq_sectors(rq); + + rq->rq_flags |= RQF_STARTED; + return rq; +} + +// Timer callback function to periodically update latency models +static void update_timer_callback(struct timer_list *t) { + struct adios_data *ad = timer_container_of(ad, t, update_timer); + + for (u8 optype = 0; optype < ADIOS_OPTYPES; optype++) + latency_model_update(ad, &ad->latency_model[optype]); +} + +// Handle the completion of a request +static void adios_completed_request(struct request *rq, u64 now) { + struct adios_data *ad = rq->q->elevator->elevator_data; + struct adios_rq_data *rd = get_rq_data(rq); + union adios_in_flight_rqs ifr = { .scalar = 0 }; + + if (rd->managed) { + union adios_in_flight_rqs ifr_to_sub = { + .count = 1, + .total_pred_lat = rd->pred_lat, + }; + ifr.scalar = atomic64_sub_return( + ifr_to_sub.scalar, &ad->in_flight_rqs.atomic); + } + u8 optype = adios_optype(rq); + + unsigned long flags; + struct adios_pcpu_completion *pc; + + local_irq_save(flags); + pc = this_cpu_ptr(ad->pcpu_completion); + + if (optype == ADIOS_OTHER) { + // Non-positional commands make the head position unpredictable. + // Invalidate our knowledge of the last completed position. + if (ad->is_rotational) + pc->last_completed_pos = 0; + local_irq_restore(flags); + return; + } + + u64 lct = pc->last_completed_time ?: rq->io_start_time_ns; + pc->last_completed_time = (ifr.count) ? now : 0; + + if (!rq->io_start_time_ns || !rd->block_size || unlikely(now < lct)) { + local_irq_restore(flags); + return; + } + + u64 latency = now - lct; + + u32 weight = 1; + if (ad->is_rotational) { + sector_t current_pos = blk_rq_pos(rq); + // Only calculate seek distance if we have a valid last position. + if (pc->last_completed_pos > 0) { + u64 seek_distance = abs( + (s64)current_pos - (s64)pc->last_completed_pos); + if (seek_distance) + weight = 65 - __builtin_clzll(seek_distance); + } + // Update (or re-synchronize) our knowledge of the head position. + pc->last_completed_pos = current_pos + blk_rq_sectors(rq); + } + + local_irq_restore(flags); + + if (latency > ad->lat_model_latency_limit) + return; + + latency_model_input(ad, &ad->latency_model[optype], + rd->block_size, latency, rd->pred_lat, weight); + timer_reduce(&ad->update_timer, jiffies + msecs_to_jiffies(100)); +} + +// Clean up after a request is finished +static void adios_finish_request(struct request *rq) { + struct adios_data *ad = rq->q->elevator->elevator_data; + + if (rq->elv.priv[0]) { + // Free adios_rq_data back to the memory pool + mempool_free(get_rq_data(rq), ad->rq_data_pool); + rq->elv.priv[0] = NULL; + } +} + +// Check if there are any requests available for dispatch +static bool adios_has_work(struct blk_mq_hw_ctx *hctx) { + struct adios_data *ad = hctx->queue->elevator->elevator_data; + + return atomic_read(&ad->state) != 0; +} + +// Initialize the scheduler-specific data when initializing the request queue +static int adios_init_sched(struct request_queue *q, struct elevator_queue *eq) { + struct adios_data *ad; + int ret = -ENOMEM; + u8 optype = 0; + + ad = kzalloc_node(sizeof(*ad), GFP_KERNEL, q->node); + if (!ad) { + pr_err("adios: Failed to create adios_data\n"); + goto put_eq; + } + + eq->elevator_data = ad; + + // Create a memory pool for adios_rq_data + ad->rq_data_cache = kmem_cache_create("adios_rq_data", + sizeof(struct adios_rq_data), + 0, SLAB_HWCACHE_ALIGN, NULL); + if (!ad->rq_data_cache) { + pr_err("adios: Failed to create rq_data_cache\n"); + goto free_ad; + } + + ad->rq_data_pool = mempool_create_slab_pool( + q->nr_requests, ad->rq_data_cache); + if (!ad->rq_data_pool) { + pr_err("adios: Failed to create rq_data_pool\n"); + goto destroy_rq_data_cache; + } + + /* Create a memory pool for dl_group */ + ad->dl_group_pool = kmem_cache_create("dl_group_pool", + sizeof(struct dl_group), + 0, SLAB_HWCACHE_ALIGN, NULL); + if (!ad->dl_group_pool) { + pr_err("adios: Failed to create dl_group_pool\n"); + goto destroy_rq_data_pool; + } + + for (int i = 0; i < ADIOS_PQ_LEVELS; i++) + INIT_LIST_HEAD(&ad->prio_queue[i]); + + for (u8 i = 0; i < ADIOS_DL_TYPES; i++) { + ad->dl_tree[i] = RB_ROOT_CACHED; + ad->dl_prio[i] = default_dl_prio[i]; + } + ad->dl_bias = 0; + + for (u8 page = 0; page < ADIOS_BQ_PAGES; page++) + for (optype = 0; optype < ADIOS_OPTYPES; optype++) + INIT_LIST_HEAD(&ad->batch_queue[page][optype]); + + ad->aggr_buckets = kzalloc(sizeof(*ad->aggr_buckets), GFP_KERNEL); + if (!ad->aggr_buckets) { + pr_err("adios: Failed to allocate aggregation buckets\n"); + goto destroy_dl_group_pool; + } + + ad->pcpu_completion = alloc_percpu(struct adios_pcpu_completion); + if (!ad->pcpu_completion) { + pr_err("adios: Failed to allocate per-CPU completion data\n"); + goto free_aggr_buckets; + } + + for (optype = 0; optype < ADIOS_OPTYPES; optype++) { + struct latency_model *model = &ad->latency_model[optype]; + struct latency_model_params *params; + + spin_lock_init(&model->update_lock); + params = kzalloc(sizeof(*params), GFP_KERNEL); + if (!params) { + pr_err("adios: Failed to allocate latency_model_params\n"); + goto free_buckets; + } + params->last_update_jiffies = jiffies; + RCU_INIT_POINTER(model->params, params); + + model->pcpu_buckets = alloc_percpu(struct lm_buckets); + if (!model->pcpu_buckets) { + pr_err("adios: Failed to allocate per-CPU buckets\n"); + kfree(params); + goto free_buckets; + } + + model->pcpu_snapshot = alloc_percpu(struct lm_buckets); + if (!model->pcpu_snapshot) { + pr_err("adios: Failed to allocate per-CPU snapshot\n"); + free_percpu(model->pcpu_buckets); + kfree(params); + goto free_buckets; + } + + model->lm_shrink_at_kreqs = default_lm_shrink_at_kreqs; + model->lm_shrink_at_gbytes = default_lm_shrink_at_gbytes; + model->lm_shrink_resist = default_lm_shrink_resist; + } + + for (optype = 0; optype < ADIOS_OPTYPES; optype++) { + ad->latency_target[optype] = default_latency_target[optype]; + ad->batch_limit[optype] = default_batch_limit[optype]; + } + + eq->elevator_data = ad; + + ad->is_rotational = !!(q->limits.features & BLK_FEAT_ROTATIONAL); + ad->global_latency_window = (ad->is_rotational)? + default_global_latency_window_rotational: + default_global_latency_window; + ad->bq_refill_below_ratio = default_bq_refill_below_ratio; + ad->lat_model_latency_limit = default_lat_model_latency_limit; + ad->batch_order = default_batch_order; + ad->compliance_flags = default_compliance_flags; + + ad->models_stable = false; + + atomic_set(&ad->state, 0); + + spin_lock_init(&ad->lock); + spin_lock_init(&ad->pq_lock); + spin_lock_init(&ad->bq_lock); + + timer_setup(&ad->update_timer, update_timer_callback, 0); + + /* We dispatch from request queue wide instead of hw queue */ + blk_queue_flag_set(QUEUE_FLAG_SQ_SCHED, q); + + ad->queue = q; + blk_stat_enable_accounting(q); + + q->elevator = eq; + adios_depth_updated(q); + return 0; + +free_buckets: + pr_err("adios: Failed to allocate per-cpu buckets\n"); + while (optype-- > 0) { + struct latency_model *prev_model = &ad->latency_model[optype]; + kfree(rcu_access_pointer(prev_model->params)); + free_percpu(prev_model->pcpu_snapshot); + free_percpu(prev_model->pcpu_buckets); + } + free_percpu(ad->pcpu_completion); +free_aggr_buckets: + kfree(ad->aggr_buckets); +destroy_dl_group_pool: + kmem_cache_destroy(ad->dl_group_pool); +destroy_rq_data_pool: + mempool_destroy(ad->rq_data_pool); +destroy_rq_data_cache: + kmem_cache_destroy(ad->rq_data_cache); +free_ad: + kfree(ad); +put_eq: + /* + * Do NOT kobject_put(&eq->kobj) here: when ->init_sched() returns an + * error, blk_mq_init_sched() drops the last reference itself (see the + * "out:" label there). mq-deadline/bfq/kyber all just return the errno. + */ + return ret; +} + +// Clean up and free resources when exiting the scheduler +static void adios_exit_sched(struct elevator_queue *e) { + struct adios_data *ad = e->elevator_data; + + timer_shutdown_sync(&ad->update_timer); + + for (int i = 0; i < 2; i++) + WARN_ON_ONCE(!list_empty(&ad->prio_queue[i])); + + for (u8 i = 0; i < ADIOS_OPTYPES; i++) { + struct latency_model *model = &ad->latency_model[i]; + struct latency_model_params *params = rcu_access_pointer(model->params); + + RCU_INIT_POINTER(model->params, NULL); + kfree_rcu(params, rcu); + + free_percpu(model->pcpu_snapshot); + free_percpu(model->pcpu_buckets); + } + + synchronize_rcu(); + + free_percpu(ad->pcpu_completion); + kfree(ad->aggr_buckets); + + mempool_destroy(ad->rq_data_pool); + kmem_cache_destroy(ad->rq_data_cache); + + if (ad->dl_group_pool) + kmem_cache_destroy(ad->dl_group_pool); + + blk_stat_disable_accounting(ad->queue); + + kfree(ad); +} + +static void sideload_latency_model( + struct latency_model *model, u64 base, u64 slope) { + struct latency_model_params *old_params, *new_params; + unsigned long flags; + + new_params = kzalloc(sizeof(*new_params), GFP_KERNEL); + if (!new_params) + return; + + spin_lock_irqsave(&model->update_lock, flags); + + old_params = rcu_dereference_protected(model->params, + lockdep_is_held(&model->update_lock)); + + new_params->last_update_jiffies = jiffies; + + // Initialize base and its statistics as a single sample. + new_params->base = base; + new_params->small_sum_delay = base; + new_params->small_count = 1; + + // Initialize slope and its statistics as a single sample. + new_params->slope = slope; + new_params->large_sum_delay = slope; + new_params->large_sum_bsize = 1024; /* Corresponds to 1 KiB */ + + lm_reset_pcpu_buckets(model); + + rcu_assign_pointer(model->params, new_params); + spin_unlock_irqrestore(&model->update_lock, flags); + + kfree_rcu(old_params, rcu); +} + +// Define sysfs attributes for operation types +#define SYSFS_OPTYPE_DECL(name, optype) \ +static ssize_t adios_lat_model_##name##_show( \ + struct elevator_queue *e, char *page) { \ + struct adios_data *ad = e->elevator_data; \ + struct latency_model *model = &ad->latency_model[optype]; \ + struct latency_model_params *params; \ + ssize_t len = 0; \ + u64 base, slope; \ + rcu_read_lock(); \ + params = rcu_dereference(model->params); \ + base = params->base; \ + slope = params->slope; \ + rcu_read_unlock(); \ + len += sprintf(page, "base : %llu ns\n", base); \ + len += sprintf(page + len, "slope: %llu ns/KiB\n", slope); \ + return len; \ +} \ +static ssize_t adios_lat_model_##name##_store( \ + struct elevator_queue *e, const char *page, size_t count) { \ + struct adios_data *ad = e->elevator_data; \ + struct latency_model *model = &ad->latency_model[optype]; \ + u64 base, slope; \ + int ret; \ + ret = sscanf(page, "%llu %llu", &base, &slope); \ + if (ret != 2) \ + return -EINVAL; \ + sideload_latency_model(model, base, slope); \ + reset_buckets(ad->aggr_buckets); \ + return count; \ +} \ +static ssize_t adios_lat_target_##name##_show( \ + struct elevator_queue *e, char *page) { \ + struct adios_data *ad = e->elevator_data; \ + return sprintf(page, "%llu\n", ad->latency_target[optype]); \ +} \ +static ssize_t adios_lat_target_##name##_store( \ + struct elevator_queue *e, const char *page, size_t count) { \ + struct adios_data *ad = e->elevator_data; \ + unsigned long nsec; \ + int ret; \ + ret = kstrtoul(page, 10, &nsec); \ + if (ret) \ + return ret; \ + sideload_latency_model(&ad->latency_model[optype], 0, 0); \ + ad->latency_target[optype] = nsec; \ + return count; \ +} \ +static ssize_t adios_batch_limit_##name##_show( \ + struct elevator_queue *e, char *page) { \ + struct adios_data *ad = e->elevator_data; \ + return sprintf(page, "%u\n", ad->batch_limit[optype]); \ +} \ +static ssize_t adios_batch_limit_##name##_store( \ + struct elevator_queue *e, const char *page, size_t count) { \ + unsigned long max_batch; \ + int ret; \ + ret = kstrtoul(page, 10, &max_batch); \ + if (ret || max_batch == 0) \ + return -EINVAL; \ + struct adios_data *ad = e->elevator_data; \ + ad->batch_limit[optype] = max_batch; \ + return count; \ +} + +SYSFS_OPTYPE_DECL(read, ADIOS_READ); +SYSFS_OPTYPE_DECL(write, ADIOS_WRITE); +SYSFS_OPTYPE_DECL(discard, ADIOS_DISCARD); + +// Show the maximum batch size actually achieved for each operation type +static ssize_t adios_batch_actual_max_show( + struct elevator_queue *e, char *page) { + struct adios_data *ad = e->elevator_data; + u32 total_count, read_count, write_count, discard_count; + + total_count = ad->batch_actual_max_total; + read_count = ad->batch_actual_max_size[ADIOS_READ]; + write_count = ad->batch_actual_max_size[ADIOS_WRITE]; + discard_count = ad->batch_actual_max_size[ADIOS_DISCARD]; + + return sprintf(page, + "Total : %u\nDiscard: %u\nRead : %u\nWrite : %u\n", + total_count, discard_count, read_count, write_count); +} + +#define SYSFS_ULL_DECL(field, min_val, max_val) \ +static ssize_t adios_##field##_show( \ + struct elevator_queue *e, char *page) { \ + struct adios_data *ad = e->elevator_data; \ + return sprintf(page, "%llu\n", ad->field); \ +} \ +static ssize_t adios_##field##_store( \ + struct elevator_queue *e, const char *page, size_t count) { \ + struct adios_data *ad = e->elevator_data; \ + unsigned long val; \ + int ret; \ + ret = kstrtoul(page, 10, &val); \ + if (ret || val < (min_val) || val > (max_val)) \ + return -EINVAL; \ + ad->field = val; \ + return count; \ +} + +SYSFS_ULL_DECL(global_latency_window, 0, ULLONG_MAX) +SYSFS_ULL_DECL(compliance_flags, 0, ULLONG_MAX) + +#define SYSFS_INT_DECL(field, min_val, max_val) \ +static ssize_t adios_##field##_show( \ + struct elevator_queue *e, char *page) { \ + struct adios_data *ad = e->elevator_data; \ + return sprintf(page, "%d\n", ad->field); \ +} \ +static ssize_t adios_##field##_store( \ + struct elevator_queue *e, const char *page, size_t count) { \ + struct adios_data *ad = e->elevator_data; \ + int val; \ + int ret; \ + ret = kstrtoint(page, 10, &val); \ + if (ret || val < (min_val) || val > (max_val)) \ + return -EINVAL; \ + ad->field = val; \ + return count; \ +} + +SYSFS_INT_DECL(bq_refill_below_ratio, 0, 100) +SYSFS_INT_DECL(lat_model_latency_limit, 0, 2*NSEC_PER_SEC) +SYSFS_INT_DECL(batch_order, ADIOS_BO_OPTYPE, !!ad->is_rotational) + +// Show the read priority +static ssize_t adios_read_priority_show( + struct elevator_queue *e, char *page) { + struct adios_data *ad = e->elevator_data; + return sprintf(page, "%d\n", ad->dl_prio[0]); +} + +// Set the read priority +static ssize_t adios_read_priority_store( + struct elevator_queue *e, const char *page, size_t count) { + struct adios_data *ad = e->elevator_data; + int prio; + int ret; + + ret = kstrtoint(page, 10, &prio); + if (ret || prio < -20 || prio > 19) + return -EINVAL; + + guard(spinlock_irqsave)(&ad->lock); + ad->dl_prio[0] = prio; + ad->dl_bias = 0; + + return count; +} + +// Reset batch queue statistics +static ssize_t adios_reset_bq_stats_store( + struct elevator_queue *e, const char *page, size_t count) { + struct adios_data *ad = e->elevator_data; + unsigned long val; + int ret; + + ret = kstrtoul(page, 10, &val); + if (ret || val != 1) + return -EINVAL; + + for (u8 i = 0; i < ADIOS_OPTYPES; i++) + ad->batch_actual_max_size[i] = 0; + + ad->batch_actual_max_total = 0; + + return count; +} + +// Reset the latency model parameters or load them from user input +static ssize_t adios_reset_lat_model_store( + struct elevator_queue *e, const char *page, size_t count) +{ + struct adios_data *ad = e->elevator_data; + struct latency_model *model; + int ret; + + /* + * Differentiate between two modes based on input format: + * 1. "1": Fully reset the model (backward compatibility). + * 2. "R_base R_slope W_base W_slope D_base D_slope": Load values. + */ + if (!strchr(page, ' ')) { + // Mode 1: Full reset. + unsigned long val; + + ret = kstrtoul(page, 10, &val); + if (ret || val != 1) + return -EINVAL; + + for (u8 i = 0; i < ADIOS_OPTYPES; i++) { + model = &ad->latency_model[i]; + sideload_latency_model(model, 0, 0); + } + } else { + // Mode 2: Load initial values for all latency models. + u64 params[3][2]; /* 0:base, 1:slope for R, W, D */ + + ret = sscanf(page, "%llu %llu %llu %llu %llu %llu", + ¶ms[ADIOS_READ ][0], ¶ms[ADIOS_READ ][1], + ¶ms[ADIOS_WRITE ][0], ¶ms[ADIOS_WRITE ][1], + ¶ms[ADIOS_DISCARD][0], ¶ms[ADIOS_DISCARD][1]); + + if (ret != 6) + return -EINVAL; + + for (u8 i = ADIOS_READ; i <= ADIOS_DISCARD; i++) { + model = &ad->latency_model[i]; + sideload_latency_model(model, params[i][0], params[i][1]); + } + } + reset_buckets(ad->aggr_buckets); + + return count; +} + +// Show the ADIOS version +static ssize_t adios_version_show(struct elevator_queue *e, char *page) { + return sprintf(page, "%s\n", ADIOS_VERSION); +} + +// Define sysfs attributes for dynamic thresholds +#define SHRINK_THRESHOLD_ATTR_RW(name, model_field, min_value, max_value) \ +static ssize_t adios_shrink_##name##_store( \ + struct elevator_queue *e, const char *page, size_t count) { \ + struct adios_data *ad = e->elevator_data; \ + unsigned long val; \ + int ret; \ + ret = kstrtoul(page, 10, &val); \ + if (ret || val < min_value || val > max_value) \ + return -EINVAL; \ + for (u8 i = 0; i < ADIOS_OPTYPES; i++) { \ + struct latency_model *model = &ad->latency_model[i]; \ + unsigned long flags; \ + spin_lock_irqsave(&model->update_lock, flags); \ + model->model_field = val; \ + spin_unlock_irqrestore(&model->update_lock, flags); \ + } \ + return count; \ +} \ +static ssize_t adios_shrink_##name##_show( \ + struct elevator_queue *e, char *page) { \ + struct adios_data *ad = e->elevator_data; \ + u32 val = 0; \ + unsigned long flags; \ + struct latency_model *model = &ad->latency_model[0]; \ + spin_lock_irqsave(&model->update_lock, flags); \ + val = model->model_field; \ + spin_unlock_irqrestore(&model->update_lock, flags); \ + return sprintf(page, "%u\n", val); \ +} + +SHRINK_THRESHOLD_ATTR_RW(at_kreqs, lm_shrink_at_kreqs, 1, 100000) +SHRINK_THRESHOLD_ATTR_RW(at_gbytes, lm_shrink_at_gbytes, 1, 1000) +SHRINK_THRESHOLD_ATTR_RW(resist, lm_shrink_resist, 1, 3) + +// Define sysfs attributes +#define AD_ATTR(name, show_func, store_func) \ + __ATTR(name, 0644, show_func, store_func) +#define AD_ATTR_RW(name) \ + __ATTR(name, 0644, adios_##name##_show, adios_##name##_store) +#define AD_ATTR_RO(name) \ + __ATTR(name, 0444, adios_##name##_show, NULL) +#define AD_ATTR_WO(name) \ + __ATTR(name, 0200, NULL, adios_##name##_store) + +// Define sysfs attributes for ADIOS scheduler +static struct elv_fs_entry adios_sched_attrs[] = { + AD_ATTR_RO(batch_actual_max), + AD_ATTR_RW(bq_refill_below_ratio), + AD_ATTR_RW(global_latency_window), + AD_ATTR_RW(lat_model_latency_limit), + AD_ATTR_RW(batch_order), + AD_ATTR_RW(compliance_flags), + + AD_ATTR_RW(batch_limit_read), + AD_ATTR_RW(batch_limit_write), + AD_ATTR_RW(batch_limit_discard), + + AD_ATTR_RW(lat_model_read), + AD_ATTR_RW(lat_model_write), + AD_ATTR_RW(lat_model_discard), + + AD_ATTR_RW(lat_target_read), + AD_ATTR_RW(lat_target_write), + AD_ATTR_RW(lat_target_discard), + + AD_ATTR_RW(shrink_at_kreqs), + AD_ATTR_RW(shrink_at_gbytes), + AD_ATTR_RW(shrink_resist), + + AD_ATTR_RW(read_priority), + + AD_ATTR_WO(reset_bq_stats), + AD_ATTR_WO(reset_lat_model), + AD_ATTR(adios_version, adios_version_show, NULL), + + __ATTR_NULL +}; + +// Define the ADIOS scheduler type +static struct elevator_type mq_adios = { + .ops = { + .next_request = elv_rb_latter_request, + .former_request = elv_rb_former_request, + .limit_depth = adios_limit_depth, + .depth_updated = adios_depth_updated, + .request_merged = adios_request_merged, + .requests_merged = adios_merged_requests, + .bio_merge = adios_bio_merge, + .insert_requests = adios_insert_requests, + .prepare_request = adios_prepare_request, + .dispatch_request = adios_dispatch_request, + .completed_request = adios_completed_request, + .finish_request = adios_finish_request, + .has_work = adios_has_work, + .init_sched = adios_init_sched, + .exit_sched = adios_exit_sched, + }, + .elevator_attrs = adios_sched_attrs, + .elevator_name = "adios", + .elevator_owner = THIS_MODULE, +}; +MODULE_ALIAS("mq-adios-iosched"); + +#define ADIOS_PROGNAME "Adaptive Deadline I/O Scheduler" +#define ADIOS_AUTHOR "Masahito Suzuki" + +// Initialize the ADIOS scheduler module +static int __init adios_init(void) { + printk(KERN_INFO "%s %s by %s\n", + ADIOS_PROGNAME, ADIOS_VERSION, ADIOS_AUTHOR); + return elv_register(&mq_adios); +} + +// Exit the ADIOS scheduler module +static void __exit adios_exit(void) { + elv_unregister(&mq_adios); +} + +module_init(adios_init); +module_exit(adios_exit); + +MODULE_AUTHOR(ADIOS_AUTHOR); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION(ADIOS_PROGNAME); \ No newline at end of file diff -ruN --no-dereference -x .git a/block/elevator.c b/block/elevator.c --- a/block/elevator.c 2026-10-11 15:27:50.000000000 +0200 +++ b/block/elevator.c 2026-10-13 14:33:30.000000000 +0200 @@ -740,6 +740,19 @@ if (q->tag_set->flags & BLK_MQ_F_NO_SCHED_BY_DEFAULT) return; +#ifdef CONFIG_MQ_IOSCHED_DEFAULT_ADIOS + ctx.name = "adios"; +#else // !CONFIG_MQ_IOSCHED_DEFAULT_ADIOS + bool is_sq = q->nr_hw_queues == 1 || blk_mq_is_shared_tags(q->tag_set->flags); +#if defined(CONFIG_CACHY) && defined(CONFIG_IOSCHED_BFQ) + if (is_sq) + ctx.name = "bfq"; +#else + if (!is_sq) + return; +#endif /* CONFIG_CACHY && CONFIG_IOSCHED_BFQ */ +#endif // CONFIG_MQ_IOSCHED_DEFAULT_ADIOS + /* * For single queue devices, default to using mq-deadline. If we * have multiple queues or mq-deadline is not available, default @@ -749,13 +762,10 @@ if (!ctx.type) return; - if ((q->nr_hw_queues == 1 || - blk_mq_is_shared_tags(q->tag_set->flags))) { - err = elevator_change(q, &ctx); - if (err < 0) - pr_warn("\"%s\" elevator initialization, failed %d, falling back to \"none\"\n", - ctx.name, err); - } + err = elevator_change(q, &ctx); + if (err < 0) + pr_warn("\"%s\" elevator initialization, failed %d, falling back to \"none\"\n", + ctx.name, err); elevator_put(ctx.type); } diff -ruN --no-dereference -x .git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c --- a/drivers/acpi/processor_idle.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/acpi/processor_idle.c 2026-10-13 14:33:30.000000000 +0200 @@ -1441,6 +1441,19 @@ if (disabled_by_idle_boot_param()) return; +#ifdef CONFIG_X86 + /* + * The bus master check on AMD is observed to be very costly and results + * in not only increased power consumption, but also lots of CPU cycles + * burned. Disable the actual bus master check while keeping + * pr->flags.bm_check enabled so there's no cache flush upon C3 entry. + */ + if (!bm_check_disable && boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { + bm_check_disable = true; + pr_info("Enabled processor.bm_check_disable for AMD cpuidle\n"); + } +#endif + if (!acpi_processor_get_power_info(pr)) pr->flags.power_setup_done = 1; diff -ruN --no-dereference -x .git a/drivers/acpi/scan.c b/drivers/acpi/scan.c --- a/drivers/acpi/scan.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/acpi/scan.c 2026-10-13 14:33:30.000000000 +0200 @@ -1757,6 +1757,7 @@ * by the drivers/platform/x86/serial-multi-instantiate.c driver, which * knows which client device id to use for each resource. */ + {"AWDZ8399", }, {"BSG1160", }, {"BSG2150", }, {"CSC3551", }, @@ -1765,6 +1766,7 @@ {"CSC3557", }, {"INT33FE", }, {"INT3515", }, + {"MAX98390", }, {"TXNW2781", }, /* Non-conforming _HID for Cirrus Logic already released */ {"CLSA0100", }, diff -ruN --no-dereference -x .git a/drivers/acpi/x86/apple.c b/drivers/acpi/x86/apple.c --- a/drivers/acpi/x86/apple.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/acpi/x86/apple.c 2026-10-13 14:33:30.000000000 +0200 @@ -6,7 +6,13 @@ #include #include +#include +#include +#include +#include +#include #include +#include #include #include "../internal.h" @@ -146,3 +152,128 @@ ACPI_FREE(props); bitmap_free(valid); } + +#ifdef CONFIG_PM_SLEEP_SMP +#define PCI_DEVICE_ID_APPLE_T2_BRIDGE 0x1801 + +/* + * The ACPI path selected by _OSI("Darwin") leaves Apple T2 systems in a + * state where bringing secondary CPUs online during early resume may take + * several seconds per CPU. Normal CPU hotplug after platform resume is not + * affected, so move it outside the generic suspend CPU hotplug window. + */ +static cpumask_var_t apple_t2_offlined_cpus; + +static void apple_t2_restore_cpus(void) +{ + unsigned int cpu; + int ret; + + for_each_cpu(cpu, apple_t2_offlined_cpus) { + ret = add_cpu(cpu); + if (ret) { + pr_err("ACPI: Apple T2 failed to restore CPU%u: %d\n", + cpu, ret); + continue; + } + + cpumask_clear_cpu(cpu, apple_t2_offlined_cpus); + } +} + +static void apple_t2_offline_cpus(void) +{ + unsigned int cpu; + int ret; + + if (!cpumask_empty(apple_t2_offlined_cpus)) { + pr_err("ACPI: Apple T2 CPUs from the previous suspend remain offline\n"); + apple_t2_restore_cpus(); + if (!cpumask_empty(apple_t2_offlined_cpus)) { + pr_err("ACPI: Apple T2 early CPU offlining skipped\n"); + return; + } + } + + for_each_online_cpu(cpu) { + if (cpu == 0) + continue; + + ret = remove_cpu(cpu); + if (ret) { + pr_err("ACPI: Apple T2 failed to offline CPU%u: %d\n", + cpu, ret); + continue; + } + + cpumask_set_cpu(cpu, apple_t2_offlined_cpus); + } +} + +static int apple_t2_cpu_prepare(struct notifier_block *nb, + unsigned long action, void *unused) +{ + if (action == PM_SUSPEND_PREPARE) + apple_t2_offline_cpus(); + + return NOTIFY_OK; +} + +static int apple_t2_cpu_restore(struct notifier_block *nb, + unsigned long action, void *unused) +{ + if (action == PM_POST_SUSPEND) + apple_t2_restore_cpus(); + + return NOTIFY_OK; +} + +/* + * The CPU core PM notifier runs at priority 0. Offline CPUs before it blocks + * hotplug, then restore them after it enables hotplug again. + */ +static struct notifier_block apple_t2_cpu_prepare_nb = { + .notifier_call = apple_t2_cpu_prepare, + .priority = 1, +}; + +static struct notifier_block apple_t2_cpu_restore_nb = { + .notifier_call = apple_t2_cpu_restore, + .priority = -1, +}; + +static int __init apple_t2_cpu_pm_init(void) +{ + struct pci_dev *t2; + int ret; + + if (!x86_apple_machine) + return 0; + + t2 = pci_get_device(PCI_VENDOR_ID_APPLE, + PCI_DEVICE_ID_APPLE_T2_BRIDGE, NULL); + if (!t2) + return 0; + pci_dev_put(t2); + + if (!alloc_cpumask_var(&apple_t2_offlined_cpus, GFP_KERNEL)) + return -ENOMEM; + + ret = register_pm_notifier(&apple_t2_cpu_prepare_nb); + if (ret) + goto free_mask; + + ret = register_pm_notifier(&apple_t2_cpu_restore_nb); + if (ret) + goto unregister_prepare; + + return 0; + +unregister_prepare: + unregister_pm_notifier(&apple_t2_cpu_prepare_nb); +free_mask: + free_cpumask_var(apple_t2_offlined_cpus); + return ret; +} +late_initcall(apple_t2_cpu_pm_init); +#endif diff -ruN --no-dereference -x .git a/drivers/ata/ahci.c b/drivers/ata/ahci.c --- a/drivers/ata/ahci.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/ata/ahci.c 2026-10-13 14:33:30.000000000 +0200 @@ -2659,7 +2659,7 @@ return IRQ_RETVAL(rc); } -static void ahci_remap_check(struct pci_dev *pdev, int bar, +static int ahci_remap_check(struct pci_dev *pdev, int bar, struct ahci_host_priv *hpriv) { int i; @@ -2672,7 +2672,7 @@ pci_resource_len(pdev, bar) < SZ_512K || bar != AHCI_PCI_BAR_STANDARD || !(readl(hpriv->mmio + AHCI_VSCAP) & 1)) - return; + return 0; cap = readq(hpriv->mmio + AHCI_REMAP_CAP); for (i = 0; i < AHCI_MAX_REMAP; i++) { @@ -2687,18 +2687,11 @@ } if (!hpriv->remapped_nvme) - return; + return 0; - dev_warn(&pdev->dev, "Found %u remapped NVMe devices.\n", - hpriv->remapped_nvme); - dev_warn(&pdev->dev, - "Switch your BIOS from RAID to AHCI mode to use them.\n"); - - /* - * Don't rely on the msi-x capability in the remap case, - * share the legacy interrupt across ahci and remapped devices. - */ - hpriv->flags |= AHCI_HFLAG_NO_MSI; + /* Abort probe, allowing intel-nvme-remap to step in when available */ + dev_info(&pdev->dev, "Device will be handled by intel-nvme-remap.\n"); + return -ENODEV; } static int ahci_get_irq_vector(struct ata_host *host, int port) @@ -2988,7 +2981,9 @@ return rc; /* detect remapped nvme devices */ - ahci_remap_check(pdev, ahci_pci_bar, hpriv); + rc = ahci_remap_check(pdev, ahci_pci_bar, hpriv); + if (rc) + return rc; sysfs_add_file_to_group(&pdev->dev.kobj, &dev_attr_remapped_nvme.attr, diff -ruN --no-dereference -x .git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c --- a/drivers/bluetooth/btusb.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/bluetooth/btusb.c 2026-10-13 14:33:30.000000000 +0200 @@ -587,6 +587,8 @@ BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3612), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0489, 0xe112), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x0489, 0xe122), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, @@ -715,6 +717,8 @@ { USB_DEVICE(0x13d3, 0x3596), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, /* MediaTek MT7922 Bluetooth devices */ + { USB_DEVICE(0x13d3, 0x3625), .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3585), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3610), .driver_info = BTUSB_MEDIATEK | diff -ruN --no-dereference -x .git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c --- a/drivers/cpufreq/amd-pstate.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/cpufreq/amd-pstate.c 2026-10-13 14:33:30.000000000 +0200 @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -262,17 +263,23 @@ if (value == prev) return 0; + /* + * Must come before MSR write to ensure amd_pstate_epp_boost_release() + * doesn't write a stale value. + */ + WRITE_ONCE(cpudata->cppc_req_cached, value); + if (fast_switch) { wrmsrq(MSR_AMD_CPPC_REQ, value); } else { int ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); - if (ret) + if (ret) { + WRITE_ONCE(cpudata->cppc_req_cached, prev); return ret; + } } - WRITE_ONCE(cpudata->cppc_req_cached, value); - return 0; } @@ -312,16 +319,17 @@ if (value == prev) return 0; + /* Must update cached value prior to wrmsrq -- see msr_update_perf() */ + WRITE_ONCE(cpudata->cppc_req_cached, value); + ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); if (ret) { + WRITE_ONCE(cpudata->cppc_req_cached, prev); pr_err("failed to set energy perf value (%d)\n", ret); return ret; } - /* update both so that msr_update_perf() can effectively check */ - WRITE_ONCE(cpudata->cppc_req_cached, value); - - return ret; + return 0; } DEFINE_STATIC_CALL(amd_pstate_set_epp, msr_set_epp); @@ -1884,6 +1892,187 @@ return false; } +/* + * Per-core EPP boost. + * + * While the epp_boost module parameter is enabled, each CPU in active (EPP) + * mode on an MSR (X86_FEATURE_CPPC) system has a cpufreq update-util hook + * registered which puts the core in EPP=performance mode when the core is + * sufficiently utilized. + * + * Enables more efficient frequency scaling on cores that have strict latency + * requirements, such as e.g. gaming workloads where render threads use a CPU + * for some period during frame windows, and then go idle until the next frame. + */ +static bool epp_boost __read_mostly; + +#define AMD_PSTATE_EPP_BOOST_SAMPLE_NS (10 * NSEC_PER_MSEC) +/* + * Busy threshold is deliberately low and the decay window wide so that a + * render thread which is only ~50-80% busy (periodic vsync and GPU-fence + * waits) holds the boost across its whole busy period at a couple of CPPC_REQ + * writes total. + */ +#define AMD_PSTATE_EPP_BOOST_DECAY_NS (300 * NSEC_PER_MSEC) +#define AMD_PSTATE_EPP_BOOST_BUSY_PCT 50 + +static void amd_pstate_epp_boost_apply(struct amd_cpudata *cpudata, u64 cached) +{ + union perf_cached perf = READ_ONCE(cpudata->perf); + + /* never boost a request whose limits are not populated */ + if (!perf.max_limit_perf) + return; + + cached &= ~AMD_CPPC_EPP_PERF_MASK; + cached |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, AMD_CPPC_EPP_PERFORMANCE); + + wrmsrq(MSR_AMD_CPPC_REQ, cached); +} + +static void amd_pstate_epp_boost_release(struct amd_cpudata *cpudata) +{ + wrmsrq(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); +} + +static void amd_pstate_epp_boost_update_util(struct update_util_data *data, + u64 time, unsigned int flags) +{ + struct amd_cpudata *cpudata = container_of(data, struct amd_cpudata, + epp_boost_update_util); + union perf_cached perf; + bool first_sample; + u64 busy_pct; + bool active; + + if (smp_processor_id() != cpudata->cpu) + return; + + first_sample = !cpudata->epp_boost_last_sample; + if (!first_sample && + time - cpudata->epp_boost_last_sample < AMD_PSTATE_EPP_BOOST_SAMPLE_NS) + return; + cpudata->epp_boost_last_sample = time; + + /* + * Counters can reset across suspend or hotplug (so their values would + * be garbage), so just recalculate baselines. + */ + if (amd_pstate_sample(cpudata) && !first_sample) { + busy_pct = div64_u64(cpudata->cur.mperf * 100, cpudata->cur.tsc); + if (busy_pct >= AMD_PSTATE_EPP_BOOST_BUSY_PCT) + cpudata->epp_boost_last_busy = time; + } + + perf = READ_ONCE(cpudata->perf); + active = perf.max_limit_perf && + (time - cpudata->epp_boost_last_busy < AMD_PSTATE_EPP_BOOST_DECAY_NS); + + if (active && !cpudata->epp_boost_active) + amd_pstate_epp_boost_apply(cpudata, + READ_ONCE(cpudata->cppc_req_cached)); + else if (!active && cpudata->epp_boost_active) + amd_pstate_epp_boost_release(cpudata); + + cpudata->epp_boost_active = active; +} + +static void amd_pstate_epp_boost_enable(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + + if (!READ_ONCE(epp_boost)) + return; + + if (cppc_state != AMD_PSTATE_ACTIVE || + !cpu_feature_enabled(X86_FEATURE_CPPC)) + return; + + if (cpudata->epp_boost_registered) + return; + + cpudata->epp_boost_last_sample = 0; + cpudata->epp_boost_last_busy = 0; + cpufreq_add_update_util_hook(policy->cpu, &cpudata->epp_boost_update_util, + amd_pstate_epp_boost_update_util); + cpudata->epp_boost_registered = true; +} + +static void amd_pstate_epp_boost_disable(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + + if (cpudata->epp_boost_registered) { + cpufreq_remove_update_util_hook(policy->cpu); + synchronize_rcu(); + cpudata->epp_boost_registered = false; + } + + if (cpudata->epp_boost_active) { + u64 cached = READ_ONCE(cpudata->cppc_req_cached); + + wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, cached); + + /* + * Cross-CPU restore is not ordered against a concurrent + * limit/EPP writer that doesn't hold the policy rwsem, so + * recheck the cache once to ensure that the MSR reflects the + * cached value + */ + if (READ_ONCE(cpudata->cppc_req_cached) != cached) + wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, + READ_ONCE(cpudata->cppc_req_cached)); + + cpudata->epp_boost_active = false; + } +} + +static int epp_boost_param_set(const char *val, const struct kernel_param *kp) +{ + unsigned int cpu; + bool enabled; + int ret; + + ret = kstrtobool(val, &enabled); + if (ret) + return ret; + + guard(mutex)(&amd_pstate_driver_lock); + + if (epp_boost == enabled) + return 0; + + WRITE_ONCE(epp_boost, enabled); + + if (!current_pstate_driver) + return 0; + + guard(cpus_read_lock)(); + + for_each_online_cpu(cpu) { + struct cpufreq_policy *policy __free(put_cpufreq_policy) = + cpufreq_cpu_get(cpu); + + if (!policy) + continue; + + if (enabled) + amd_pstate_epp_boost_enable(policy); + else + amd_pstate_epp_boost_disable(policy); + } + + return 0; +} + +static const struct kernel_param_ops epp_boost_param_ops = { + .set = epp_boost_param_set, + .get = param_get_bool, +}; +module_param_cb(epp_boost, &epp_boost_param_ops, &epp_boost, 0644); +MODULE_PARM_DESC(epp_boost, + "boost EPP toward performance on recently-busy CPUs (active mode, MSR systems only)"); + static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata; @@ -1938,12 +2127,13 @@ policy->boost_supported = READ_ONCE(cpudata->boost_supported); - /* Fetch the firmware programmed default EPP value */ + /* Cache the firmware programmed EPP */ default_epp = amd_pstate_get_epp(cpudata); if (default_epp < 0) { ret = default_epp; goto free_cpudata1; } + FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &cpudata->cppc_req_cached, default_epp); /* * Set the policy to provide a valid fallback value in case @@ -1976,6 +2166,8 @@ current_pstate_driver->adjust_perf = NULL; + amd_pstate_epp_boost_enable(policy); + return 0; free_cpudata1: @@ -1995,6 +2187,8 @@ if (cpudata->dynamic_epp) amd_pstate_clear_dynamic_epp(policy); + amd_pstate_epp_boost_disable(policy); + /* Reset CPPC_REQ MSR to the BIOS value */ amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); @@ -2063,7 +2257,13 @@ return ret; cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); - return amd_pstate_set_floor_perf(policy, cached_floor_perf); + ret = amd_pstate_set_floor_perf(policy, cached_floor_perf); + if (ret) + return ret; + + amd_pstate_epp_boost_enable(policy); + + return 0; } static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) @@ -2072,6 +2272,8 @@ union perf_cached perf = READ_ONCE(cpudata->perf); int ret; + amd_pstate_epp_boost_disable(policy); + /* * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified * min_perf value across kexec reboots. If this CPU is just onlined normally after this, the @@ -2094,6 +2296,8 @@ union perf_cached perf = READ_ONCE(cpudata->perf); int ret; + amd_pstate_epp_boost_disable(policy); + /* * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified * min_perf value across kexec reboots. If this CPU is just resumed back without kexec, @@ -2140,10 +2344,9 @@ struct amd_cpudata *cpudata = policy->driver_data; union perf_cached perf = READ_ONCE(cpudata->perf); u8 cached_floor_perf; + int ret; if (cpudata->suspended) { - int ret; - /* enable amd pstate from suspend state*/ ret = amd_pstate_epp_update_limit(policy, false); if (ret) @@ -2153,7 +2356,13 @@ } cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); - return amd_pstate_set_floor_perf(policy, cached_floor_perf); + ret = amd_pstate_set_floor_perf(policy, cached_floor_perf); + if (ret) + return ret; + + amd_pstate_epp_boost_enable(policy); + + return 0; } static struct cpufreq_driver amd_pstate_driver = { diff -ruN --no-dereference -x .git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h --- a/drivers/cpufreq/amd-pstate.h 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/cpufreq/amd-pstate.h 2026-10-13 14:33:30.000000000 +0200 @@ -10,6 +10,7 @@ #include #include +#include /********************************************************************* * AMD P-state INTERFACE * @@ -32,6 +33,7 @@ * @min_limit_perf: Cached value of the performance corresponding to policy->min * @max_limit_perf: Cached value of the performance corresponding to policy->max * @bios_min_perf: Cached perf value corresponding to the "Requested CPU Min Frequency" BIOS option + * @val: Raw 64-bit value for atomic access via READ_ONCE()/WRITE_ONCE() */ union perf_cached { struct { @@ -89,7 +91,18 @@ * @epp_default_ac: Default EPP value for AC power source * @epp_default_dc: Default EPP value for DC power source * @dynamic_epp: Whether dynamic EPP is enabled + * @raw_epp: Whether the last EPP write was a raw numeric value rather than a + * named preference * @power_nb: Notifier block for power events + * @current_profile: Currently selected platform profile option + * @ppdev: Device registered with the platform profile handler + * @profile_name: Name under which @ppdev is registered + * @epp_boost_update_util: update-util hook for the per-core EPP boost + * @epp_boost_last_sample: last C0-residency sample time (ns; 0 means the + * next callback only re-baselines the counters) + * @epp_boost_last_busy: last time the CPU sampled as busy (ns) + * @epp_boost_active: EPP boost currently applied to MSR_AMD_CPPC_REQ + * @epp_boost_registered: update-util hook currently registered * * The amd_cpudata is key private data for each CPU thread in AMD P-State, and * represents all the attributes and goals that AMD P-State requests at runtime. @@ -133,6 +146,13 @@ enum platform_profile_option current_profile; struct device *ppdev; char *profile_name; + + /* per-core EPP boost (active mode, MSR systems only) */ + struct update_util_data epp_boost_update_util; + u64 epp_boost_last_sample; + u64 epp_boost_last_busy; + bool epp_boost_active; + bool epp_boost_registered; }; /* diff -ruN --no-dereference -x .git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c --- a/drivers/cpufreq/intel_pstate.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/cpufreq/intel_pstate.c 2026-10-13 14:33:30.000000000 +0200 @@ -3927,6 +3927,8 @@ if (!strcmp(str, "disable")) no_load = 1; + else if (!strcmp(str, "enable")) + no_load = 0; else if (!strcmp(str, "active")) default_driver = &intel_pstate; else if (!strcmp(str, "passive")) diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h 2026-10-13 14:33:30.000000000 +0200 @@ -161,6 +161,7 @@ */ extern int amdgpu_modeset; extern unsigned int amdgpu_vram_limit; +extern int amdgpu_ignore_min_pcap; extern int amdgpu_vis_vram_limit; extern int amdgpu_gart_size; extern int amdgpu_gtt_size; diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c 2026-10-13 14:33:30.000000000 +0200 @@ -150,6 +150,7 @@ }; unsigned int amdgpu_vram_limit = UINT_MAX; +int amdgpu_ignore_min_pcap = 0; /* do not ignore by default */ int amdgpu_vis_vram_limit; int amdgpu_gart_size = -1; /* auto */ int amdgpu_gtt_size = -1; /* auto */ @@ -272,6 +273,15 @@ }; /** + * DOC: ignore_min_pcap (int) + * Ignore the minimum power cap. + * Useful on graphics cards where the minimum power cap is very high. + * The default is 0 (Do not ignore). + */ +MODULE_PARM_DESC(ignore_min_pcap, "Ignore the minimum power cap"); +module_param_named(ignore_min_pcap, amdgpu_ignore_min_pcap, int, 0600); + +/** * DOC: vramlimit (int) * Restrict the total amount of VRAM in MiB for testing. The default is 0 (Use full VRAM). */ @@ -2383,6 +2393,9 @@ int ret, retry = 0, i; bool supports_atomic = false; + if (vga_switcheroo_client_probe_defer(pdev)) + return -EPROBE_DEFER; + if ((pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA || (pdev->class >> 8) == PCI_CLASS_DISPLAY_OTHER) { if (drm_firmware_drivers_only() && amdgpu_modeset == -1) diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig --- a/drivers/gpu/drm/amd/display/Kconfig 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/display/Kconfig 2026-10-13 14:33:30.000000000 +0200 @@ -68,4 +68,10 @@ For more information on KUnit and unit tests in general, please refer to the KUnit documentation in Documentation/dev-tools/kunit/. +config AMD_PRIVATE_COLOR + bool "Enable KMS color management by AMD for AMD" + default n + help + This option extends the KMS color management API with AMD driver-specific properties to enhance the color management support on AMD Steam Deck. + endmenu diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c 2026-10-13 14:33:30.000000000 +0200 @@ -2258,6 +2258,9 @@ if (amdgpu_dc_debug_mask & DC_SKIP_DETECTION_LT) adev->dm.dc->debug.skip_detection_link_training = true; + if (amdgpu_dc_debug_mask & DC_OVERRIDE_PCON_VRR_ID_CHECK) + adev->dm.dc->debug.override_pcon_vrr_id_check = true; + adev->dm.dc->debug.visual_confirm = amdgpu_dc_visual_confirm; /* TODO: Remove after DP2 receiver gets proper support of Cable ID feature */ @@ -5220,7 +5223,7 @@ if (r) return r; -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR if (amdgpu_dm_create_color_properties(adev)) return -ENOMEM; #endif @@ -7088,16 +7091,11 @@ timing_out->v_border_top = 0; timing_out->v_border_bottom = 0; /* TODO: un-hardcode */ - if (drm_mode_is_420_only(info, mode_in) - && (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A || - stream->signal == SIGNAL_TYPE_HDMI_FRL) - && aconnector - && aconnector->force_yuv_pixel_format == PIXEL_ENCODING_YCBCR420) - timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR420; - else if (drm_mode_is_420_also(info, mode_in) - && aconnector - && (aconnector->force_yuv_pixel_format == PIXEL_ENCODING_YCBCR420 - || aconnector->force_yuv420_output)) + if (drm_mode_is_420_only(info, mode_in) || + (aconnector && + (aconnector->force_yuv_pixel_format == PIXEL_ENCODING_YCBCR420 || + aconnector->force_yuv420_output) && + drm_mode_is_420_also(info, mode_in))) timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR420; else if ((connector->display_info.color_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422)) && aconnector @@ -7188,6 +7186,11 @@ stream->output_color_space = get_output_color_space(timing_out, connector_state); stream->content_type = get_output_content_type(connector_state); + + /* DisplayID Type VII pass-through timings. */ + if (mode_in->dsc_passthrough_timings_support && info->dp_dsc_bpp_x16 != 0) { + stream->timing.dsc_fixed_bits_per_pixel_x16 = info->dp_dsc_bpp_x16; + } } static void fill_audio_info(struct audio_info *audio_info, @@ -7708,6 +7711,7 @@ struct drm_display_mode mode; struct drm_display_mode saved_mode; struct drm_display_mode *freesync_mode = NULL; + struct drm_display_mode *dsc_passthru_mode = NULL; bool native_mode_found = false; bool recalculate_timing = false; bool scale = dm_state->scaling != RMX_OFF; @@ -7799,6 +7803,16 @@ } } + list_for_each_entry(dsc_passthru_mode, &connector->modes, head) { + if (dsc_passthru_mode->hdisplay == mode.hdisplay && + dsc_passthru_mode->vdisplay == mode.vdisplay && + drm_mode_vrefresh(dsc_passthru_mode) == mode_refresh) { + mode.dsc_passthrough_timings_support = + dsc_passthru_mode->dsc_passthrough_timings_support; + break; + } + } + if (recalculate_timing) drm_mode_set_crtcinfo(&saved_mode, 0); @@ -7845,7 +7859,7 @@ if (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A || stream->signal == SIGNAL_TYPE_HDMI_FRL) - mod_build_hf_vsif_infopacket(stream, &stream->vsp_infopacket, false, false); + mod_build_hf_vsif_infopacket(stream, &stream->hfvsif_infopacket, false, false); if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT || stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST || @@ -10138,7 +10152,11 @@ aconn = (struct amdgpu_dm_connector *)new_stream->dm_stream_context; - if (aconn && (aconn->as_type == FREESYNC_TYPE_PCON_IN_WHITELIST || aconn->vsdb_info.replay_mode)) { + if (aconn && aconn->as_type == ADAPTIVE_SYNC_TYPE_HDMI) + packet_type = PACKET_TYPE_VTEM; + + else if (aconn && (aconn->as_type == ADAPTIVE_SYNC_TYPE_PCON_ALLOWED || + aconn->vsdb_info.replay_mode)) { pack_sdp_v1_3 = aconn->pack_sdp_v1_3; if (aconn->vsdb_info.amd_vsdb_version == 1) @@ -10224,6 +10242,10 @@ VRR_STATE_ACTIVE_VARIABLE : VRR_STATE_INACTIVE; } + + /* Enable freesync on desktop for HDMI */ + if (dc_is_hdmi_signal(new_stream->signal)) + new_stream->freesync_on_desktop = true; } else { config.state = VRR_STATE_UNSUPPORTED; } @@ -13965,7 +13987,7 @@ return ret; } -static void parse_edid_displayid_vrr(struct drm_connector *connector, +static void parse_edid_displayid_vrr(struct amdgpu_dm_connector *aconn, const struct edid *edid) { u8 *edid_ext = NULL; @@ -13998,8 +14020,8 @@ max_vfreq = edid_ext[j+10]; if (max_vfreq && min_vfreq) { - connector->display_info.monitor_range.max_vfreq = max_vfreq; - connector->display_info.monitor_range.min_vfreq = min_vfreq; + aconn->max_vfreq = max_vfreq; + aconn->min_vfreq = min_vfreq; return; } @@ -14019,13 +14041,13 @@ return connector->display_info.amd_vsdb.version != 0; } -static int parse_hdmi_amd_vsdb(struct amdgpu_dm_connector *aconnector, +static bool parse_amd_vsdb_cea(struct amdgpu_dm_connector *aconnector, const struct edid *edid, struct amdgpu_hdmi_vsdb_info *vsdb_info) { + struct amdgpu_hdmi_vsdb_info vsdb_local = {0}; u8 *edid_ext = NULL; int i; - bool valid_vsdb_found = false; /*----- drm_find_cea_extension() -----*/ /* No EDID or EDID extensions */ @@ -14046,9 +14068,101 @@ if (edid_ext[0] != CEA_EXT) return -ENODEV; - valid_vsdb_found = parse_edid_cea(aconnector, edid_ext, EDID_LENGTH, vsdb_info); + if (!parse_edid_cea(aconnector, edid_ext, EDID_LENGTH, &vsdb_local)) + return -ENODEV; + + *vsdb_info = vsdb_local; + return false; +} + +static bool is_aconn_range_invalid(const struct amdgpu_dm_connector *aconn) +{ + return aconn->min_vfreq == 0 || aconn->max_vfreq == 0; +} + +/* + * Returns true if (max_vfreq - min_vfreq) > 10 + */ +static bool is_freesync_capable(const struct amdgpu_dm_connector *aconn) +{ + if (is_aconn_range_invalid(aconn)) + return false; + + return (aconn->max_vfreq - aconn->min_vfreq) > 10; +} + +static void aconn_range_from_vsdb(struct amdgpu_dm_connector *aconn, + const struct amdgpu_hdmi_vsdb_info *vsdb) +{ + aconn->min_vfreq = vsdb->min_refresh_rate_hz; + aconn->max_vfreq = vsdb->max_refresh_rate_hz; +} + +/** + * Get VRR range from HDMI VRR info in EDID. If VRRmax == 0, + * try getting upper bound from AMD vsdb. + * + * @conn: drm_connector with HDMI VRR info + * @vsdb: AMD vsdb from CAE + */ +static void aconn_range_from_hdmi(struct amdgpu_dm_connector *aconn, + const struct drm_hdmi_vrr_cap *hdmi, + const struct amdgpu_hdmi_vsdb_info *vsdb) +{ + u16 vrr_max = hdmi->vrr_max; + + /* Try getting upper vrr bound from AMD vsdb */ + if (vrr_max == 0) + vrr_max = vsdb->max_refresh_rate_hz; + + /* Use max possible BRR value as a last resort */ + if (vrr_max == 0) + vrr_max = VTEM_BRR_MAX; + + aconn->min_vfreq = hdmi->vrr_min; + aconn->max_vfreq = vrr_max; +} + +/* + * Returns true if connector is capable of freesync + * Optionally, can fetch the range from AMD vsdb + */ +static void copy_range_to_amdgpu_connector(struct amdgpu_dm_connector *aconn, + const struct drm_connector *conn) +{ + aconn->min_vfreq = conn->display_info.monitor_range.min_vfreq; + aconn->max_vfreq = conn->display_info.monitor_range.max_vfreq; +} + +/* + * Sometimes, Monitor Ranges contain limited VRR range and only AMD vsdb has + * the full, advertised VRR range. + * + * Only extend lower limit if current one doesn't support Low + * Framerate Compensation. + * + * During widespread testing, we found that some manufacturers probably + * had issues with their monitors' lower VRR boundaries and adjusted + * them up (Gigabyte X34GS with official range: 48-180, AMD vsdb: 48- + * 180 yet Monitor Ranges: 55-180). After setting the lower boundary + * from AMD vsdb, such monitors start having blanking issues. + */ +static void extend_range_from_vsdb(struct amdgpu_dm_connector *aconn, + const struct amdgpu_hdmi_vsdb_info *vsdb) +{ + /* Always extend upper limit */ + if (vsdb->max_refresh_rate_hz > aconn->max_vfreq) + aconn->max_vfreq = vsdb->max_refresh_rate_hz; + + /* Current min_vfreq supports LFC */ + if ((aconn->min_vfreq * 2) < aconn->max_vfreq) + return; + + if (vsdb->min_refresh_rate_hz == 0) + return; - return valid_vsdb_found ? i : -ENODEV; + if (vsdb->min_refresh_rate_hz < aconn->min_vfreq) + aconn->min_vfreq = vsdb->min_refresh_rate_hz; } /** @@ -14069,16 +14183,19 @@ void amdgpu_dm_update_freesync_caps(struct drm_connector *connector, const struct drm_edid *drm_edid, bool do_mccs) { - int i = 0; struct amdgpu_dm_connector *amdgpu_dm_connector = to_amdgpu_dm_connector(connector); struct dm_connector_state *dm_con_state = NULL; struct dc_sink *sink; struct amdgpu_device *adev = drm_to_adev(connector->dev); struct amdgpu_hdmi_vsdb_info vsdb_info = {0}; + struct amdgpu_hdmi_vsdb_info vsdb_did = {0}; + struct drm_hdmi_vrr_cap hdmi_vrr = {0}; + struct dpcd_caps dpcd_caps = {0}; const struct edid *edid; bool freesync_capable = false; - enum adaptive_sync_type as_type = ADAPTIVE_SYNC_TYPE_NONE; + bool pcon_allowed = false; + bool is_pcon = false; if (!connector->state) { drm_err(adev_to_drm(adev), "%s - Connector has no state", __func__); @@ -14106,71 +14223,67 @@ if (!adev->dm.freesync_module || !dc_supports_vrr(sink->ctx->dce_version)) goto update; + /* Gather all data */ edid = drm_edid_raw(drm_edid); // FIXME: Get rid of drm_edid_raw() + parse_amd_vsdb_cea(amdgpu_dm_connector, edid, &vsdb_info); + amdgpu_dm_connector->vsdb_info = vsdb_info; + hdmi_vrr = connector->display_info.hdmi.vrr_cap; + + if (amdgpu_dm_connector->dc_link) { + dpcd_caps = amdgpu_dm_connector->dc_link->dpcd_caps; + is_pcon = dpcd_caps.dongle_type == DISPLAY_DONGLE_DP_HDMI_CONVERTER; + pcon_allowed = dm_helpers_is_vrr_pcon_compatible( + amdgpu_dm_connector->dc_link, connector->dev); + } + + /* Copy current range and do not touch display_info afterwards */ + copy_range_to_amdgpu_connector(amdgpu_dm_connector, connector); + + /* DP & eDP excluding PCONs */ + if (dc_is_dp_sst_signal(sink->sink_signal) && !is_pcon) { + /* Some eDP panels only have the refresh rate range info in DisplayID */ + if (is_aconn_range_invalid(amdgpu_dm_connector)) + parse_edid_displayid_vrr(amdgpu_dm_connector, edid); + /* + * Many monitors expose AMD vsdb in CAE even for DP and their + * monitor ranges do not contain Range Limits Only flag + */ + if (is_aconn_range_invalid(amdgpu_dm_connector)) + aconn_range_from_vsdb(amdgpu_dm_connector, &vsdb_info); - /* Some eDP panels only have the refresh rate range info in DisplayID */ - if ((connector->display_info.monitor_range.min_vfreq == 0 || - connector->display_info.monitor_range.max_vfreq == 0)) - parse_edid_displayid_vrr(connector, edid); - - if (edid && (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT || - sink->sink_signal == SIGNAL_TYPE_EDP)) { - if (amdgpu_dm_connector->dc_link && - amdgpu_dm_connector->dc_link->dpcd_caps.allow_invalid_MSA_timing_param) { - amdgpu_dm_connector->min_vfreq = connector->display_info.monitor_range.min_vfreq; - amdgpu_dm_connector->max_vfreq = connector->display_info.monitor_range.max_vfreq; - if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10) - freesync_capable = true; - } + /* Try extending range if found in AMD vsdb */ + if (vsdb_info.freesync_supported) + extend_range_from_vsdb(amdgpu_dm_connector, &vsdb_info); - get_amd_vsdb(amdgpu_dm_connector, &vsdb_info); + if (dpcd_caps.allow_invalid_MSA_timing_param) + freesync_capable = is_freesync_capable(amdgpu_dm_connector); - if (vsdb_info.replay_mode) { - amdgpu_dm_connector->vsdb_info.replay_mode = vsdb_info.replay_mode; - amdgpu_dm_connector->vsdb_info.amd_vsdb_version = vsdb_info.amd_vsdb_version; + /* eDP */ + get_amd_vsdb(amdgpu_dm_connector, &vsdb_info); + if (vsdb_did.replay_mode) { + amdgpu_dm_connector->vsdb_info.replay_mode = vsdb_did.replay_mode; + amdgpu_dm_connector->vsdb_info.amd_vsdb_version = vsdb_did.amd_vsdb_version; amdgpu_dm_connector->as_type = ADAPTIVE_SYNC_TYPE_EDP; } - } else if (drm_edid && sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A) { - i = parse_hdmi_amd_vsdb(amdgpu_dm_connector, edid, &vsdb_info); - if (i >= 0) { + /* HDMI and DP -> HDMI PCONs */ + } else if (dc_is_hdmi_signal(sink->sink_signal) || pcon_allowed) { + /* Prefer HDMI VRR */ + if (hdmi_vrr.supported) { + amdgpu_dm_connector->as_type = ADAPTIVE_SYNC_TYPE_HDMI; + aconn_range_from_hdmi(amdgpu_dm_connector, &hdmi_vrr, &vsdb_info); + } else if (vsdb_info.freesync_supported) { amdgpu_dm_connector->vsdb_info = vsdb_info; - sink->edid_caps.freesync_vcp_code = vsdb_info.freesync_mccs_vcp_code; - - if (vsdb_info.freesync_supported) { - amdgpu_dm_connector->min_vfreq = vsdb_info.min_refresh_rate_hz; - amdgpu_dm_connector->max_vfreq = vsdb_info.max_refresh_rate_hz; - if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10) - freesync_capable = true; - - connector->display_info.monitor_range.min_vfreq = vsdb_info.min_refresh_rate_hz; - connector->display_info.monitor_range.max_vfreq = vsdb_info.max_refresh_rate_hz; - } + aconn_range_from_vsdb(amdgpu_dm_connector, &vsdb_info); } - } - if (amdgpu_dm_connector->dc_link) - as_type = dm_get_adaptive_sync_support_type(amdgpu_dm_connector->dc_link); - - if (as_type == FREESYNC_TYPE_PCON_IN_WHITELIST) { - i = parse_hdmi_amd_vsdb(amdgpu_dm_connector, edid, &vsdb_info); - if (i >= 0) { - amdgpu_dm_connector->vsdb_info = vsdb_info; - sink->edid_caps.freesync_vcp_code = vsdb_info.freesync_mccs_vcp_code; - - if (vsdb_info.freesync_supported && vsdb_info.amd_vsdb_version > 0) { - amdgpu_dm_connector->pack_sdp_v1_3 = true; - amdgpu_dm_connector->as_type = as_type; - - amdgpu_dm_connector->min_vfreq = vsdb_info.min_refresh_rate_hz; - amdgpu_dm_connector->max_vfreq = vsdb_info.max_refresh_rate_hz; - if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10) - freesync_capable = true; - - connector->display_info.monitor_range.min_vfreq = vsdb_info.min_refresh_rate_hz; - connector->display_info.monitor_range.max_vfreq = vsdb_info.max_refresh_rate_hz; - } + if (pcon_allowed) { + amdgpu_dm_connector->pack_sdp_v1_3 = true; + amdgpu_dm_connector->as_type = ADAPTIVE_SYNC_TYPE_PCON_ALLOWED; } + + sink->edid_caps.freesync_vcp_code = vsdb_info.freesync_mccs_vcp_code; + freesync_capable = is_freesync_capable(amdgpu_dm_connector); } /* Handle MCCS */ @@ -14188,6 +14301,11 @@ if (dm_con_state) dm_con_state->freesync_capable = freesync_capable; + if (freesync_capable) { + connector->display_info.monitor_range.min_vfreq = amdgpu_dm_connector->min_vfreq; + connector->display_info.monitor_range.max_vfreq = amdgpu_dm_connector->max_vfreq; + } + if (connector->state && amdgpu_dm_connector->dc_link && !freesync_capable && amdgpu_dm_connector->dc_link->replay_settings.config.replay_supported) { amdgpu_dm_connector->dc_link->replay_settings.config.replay_supported = false; diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c 2026-10-13 14:33:30.000000000 +0200 @@ -187,7 +187,7 @@ } EXPORT_IF_KUNIT(amdgpu_dm_fixpt_from_s3132); -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR /* Pre-defined Transfer Functions (TF) * * AMD driver supports pre-defined mathematical functions for transferring diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c 2026-10-13 14:33:30.000000000 +0200 @@ -484,7 +484,7 @@ } #endif -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR /** * dm_crtc_additional_color_mgmt - enable additional color properties * @crtc: DRM CRTC @@ -566,7 +566,7 @@ #if defined(CONFIG_DEBUG_FS) .late_register = amdgpu_dm_crtc_late_register, #endif -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR .atomic_set_property = amdgpu_dm_atomic_crtc_set_property, .atomic_get_property = amdgpu_dm_atomic_crtc_get_property, #endif @@ -792,7 +792,7 @@ drm_mode_crtc_set_gamma_size(&acrtc->base, MAX_COLOR_LEGACY_LUT_ENTRIES); -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR dm_crtc_additional_color_mgmt(&acrtc->base); #endif return 0; diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c 2026-10-13 14:33:30.000000000 +0200 @@ -194,7 +194,12 @@ edid_caps->display_name, AUDIO_INFO_DISPLAY_NAME_SIZE_IN_CHARS); - edid_caps->edid_hdmi = connector->display_info.is_hdmi; + if (connector->display_info.is_hdmi) { + edid_caps->edid_hdmi = true; + edid_caps->allm = connector->display_info.hdmi.allm; + edid_caps->fva = connector->display_info.hdmi.vrr_cap.fva; + edid_caps->hdmi_vrr = connector->display_info.hdmi.vrr_cap.supported; + } if (edid_caps->edid_hdmi) { edid_caps->qs_bit = connector->display_info.rgb_quant_range_selectable; @@ -1577,42 +1582,42 @@ // TODO } -static bool dm_is_freesync_pcon_whitelist(const uint32_t branch_dev_id) +static bool dm_helpers_is_vrr_pcon_allowlist(const uint32_t branch_dev_id) { - bool ret_val = false; - switch (branch_dev_id) { case DP_BRANCH_DEVICE_ID_0060AD: case DP_BRANCH_DEVICE_ID_00E04C: case DP_BRANCH_DEVICE_ID_90CC24: case DP_BRANCH_DEVICE_ID_001CF8: case DP_BRANCH_DEVICE_ID_001FF2: - ret_val = true; - break; - default: - break; + case DP_BRANCH_DEVICE_ID_2B02F0: + return true; } - return ret_val; + return false; } -enum adaptive_sync_type dm_get_adaptive_sync_support_type(struct dc_link *link) +bool dm_helpers_is_vrr_pcon_compatible(const struct dc_link *link, const struct drm_device *dev) { - struct dpcd_caps *dpcd_caps = &link->dpcd_caps; - enum adaptive_sync_type as_type = ADAPTIVE_SYNC_TYPE_NONE; + if (link->dpcd_caps.dongle_type != DISPLAY_DONGLE_DP_HDMI_CONVERTER) + return false; - switch (dpcd_caps->dongle_type) { - case DISPLAY_DONGLE_DP_HDMI_CONVERTER: - if (dpcd_caps->adaptive_sync_caps.dp_adap_sync_caps.bits.ADAPTIVE_SYNC_SDP_SUPPORT == true && - dpcd_caps->allow_invalid_MSA_timing_param == true && - dm_is_freesync_pcon_whitelist(dpcd_caps->branch_dev_id)) - as_type = FREESYNC_TYPE_PCON_IN_WHITELIST; - break; - default: - break; + if (!link->dpcd_caps.allow_invalid_MSA_timing_param) + return false; + + if (!link->dpcd_caps.adaptive_sync_caps.dp_adap_sync_caps.bits.ADAPTIVE_SYNC_SDP_SUPPORT) + return false; + + if (dm_helpers_is_vrr_pcon_allowlist(link->dpcd_caps.branch_dev_id)) + return true; + + if (link->dc->debug.override_pcon_vrr_id_check) { + drm_info(dev, "Overriding VRR PCON check for ID: 0x%06x\n", + link->dpcd_caps.branch_dev_id); + return true; } - return as_type; + return false; } bool dm_helpers_is_fullscreen(struct dc_context *ctx, struct dc_stream_state *stream) @@ -1716,7 +1721,7 @@ if (dc_is_dp_signal(link->connector_signal)) { if ((dpcd_caps->dpcd_rev.raw >= DPCD_REV_14) && (dpcd_caps->dongle_type == DISPLAY_DONGLE_DP_HDMI_CONVERTER) && - dm_is_freesync_pcon_whitelist(dpcd_caps->branch_dev_id) && + dm_helpers_is_vrr_pcon_allowlist(dpcd_caps->branch_dev_id) && (dpcd_caps->adaptive_sync_caps.dp_adap_sync_caps.bits.ADAPTIVE_SYNC_SDP_SUPPORT == true)) mccs_op = true; diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c 2026-10-13 14:33:30.000000000 +0200 @@ -1628,7 +1628,7 @@ drm_atomic_helper_plane_destroy_state(plane, state); } -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR static void dm_atomic_plane_attach_color_mgmt_properties(struct amdgpu_display_manager *dm, struct drm_plane *plane) @@ -1858,7 +1858,7 @@ .atomic_destroy_state = amdgpu_dm_plane_drm_plane_destroy_state, .format_mod_supported = amdgpu_dm_plane_format_mod_supported, .format_mod_supported_async = amdgpu_dm_plane_format_mod_supported, -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR .atomic_set_property = dm_atomic_plane_set_property, .atomic_get_property = dm_atomic_plane_get_property, #endif @@ -1955,7 +1955,7 @@ else drm_plane_helper_add(plane, &dm_plane_helper_funcs); -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR dm_atomic_plane_attach_color_mgmt_properties(dm, plane); #else res = dm_plane_init_colorops(plane); @@ -1980,4 +1980,3 @@ return false; } - diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h --- a/drivers/gpu/drm/amd/display/dc/dc.h 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/display/dc/dc.h 2026-10-13 14:33:30.000000000 +0200 @@ -1117,6 +1117,7 @@ bool scl_reset_length10; bool hdmi20_disable; bool skip_detection_link_training; + bool override_pcon_vrr_id_check; uint32_t edid_read_retry_times; uint8_t force_odm_combine; //bit vector based on otg inst diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h --- a/drivers/gpu/drm/amd/display/dc/dc_types.h 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/display/dc/dc_types.h 2026-10-13 14:33:30.000000000 +0200 @@ -222,9 +222,14 @@ uint32_t max_tmds_clk_mhz; - /*HDMI 2.0 caps*/ + /* HDMI 2.0 caps */ bool lte_340mcsc_scramble; + /* HDMI 2.1 caps */ + bool allm; + bool fva; + bool hdmi_vrr; + bool edid_hdmi; bool hdr_supported; bool rr_capable; diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/display/dc/dm_helpers.h b/drivers/gpu/drm/amd/display/dc/dm_helpers.h --- a/drivers/gpu/drm/amd/display/dc/dm_helpers.h 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/display/dc/dm_helpers.h 2026-10-13 14:33:30.000000000 +0200 @@ -231,10 +231,10 @@ const struct dc_link *link, struct set_config_cmd_payload *payload, enum set_config_status *operation_result); -enum adaptive_sync_type dm_get_adaptive_sync_support_type(struct dc_link *link); enum dc_edid_status dm_helpers_get_sbios_edid(struct dc_link *link, struct dc_edid *edid); +bool dm_helpers_is_vrr_pcon_compatible(const struct dc_link *link, const struct drm_device *dev); bool dm_helpers_is_fullscreen(struct dc_context *ctx, struct dc_stream_state *stream); bool dm_helpers_is_hdr_on(struct dc_context *ctx, struct dc_stream_state *stream); diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/display/dc/dml/dsc/qp_tables.h b/drivers/gpu/drm/amd/display/dc/dml/dsc/qp_tables.h --- a/drivers/gpu/drm/amd/display/dc/dml/dsc/qp_tables.h 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/display/dc/dml/dsc/qp_tables.h 2026-10-13 14:33:30.000000000 +0200 @@ -63,7 +63,7 @@ { 6.5, { 4, 6, 7, 8, 8, 8, 9, 10, 11, 11, 12, 12, 12, 13, 15} }, { 7, { 4, 5, 7, 7, 8, 8, 8, 9, 10, 11, 11, 12, 12, 13, 14} }, { 7.5, { 4, 5, 6, 7, 7, 8, 8, 9, 10, 10, 11, 11, 12, 13, 14} }, - { 8, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} }, + { 8, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 11, 12, 13, 13, 15} }, { 8.5, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} }, { 9, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 11, 13} }, { 9.5, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 11, 13} }, @@ -211,7 +211,7 @@ { 6.5, { 0, 1, 2, 3, 4, 4, 5, 5, 5, 5, 6, 6, 6, 9, 14} }, { 7, { 0, 0, 2, 2, 4, 4, 4, 4, 4, 5, 5, 6, 6, 9, 13} }, { 7.5, { 0, 0, 2, 2, 3, 4, 4, 4, 4, 4, 5, 5, 6, 9, 13} }, - { 8, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 4, 5, 5, 5, 8, 12} }, + { 8, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 13} }, { 8.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 4, 5, 5, 5, 8, 12} }, { 9, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 12} }, { 9.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 12} }, diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c --- a/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c 2026-10-13 14:33:30.000000000 +0200 @@ -123,7 +123,7 @@ *p++ = (int)((bpp <= 12) ? (-8) : ((bpp >= 15) ? (-6) : (-8 + dsc_roundf((bpp - 12) * (2 / 3.0))))); *p++ = (int)((bpp <= 12) ? (-10) : ((bpp >= 15) ? (-8) : (-10 + dsc_roundf((bpp - 12) * (2 / 3.0))))); *p++ = -10; - *p++ = (int)((bpp <= 6) ? (-12) : ((bpp >= 8) ? (-10) : (-12 + dsc_roundf((bpp - 6) * (2 / 2.0))))); + *p++ = (int)((bpp <= 6) ? (-12) : ((bpp >= 8) ? (-12) : (-12 + dsc_roundf((bpp - 6) * (2 / 2.0))))); *p++ = -12; *p++ = -12; *p++ = -12; @@ -257,4 +257,3 @@ rc->rc_buf_thresh[12] = 8000; rc->rc_buf_thresh[13] = 8064; } - diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/display/include/ddc_service_types.h b/drivers/gpu/drm/amd/display/include/ddc_service_types.h --- a/drivers/gpu/drm/amd/display/include/ddc_service_types.h 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/display/include/ddc_service_types.h 2026-10-13 14:33:30.000000000 +0200 @@ -37,6 +37,7 @@ #define DP_BRANCH_DEVICE_ID_001CF8 0x001CF8 #define DP_BRANCH_DEVICE_ID_0060AD 0x0060AD #define DP_BRANCH_DEVICE_ID_001FF2 0x001FF2 +#define DP_BRANCH_DEVICE_ID_2B02F0 0x2B02F0 /* Chrontel CH7218 */ #define DP_BRANCH_HW_REV_10 0x10 #define DP_BRANCH_HW_REV_20 0x20 diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c --- a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c 2026-10-13 14:33:30.000000000 +0200 @@ -27,6 +27,7 @@ #include "dc.h" #include "mod_freesync.h" #include "core_types.h" +#include "mod_info_packet.h" #define MOD_FREESYNC_MAX_CONCURRENT_STREAMS 32 @@ -967,6 +968,9 @@ return; switch (packet_type) { + case PACKET_TYPE_VTEM: + mod_build_vtem_infopacket(stream, vrr, infopacket); + break; case PACKET_TYPE_FS_V3: build_vrr_infopacket_v3(stream->signal, vrr, app_tf, infopacket, stream->freesync_on_desktop); break; diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h --- a/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h 2026-10-13 14:33:30.000000000 +0200 @@ -34,6 +34,8 @@ struct dc_info_packet; struct mod_vrr_params; +#define VTEM_BRR_MAX 1023 + void set_vsc_packet_colorimetry_data( const struct dc_stream_state *stream, struct dc_info_packet *info_packet, @@ -48,6 +50,10 @@ void mod_build_hf_vsif_infopacket(const struct dc_stream_state *stream, struct dc_info_packet *info_packet, int ALLMEnabled, int ALLMValue); +void mod_build_vtem_infopacket(const struct dc_stream_state *stream, + const struct mod_vrr_params *vrr, + struct dc_info_packet *infopacket); + enum adaptive_sync_sdp_version { AS_SDP_VER_0 = 0x0, AS_SDP_VER_1 = 0x1, diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet_types.h b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet_types.h --- a/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet_types.h 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet_types.h 2026-10-13 14:33:30.000000000 +0200 @@ -29,9 +29,10 @@ enum adaptive_sync_type { ADAPTIVE_SYNC_TYPE_NONE = 0, ADAPTIVE_SYNC_TYPE_DP = 1, - FREESYNC_TYPE_PCON_IN_WHITELIST = 2, - FREESYNC_TYPE_PCON_NOT_IN_WHITELIST = 3, + ADAPTIVE_SYNC_TYPE_PCON_ALLOWED = 2, + ADAPTIVE_SYNC_TYPE_PCON_NOT_ALLOWED = 3, ADAPTIVE_SYNC_TYPE_EDP = 4, + ADAPTIVE_SYNC_TYPE_HDMI = 5, }; #endif /* MOD_INFO_PACKET_TYPES_H_ */ diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c --- a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c 2026-10-13 14:33:30.000000000 +0200 @@ -48,6 +48,7 @@ vsc_packet_rev7 = 7, }; +#define HDMI_INFOFRAME_TYPE_EMP 0x7F #define HDMI_INFOFRAME_TYPE_VENDOR 0x81 #define HF_VSIF_VERSION 1 @@ -60,65 +61,52 @@ #define VTEM_PB5 5 #define VTEM_PB6 6 -#define VTEM_MD0 7 -#define VTEM_MD1 8 -#define VTEM_MD2 9 -#define VTEM_MD3 10 - - -// VTEM Byte Masks -//PB0 -#define MASK_VTEM_PB0__RESERVED0 0x01 -#define MASK_VTEM_PB0__SYNC 0x02 -#define MASK_VTEM_PB0__VFR 0x04 -#define MASK_VTEM_PB0__AFR 0x08 -#define MASK_VTEM_PB0__DS_TYPE 0x30 - //0: Periodic pseudo-static EM Data Set - //1: Periodic dynamic EM Data Set - //2: Unique EM Data Set - //3: Reserved -#define MASK_VTEM_PB0__END 0x40 -#define MASK_VTEM_PB0__NEW 0x80 - -//PB1 -#define MASK_VTEM_PB1__RESERVED1 0xFF - -//PB2 -#define MASK_VTEM_PB2__ORGANIZATION_ID 0xFF - //0: This is a Vendor Specific EM Data Set - //1: This EM Data Set is defined by This Specification (HDMI 2.1 r102.clean) - //2: This EM Data Set is defined by CTA-861-G - //3: This EM Data Set is defined by VESA -//PB3 -#define MASK_VTEM_PB3__DATA_SET_TAG_MSB 0xFF -//PB4 -#define MASK_VTEM_PB4__DATA_SET_TAG_LSB 0xFF -//PB5 -#define MASK_VTEM_PB5__DATA_SET_LENGTH_MSB 0xFF -//PB6 -#define MASK_VTEM_PB6__DATA_SET_LENGTH_LSB 0xFF +#define VTEM_ORG_ID 1 +#define VTEM_DATA_SET_TAG 1 +#define VTEM_DATA_SET_LENGTH 4 + +#define VTEM_M_CONST 0 +#define VTEM_FVA_FACTOR 0 + +#define VTEM_BRR_MASK_UPPER 0x03 +#define VTEM_BRR_MASK_LOWER 0xFF + +/* VTEM Byte Offset */ +#define VTEM_PB0 0 +#define VTEM_PB1 1 +#define VTEM_PB2 2 +#define VTEM_PB3 3 +#define VTEM_PB4 4 +#define VTEM_PB5 5 +#define VTEM_PB6 6 + +#define VTEM_MD0 7 +#define VTEM_MD1 8 +#define VTEM_MD2 9 +#define VTEM_MD3 10 + +/* Extended Metadata Packet */ +/* Header */ +#define EMP_LAST_BIT 6 +#define EMP_FIRST_BIT 7 +/* PB0 */ +#define EMP_SNC_BIT 1 +#define EMP_VFR_BIT 2 +#define EMP_AFR_BIT 3 +#define EMP_DST_BIT 4 +#define EMP_END_BIT 6 +#define EMP_NEW_BIT 7 +/* PB7 = MD0 */ +#define VTEM_VRR_BIT 0 +#define VTEM_M_CONST_BIT 1 +#define VTEM_FVA_BIT 4 +/* MD1 Base_Vfront */ +/* MD2 */ +#define VTEM_BRR_UPPER_BIT 0 +#define VTEM_RB_BIT 2 +/* MD3 BRR Lower */ - -//PB7-27 (20 bytes): -//PB7 = MD0 -#define MASK_VTEM_MD0__VRR_EN 0x01 -#define MASK_VTEM_MD0__M_CONST 0x02 -#define MASK_VTEM_MD0__QMS_EN 0x04 -#define MASK_VTEM_MD0__RESERVED2 0x08 -#define MASK_VTEM_MD0__FVA_FACTOR_M1 0xF0 - -//MD1 -#define MASK_VTEM_MD1__BASE_VFRONT 0xFF - -//MD2 -#define MASK_VTEM_MD2__BASE_REFRESH_RATE_98 0x03 -#define MASK_VTEM_MD2__RB 0x04 -#define MASK_VTEM_MD2__NEXT_TFR 0xF8 - -//MD3 -#define MASK_VTEM_MD3__BASE_REFRESH_RATE_07 0xFF - enum ColorimetryRGBDP { ColorimetryRGB_DP_sRGB = 0, ColorimetryRGB_DP_AdobeRGB = 3, @@ -644,6 +632,105 @@ info_packet->valid = true; } +static void build_vtem_infopacket_header(struct dc_info_packet *infopacket) +{ + uint8_t pb0 = 0; + + /* might need logic in the future */ + pb0 |= 0 << EMP_SNC_BIT; + pb0 |= 1 << EMP_VFR_BIT; + pb0 |= 0 << EMP_AFR_BIT; + pb0 |= 0 << EMP_DST_BIT; + pb0 |= 0 << EMP_END_BIT; + pb0 |= 1 << EMP_NEW_BIT; + + infopacket->hb0 = HDMI_INFOFRAME_TYPE_EMP; + infopacket->hb1 = (1 << EMP_FIRST_BIT) | (1 << EMP_LAST_BIT); + infopacket->hb2 = 0; // sequence + + infopacket->sb[VTEM_PB0] = pb0; + infopacket->sb[VTEM_PB2] = VTEM_ORG_ID; + infopacket->sb[VTEM_PB4] = VTEM_DATA_SET_TAG; + infopacket->sb[VTEM_PB6] = VTEM_DATA_SET_LENGTH; +} + +static bool is_hdmi_vic_mode(const struct dc_stream_state *stream) +{ + if (stream->timing.hdmi_vic == 0) + return false; + + if (stream->timing.h_total < 3840 || + stream->timing.v_total < 2160) + return false; + + /* 3D/ALLM forces HDMI VIC -> CTA VIC translation */ + if (stream->view_format != VIEW_3D_FORMAT_NONE) + return false; + + return true; +} + +static void build_vtem_infopacket_data(const struct dc_stream_state *stream, + const struct mod_vrr_params *vrr, + struct dc_info_packet *infopacket) +{ + unsigned int hblank = 0; + unsigned int brr = 0; + bool vrr_active = false; + bool rb = false; + + vrr_active = vrr->state == VRR_STATE_ACTIVE_VARIABLE || + vrr->state == VRR_STATE_ACTIVE_FIXED; + /* + * Enables FreeSync-like behavior by keeping HDMI VRR signalling active + * in fixed refresh rate conditions like normal desktop work/web browsing. + * Functinally behaves like non-VRR mode by keeping the actual refresh + * rate fixed. + */ + if (stream->freesync_on_desktop) + vrr_active |= vrr->state == VRR_STATE_INACTIVE; + + infopacket->sb[VTEM_MD0] = VTEM_M_CONST << VTEM_M_CONST_BIT; + infopacket->sb[VTEM_MD0] |= VTEM_FVA_FACTOR << VTEM_FVA_BIT; + infopacket->sb[VTEM_MD0] |= vrr_active << VTEM_VRR_BIT; + + infopacket->sb[VTEM_MD1] = 0; + infopacket->sb[VTEM_MD2] = 0; + infopacket->sb[VTEM_MD3] = 0; + + if (!vrr_active || is_hdmi_vic_mode(stream)) + return; + /* + * In accordance with CVT 1.2 and CVT 2.1: + * Reduced Blanking standard defines a fixed value of + * 160 for hblank, further reduced to 80 in RB2. RB3 uses + * fixed hblank of 80 pixels + up to 120 additional pixels + * in 8-pixel steps. + */ + hblank = stream->timing.h_total - stream->timing.h_addressable; + rb = (hblank >= 80 && hblank <= 200 && hblank % 8 == 0); + brr = div_u64(mod_freesync_calc_nominal_field_rate(stream), 1000000); + + if (brr > VTEM_BRR_MAX) { + infopacket->valid = false; + return; + } + + infopacket->sb[VTEM_MD1] = (uint8_t) stream->timing.v_front_porch; + infopacket->sb[VTEM_MD2] = rb << VTEM_RB_BIT; + infopacket->sb[VTEM_MD2] |= (brr >> 8) & VTEM_BRR_MASK_UPPER; + infopacket->sb[VTEM_MD3] = brr & VTEM_BRR_MASK_LOWER; +} + +void mod_build_vtem_infopacket(const struct dc_stream_state *stream, + const struct mod_vrr_params *vrr, + struct dc_info_packet *infopacket) +{ + infopacket->valid = true; + build_vtem_infopacket_header(infopacket); + build_vtem_infopacket_data(stream, vrr, infopacket); +} + void mod_build_adaptive_sync_infopacket(const struct dc_stream_state *stream, enum adaptive_sync_type asType, const struct AS_Df_params *param, @@ -658,7 +745,7 @@ if (stream != NULL) mod_build_adaptive_sync_infopacket_v2(stream, param, info_packet); break; - case FREESYNC_TYPE_PCON_IN_WHITELIST: + case ADAPTIVE_SYNC_TYPE_PCON_ALLOWED: case ADAPTIVE_SYNC_TYPE_EDP: if (stream && stream->link->replay_settings.config.replay_supported && stream->link->replay_settings.config.replay_version == DC_VESA_PANEL_REPLAY) @@ -667,7 +754,8 @@ mod_build_adaptive_sync_infopacket_v1(info_packet); break; case ADAPTIVE_SYNC_TYPE_NONE: - case FREESYNC_TYPE_PCON_NOT_IN_WHITELIST: + case ADAPTIVE_SYNC_TYPE_PCON_NOT_ALLOWED: + case ADAPTIVE_SYNC_TYPE_HDMI: default: break; } diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h --- a/drivers/gpu/drm/amd/include/amd_shared.h 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/include/amd_shared.h 2026-10-13 14:33:30.000000000 +0200 @@ -416,6 +416,12 @@ * @DC_SKIP_DETECTION_LT: (0x200000) If set, skip detection link training */ DC_SKIP_DETECTION_LT = 0x200000, + + /** + * @DC_OVERRIDE_PCON_VRR_ID_CHECK: (0x400000) If set, always return true if checking for + * PCON VRR compatibility and print it's ID in kernel log. + */ + DC_OVERRIDE_PCON_VRR_ID_CHECK = 0x400000, }; enum amd_dpm_forced_level; diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c --- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c 2026-10-13 14:33:30.000000000 +0200 @@ -3410,6 +3410,9 @@ struct device_attribute *attr, char *buf) { + if (amdgpu_ignore_min_pcap) + return sysfs_emit(buf, "%i\n", 0); + return amdgpu_hwmon_show_power_cap_generic(dev, attr, buf, PP_PWR_LIMIT_MIN); } diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c 2026-10-13 14:33:30.000000000 +0200 @@ -3001,7 +3001,10 @@ *limit = smu->max_power_limit; break; case SMU_PPT_LIMIT_MIN: - *limit = smu->min_power_limit; + if (amdgpu_ignore_min_pcap) + *limit = 0; + else + *limit = smu->min_power_limit; break; default: return -EINVAL; @@ -3022,7 +3025,7 @@ if (limit_type == SMU_DEFAULT_PPT_LIMIT) { if (!limit) limit = smu->current_power_limit; - if ((limit > smu->max_power_limit) || (limit < smu->min_power_limit)) { + if ((limit > smu->max_power_limit) || (!amdgpu_ignore_min_pcap && (limit < smu->min_power_limit))) { dev_err(smu->adev->dev, "New power limit (%d) is out of range [%d,%d]\n", limit, smu->min_power_limit, smu->max_power_limit); diff -ruN --no-dereference -x .git a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c 2026-10-13 14:33:30.000000000 +0200 @@ -674,6 +674,22 @@ int ret = 0; uint32_t feature_mask[2]; + if (smu->adev->pdev->vendor == 0x1002 && smu->adev->pdev->device == 0x7340 && + smu->adev->pdev->subsystem_vendor == 0x106B && + smu->adev->pdev->subsystem_device == 0x0219) { + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetAllowedFeaturesMaskLow, + 0xABC9AFBB, NULL); + if (ret) + goto failed; + + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetAllowedFeaturesMaskHigh, + 0xFFFFFD42, NULL); + if (ret) + goto failed; + + return 0; + } + if (smu_feature_list_is_empty(smu, SMU_FEATURE_LIST_ALLOWED) || feature->feature_num < SMU_FEATURE_NUM_DEFAULT) { ret = -EINVAL; diff -ruN --no-dereference -x .git a/drivers/gpu/drm/drm_displayid_internal.h b/drivers/gpu/drm/drm_displayid_internal.h --- a/drivers/gpu/drm/drm_displayid_internal.h 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/drm_displayid_internal.h 2026-10-13 14:33:30.000000000 +0200 @@ -67,6 +67,7 @@ #define DATA_BLOCK_2_TILED_DISPLAY_TOPOLOGY 0x28 #define DATA_BLOCK_2_CONTAINER_ID 0x29 #define DATA_BLOCK_2_TYPE_10_FORMULA_TIMING 0x2a +#define DATA_BLOCK_2_ADAPTIVE_SYNC 0x2b #define DATA_BLOCK_2_VENDOR_SPECIFIC 0x7e #define DATA_BLOCK_2_CTA_DISPLAY_ID 0x81 @@ -97,6 +98,7 @@ u8 ext_count; } __packed; +#define DISPLAYID_BLOCK_REV GENMASK(2, 0) struct displayid_block { u8 tag; u8 rev; @@ -125,6 +127,7 @@ __le16 vsw; } __packed; +#define DISPLAYID_BLOCK_PASSTHROUGH_TIMINGS_SUPPORT BIT(3) struct displayid_detailed_timing_block { struct displayid_block base; struct displayid_detailed_timings_1 timings[]; @@ -137,19 +140,28 @@ u8 vrefresh; } __packed; +#define DISPLAYID_BLOCK_DESCRIPTOR_PAYLOAD_BYTES GENMASK(6, 4) struct displayid_formula_timing_block { struct displayid_block base; struct displayid_formula_timings_9 timings[]; } __packed; +#define DISPLAYID_VESA_DP_TYPE GENMASK(2, 0) #define DISPLAYID_VESA_MSO_OVERLAP GENMASK(3, 0) #define DISPLAYID_VESA_MSO_MODE GENMASK(6, 5) +#define DISPLAYID_VESA_DSC_BPP_INT GENMASK(5, 0) +#define DISPLAYID_VESA_DSC_BPP_FRACT GENMASK(3, 0) + +#define DISPLAYID_VESA_DP_TYPE_EDP 0 +#define DISPLAYID_VESA_DP_TYPE_DP 1 struct displayid_vesa_vendor_specific_block { struct displayid_block base; u8 oui[3]; u8 data_structure_type; u8 mso; + u8 dsc_bpp_int; + u8 dsc_bpp_fract; } __packed; /* diff -ruN --no-dereference -x .git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c --- a/drivers/gpu/drm/drm_edid.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/drm_edid.c 2026-10-13 14:33:30.000000000 +0200 @@ -45,6 +45,7 @@ #include #include #include +#include #include #include "drm_crtc_internal.h" @@ -6182,6 +6183,33 @@ hdmi->y420_dc_modes = dc_mask; } +static void drm_parse_hdmi_gaming_info(struct drm_hdmi_info *hdmi, const u8 *db) +{ + struct drm_hdmi_vrr_cap *vrr = &hdmi->vrr_cap; + + if (cea_db_payload_len(db) < 8) + return; + + hdmi->fapa_start_location = db[8] & DRM_EDID_FAPA_START_LOCATION; + hdmi->allm = db[8] & DRM_EDID_ALLM; + vrr->fva = db[8] & DRM_EDID_FVA; + vrr->cnmvrr = db[8] & DRM_EDID_CNMVRR; + vrr->cinema_vrr = db[8] & DRM_EDID_CINEMA_VRR; + vrr->mdelta = db[8] & DRM_EDID_MDELTA; + + if (cea_db_payload_len(db) < 9) + return; + + vrr->vrr_min = db[9] & DRM_EDID_VRR_MIN_MASK; + vrr->supported = (vrr->vrr_min > 0 && vrr->vrr_min <= 48); + + if (cea_db_payload_len(db) < 10) + return; + + vrr->vrr_max = (db[9] & DRM_EDID_VRR_MAX_UPPER_MASK) << 2 | db[10]; + vrr->supported &= (vrr->vrr_max == 0 || vrr->vrr_max >= 100); +} + static void drm_parse_dsc_info(struct drm_hdmi_dsc_cap *hdmi_dsc, const u8 *hf_scds) { @@ -6307,7 +6335,7 @@ } drm_parse_ycbcr420_deep_color_info(connector, hf_scds); - + drm_parse_hdmi_gaming_info(&connector->display_info.hdmi, hf_scds); if (cea_db_payload_len(hf_scds) >= 11 && hf_scds[11]) { drm_parse_dsc_info(hdmi_dsc, hf_scds); dsc_support = true; @@ -6317,6 +6345,18 @@ "[CONNECTOR:%d:%s] HF-VSDB: max TMDS clock: %d KHz, HDMI 2.1 support: %s, DSC 1.2 support: %s\n", connector->base.id, connector->name, max_tmds_clock, str_yes_no(max_frl_rate), str_yes_no(dsc_support)); + drm_dbg_kms(connector->dev, + "[CONNECTOR:%d:%s] FAPA in blanking: %s, ALLM support: %s, Fast Vactive support: %s\n", + connector->base.id, connector->name, str_yes_no(hdmi->fapa_start_location), + str_yes_no(hdmi->allm), str_yes_no(hdmi->vrr_cap.fva)); + drm_dbg_kms(connector->dev, + "[CONNECTOR:%d:%s] Negative M VRR support: %s, CinemaVRR support: %s, Mdelta: %d\n", + connector->base.id, connector->name, str_yes_no(hdmi->vrr_cap.cnmvrr), + str_yes_no(hdmi->vrr_cap.cinema_vrr), hdmi->vrr_cap.mdelta); + drm_dbg_kms(connector->dev, + "[CONNECTOR:%d:%s] VRRmin: %u, VRRmax: %u, VRR supported: %s\n", + connector->base.id, connector->name, hdmi->vrr_cap.vrr_min, + hdmi->vrr_cap.vrr_max, str_yes_no(hdmi->vrr_cap.supported)); } static void drm_parse_hdmi_deep_color_info(struct drm_connector *connector, @@ -6598,12 +6638,53 @@ info->monitor_range.min_vfreq, info->monitor_range.max_vfreq); } -static void drm_parse_vesa_mso_data(struct drm_connector *connector, - const struct displayid_block *block) +#define DISPLAYID_ADAPTIVE_SYNC_DESC_SIZE 6 + +static void drm_update_displayid_adaptive_sync_range(struct drm_connector *connector, + const struct displayid_block *block) +{ + struct drm_monitor_range_info *range = &connector->display_info.monitor_range; + const u8 *data = (const u8 *)(block + 1); + u16 best_min = 0, best_max = 0; + unsigned int best_span = 0; + int i; + + if (block->rev != 0 || block->num_bytes < DISPLAYID_ADAPTIVE_SYNC_DESC_SIZE || + block->num_bytes % DISPLAYID_ADAPTIVE_SYNC_DESC_SIZE) + return; + + for (i = 0; i < block->num_bytes; i += DISPLAYID_ADAPTIVE_SYNC_DESC_SIZE) { + const u8 *desc = &data[i]; + u16 min_vfreq = desc[2]; + u16 max_vfreq = ((((u16)desc[4]) & 0x3) << 8) | desc[3]; + unsigned int span; + + max_vfreq += 1; + if (!min_vfreq || max_vfreq <= min_vfreq) + continue; + + span = max_vfreq - min_vfreq; + if (span <= best_span) + continue; + + best_min = min_vfreq; + best_max = max_vfreq; + best_span = span; + } + + if (best_span) { + range->min_vfreq = best_min; + range->max_vfreq = best_max; + } +} + +static void drm_parse_vesa_specific_block(struct drm_connector *connector, + const struct displayid_block *block) { struct displayid_vesa_vendor_specific_block *vesa = (struct displayid_vesa_vendor_specific_block *)block; struct drm_display_info *info = &connector->display_info; + int dp_type; if (block->num_bytes < 3) { drm_dbg_kms(connector->dev, @@ -6615,51 +6696,73 @@ if (oui(vesa->oui[0], vesa->oui[1], vesa->oui[2]) != VESA_IEEE_OUI) return; - if (sizeof(*vesa) != sizeof(*block) + block->num_bytes) { + if (block->num_bytes < 5) { drm_dbg_kms(connector->dev, "[CONNECTOR:%d:%s] Unexpected VESA vendor block size\n", connector->base.id, connector->name); return; } - switch (FIELD_GET(DISPLAYID_VESA_MSO_MODE, vesa->mso)) { - default: - drm_dbg_kms(connector->dev, "[CONNECTOR:%d:%s] Reserved MSO mode value\n", + dp_type = FIELD_GET(DISPLAYID_VESA_DP_TYPE, vesa->data_structure_type); + if (dp_type > 1) { + drm_dbg_kms(connector->dev, "[CONNECTOR:%d:%s] Reserved dp type value\n", connector->base.id, connector->name); - fallthrough; - case 0: - info->mso_stream_count = 0; - break; - case 1: - info->mso_stream_count = 2; /* 2 or 4 links */ - break; - case 2: - info->mso_stream_count = 4; /* 4 links */ - break; } - if (!info->mso_stream_count) { + /* MSO is only supported for eDP */ + if (dp_type == DISPLAYID_VESA_DP_TYPE_EDP) { + switch (FIELD_GET(DISPLAYID_VESA_MSO_MODE, vesa->mso)) { + default: + drm_dbg_kms(connector->dev, "[CONNECTOR:%d:%s] Reserved MSO mode value\n", + connector->base.id, connector->name); + fallthrough; + case 0: + info->mso_stream_count = 0; + break; + case 1: + info->mso_stream_count = 2; /* 2 or 4 links */ + break; + case 2: + info->mso_stream_count = 4; /* 4 links */ + break; + } + } + + if (info->mso_stream_count) { + info->mso_pixel_overlap = FIELD_GET(DISPLAYID_VESA_MSO_OVERLAP, vesa->mso); + if (info->mso_pixel_overlap > 8) { + drm_dbg_kms(connector->dev, + "[CONNECTOR:%d:%s] Reserved MSO pixel overlap value %u\n", + connector->base.id, connector->name, + info->mso_pixel_overlap); + info->mso_pixel_overlap = 8; + } + drm_dbg_kms(connector->dev, + "[CONNECTOR:%d:%s] MSO stream count %u, pixel overlap %u\n", + connector->base.id, connector->name, + info->mso_stream_count, info->mso_pixel_overlap); + } else { info->mso_pixel_overlap = 0; + } + + if (block->num_bytes < 7) { + /* DSC bpp is optional */ return; } - info->mso_pixel_overlap = FIELD_GET(DISPLAYID_VESA_MSO_OVERLAP, vesa->mso); - if (info->mso_pixel_overlap > 8) { + info->dp_dsc_bpp_x16 = FIELD_GET(DISPLAYID_VESA_DSC_BPP_INT, vesa->dsc_bpp_int) << 4 | + FIELD_GET(DISPLAYID_VESA_DSC_BPP_FRACT, vesa->dsc_bpp_fract); + + if (info->dp_dsc_bpp_x16 > 0) { drm_dbg_kms(connector->dev, - "[CONNECTOR:%d:%s] Reserved MSO pixel overlap value %u\n", + "[CONNECTOR:%d:%s] DSC bits per pixel " FXP_Q4_FMT "\n", connector->base.id, connector->name, - info->mso_pixel_overlap); - info->mso_pixel_overlap = 8; + FXP_Q4_ARGS(info->dp_dsc_bpp_x16)); } - - drm_dbg_kms(connector->dev, - "[CONNECTOR:%d:%s] MSO stream count %u, pixel overlap %u\n", - connector->base.id, connector->name, - info->mso_stream_count, info->mso_pixel_overlap); } -static void drm_update_mso(struct drm_connector *connector, - const struct drm_edid *drm_edid) +static void drm_update_vesa_specific_block(struct drm_connector *connector, + const struct drm_edid *drm_edid) { const struct displayid_block *block; struct displayid_iter iter; @@ -6667,7 +6770,7 @@ displayid_iter_edid_begin(drm_edid, &iter); displayid_iter_for_each(block, &iter) { if (block->tag == DATA_BLOCK_2_VENDOR_SPECIFIC) - drm_parse_vesa_mso_data(connector, block); + drm_parse_vesa_specific_block(connector, block); } displayid_iter_end(&iter); } @@ -6704,6 +6807,7 @@ info->mso_stream_count = 0; info->mso_pixel_overlap = 0; info->max_dsc_bpp = 0; + info->dp_dsc_bpp_x16 = 0; kfree(info->vics); info->vics = NULL; @@ -6720,26 +6824,41 @@ { struct drm_display_info *info = &connector->display_info; const struct displayid_block *block; + bool displayid_base_logged = false; struct displayid_iter iter; displayid_iter_edid_begin(drm_edid, &iter); displayid_iter_for_each(block, &iter) { - drm_dbg_kms(connector->dev, - "[CONNECTOR:%d:%s] DisplayID extension version 0x%02x, primary use 0x%02x\n", - connector->base.id, connector->name, - displayid_version(&iter), - displayid_primary_use(&iter)); - if (displayid_version(&iter) == DISPLAY_ID_STRUCTURE_VER_20 && - (displayid_primary_use(&iter) == PRIMARY_USE_HEAD_MOUNTED_VR || - displayid_primary_use(&iter) == PRIMARY_USE_HEAD_MOUNTED_AR)) - info->non_desktop = true; - /* - * We're only interested in the base section here, no need to - * iterate further. + * Primary use is a DisplayID base section property, but later + * blocks may still carry useful metadata like adaptive sync ranges. */ - break; + if (!displayid_base_logged) { + drm_dbg_kms(connector->dev, + "[CONNECTOR:%d:%s] DisplayID extension version 0x%02x, primary use 0x%02x\n", + connector->base.id, connector->name, + displayid_version(&iter), + displayid_primary_use(&iter)); + if (displayid_version(&iter) == DISPLAY_ID_STRUCTURE_VER_20 && + (displayid_primary_use(&iter) == PRIMARY_USE_HEAD_MOUNTED_VR || + displayid_primary_use(&iter) == PRIMARY_USE_HEAD_MOUNTED_AR)) + info->non_desktop = true; + + displayid_base_logged = true; + } + + if (!info->monitor_range.min_vfreq && !info->monitor_range.max_vfreq && + block->tag == DATA_BLOCK_2_ADAPTIVE_SYNC) + drm_update_displayid_adaptive_sync_range(connector, block); } + + if (info->monitor_range.min_vfreq && info->monitor_range.max_vfreq) + drm_dbg_kms(connector->dev, + "[CONNECTOR:%d:%s] DisplayID adaptive sync refresh rate range is %d Hz - %d Hz\n", + connector->base.id, connector->name, + info->monitor_range.min_vfreq, + info->monitor_range.max_vfreq); + displayid_iter_end(&iter); } @@ -6828,7 +6947,7 @@ if (edid->features & DRM_EDID_FEATURE_RGB_YCRCB422) info->color_formats |= BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422); - drm_update_mso(connector, drm_edid); + drm_update_vesa_specific_block(connector, drm_edid); out: if (drm_edid_has_internal_quirk(connector, EDID_QUIRK_NON_DESKTOP)) { @@ -6858,8 +6977,8 @@ } static struct drm_display_mode *drm_mode_displayid_detailed(struct drm_device *dev, - const struct displayid_detailed_timings_1 *timings, - bool type_7) + const struct displayid_block *block, + const struct displayid_detailed_timings_1 *timings) { struct drm_display_mode *mode; unsigned int pixel_clock = (timings->pixel_clock[0] | @@ -6875,11 +6994,16 @@ unsigned int vsync_width = le16_to_cpu(timings->vsw) + 1; bool hsync_positive = le16_to_cpu(timings->hsync) & (1 << 15); bool vsync_positive = le16_to_cpu(timings->vsync) & (1 << 15); + bool type_7 = block->tag == DATA_BLOCK_2_TYPE_7_DETAILED_TIMING; mode = drm_mode_create(dev); if (!mode) return NULL; + if (type_7 && FIELD_GET(DISPLAYID_BLOCK_REV, block->rev) >= 1) + mode->dsc_passthrough_timings_support = + block->rev & DISPLAYID_BLOCK_PASSTHROUGH_TIMINGS_SUPPORT; + /* resolution is kHz for type VII, and 10 kHz for type I */ mode->clock = type_7 ? pixel_clock : pixel_clock * 10; mode->hdisplay = hactive; @@ -6912,7 +7036,6 @@ int num_timings; struct drm_display_mode *newmode; int num_modes = 0; - bool type_7 = block->tag == DATA_BLOCK_2_TYPE_7_DETAILED_TIMING; /* blocks must be multiple of 20 bytes length */ if (block->num_bytes % 20) return 0; @@ -6921,7 +7044,7 @@ for (i = 0; i < num_timings; i++) { struct displayid_detailed_timings_1 *timings = &det->timings[i]; - newmode = drm_mode_displayid_detailed(connector->dev, timings, type_7); + newmode = drm_mode_displayid_detailed(connector->dev, block, timings); if (!newmode) continue; @@ -6968,7 +7091,8 @@ struct drm_display_mode *newmode; int num_modes = 0; bool type_10 = block->tag == DATA_BLOCK_2_TYPE_10_FORMULA_TIMING; - int timing_size = 6 + ((formula_block->base.rev & 0x70) >> 4); + int timing_size = 6 + + FIELD_GET(DISPLAYID_BLOCK_DESCRIPTOR_PAYLOAD_BYTES, formula_block->base.rev); /* extended blocks are not supported yet */ if (timing_size != 6) diff -ruN --no-dereference -x .git a/drivers/gpu/drm/gud/gud_connector.c b/drivers/gpu/drm/gud/gud_connector.c --- a/drivers/gpu/drm/gud/gud_connector.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/gud/gud_connector.c 2026-10-13 14:33:30.000000000 +0200 @@ -399,11 +399,8 @@ for (i = 0; i < num_modes; i++) { char *mode = &buf[i * GUD_CONNECTOR_TV_MODE_NAME_LEN]; - if (!memchr(mode, '\0', GUD_CONNECTOR_TV_MODE_NAME_LEN)) { - ret = -EIO; - goto free; - } - + /* The device is not trusted to NUL-terminate the name */ + mode[GUD_CONNECTOR_TV_MODE_NAME_LEN - 1] = '\0'; modes[i] = mode; } diff -ruN --no-dereference -x .git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c --- a/drivers/gpu/drm/i915/display/intel_ddi.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/i915/display/intel_ddi.c 2026-10-13 14:33:30.000000000 +0200 @@ -4940,6 +4940,9 @@ if (dig_port->ddi_a_4_lanes) return false; + if (intel_has_quirk(display, QUIRK_DDI_A_FORCE_4_LANES)) + return true; + /* Broxton/Geminilake: Bspec says that DDI_A_4_LANES is the only * supported configuration */ diff -ruN --no-dereference -x .git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c --- a/drivers/gpu/drm/i915/display/intel_dp.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/i915/display/intel_dp.c 2026-10-13 14:33:30.000000000 +0200 @@ -4889,6 +4889,38 @@ intel_edp_set_data_override_rates(intel_dp); } +static void intel_edp_wake_sink(struct intel_dp *intel_dp) +{ + u8 value = 0; + int ret; + + /* + * Read the current sink power state. drm_dp_dpcd_read_byte() already + * retries the AUX transaction internally, so a single read suffices. + * First commercial eDP panels are Ver1.0 or 1.1, on which DPCD + * DP_SET_POWER is supported. + */ + ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_SET_POWER, &value); + + /* + * If the AUX read failed the sink may be asleep and not responding, + * or it read back D3; in either case wake it up to D0. + * In case of AUX read failure which is usually a POR case, the + * remaining bits of register 0x600 is set to '0' on POR. So a bare + * write should be fine. + */ + if (ret < 0 || value == DP_SET_POWER_D3) { + value &= ~DP_SET_POWER_MASK; + value |= DP_SET_POWER_D0; + drm_dp_dpcd_write_byte(&intel_dp->aux, DP_SET_POWER, + value); + /* After setting to D0 need a min of 1ms to wake (Spec DP2.1 sec 2.3.1.2) */ + fsleep(1000); + drm_dp_dpcd_write_byte(&intel_dp->aux, DP_SET_POWER, + value); + } +} + static bool intel_edp_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *connector) { @@ -4899,6 +4931,12 @@ /* this function is meant to be called only once */ drm_WARN_ON(display->drm, intel_dp->dpcd[DP_DPCD_REV] != 0); + /* + * Spec DP2.1 Section 3.5.2.16 page 966. + * Also if sink is asleep, this will wake the sink. + */ + intel_edp_wake_sink(intel_dp); + if (drm_dp_read_dpcd_caps(&intel_dp->aux, intel_dp->dpcd) != 0) return false; diff -ruN --no-dereference -x .git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c --- a/drivers/gpu/drm/i915/display/intel_fbdev.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/i915/display/intel_fbdev.c 2026-10-13 14:33:30.000000000 +0200 @@ -280,10 +280,10 @@ ifbdev->fb = NULL; if (fb && - (sizes->fb_width > fb->base.width || - sizes->fb_height > fb->base.height)) { + (sizes->fb_width != fb->base.width || + sizes->fb_height != fb->base.height)) { drm_dbg_kms(display->drm, - "BIOS fb too small (%dx%d), we require (%dx%d)," + "BIOS fb not valid (%dx%d), we require (%dx%d)," " releasing it\n", fb->base.width, fb->base.height, sizes->fb_width, sizes->fb_height); diff -ruN --no-dereference -x .git a/drivers/gpu/drm/i915/display/intel_frontbuffer.c b/drivers/gpu/drm/i915/display/intel_frontbuffer.c --- a/drivers/gpu/drm/i915/display/intel_frontbuffer.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.c 2026-10-13 14:33:30.000000000 +0200 @@ -97,6 +97,7 @@ might_sleep(); intel_td_flush(display); intel_drrs_flush(display, frontbuffer_bits); + intel_psr_panel_replay_exit(display); intel_psr_flush(display, frontbuffer_bits, origin); intel_fbc_flush(display, frontbuffer_bits, origin); } diff -ruN --no-dereference -x .git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c --- a/drivers/gpu/drm/i915/display/intel_psr.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/i915/display/intel_psr.c 2026-10-13 14:33:30.000000000 +0200 @@ -714,8 +714,14 @@ * To support PSR version 02h and PSR version 03h without * Y-coordinate requirement panels we would need to enable * GTC first. + * + * Early Transport (version 04h) implies Y-coordinate + * support. Accept it without explicit Y-coordinate + * requirement bit. */ - connector->dp.psr_caps.su_support = y_req && + connector->dp.psr_caps.su_support = + (y_req || connector->dp.psr_caps.dpcd[0] == + DP_PSR2_WITH_Y_COORD_ET_SUPPORTED) && intel_alpm_aux_wake_supported(intel_dp); drm_dbg_kms(display->drm, "PSR2 %ssupported\n", connector->dp.psr_caps.su_support ? "" : "not "); @@ -1236,6 +1242,8 @@ psr2_program_idle_frames(intel_dp, psr_compute_idle_frames(intel_dp)); } +static void intel_psr_activate(struct intel_dp *intel_dp); + static void tgl_dc3co_disable_work(struct work_struct *work) { struct intel_dp *intel_dp = @@ -1246,6 +1254,13 @@ if (delayed_work_pending(&intel_dp->psr.dc3co_work)) goto unlock; + /* Panel Replay ALPM cursor-lag workaround re-activation path. */ + if (intel_dp->psr.panel_replay_enabled && + intel_dp->psr.enabled && !intel_dp->psr.active) { + intel_psr_activate(intel_dp); + goto unlock; + } + tgl_psr2_disable_dc3co(intel_dp); unlock: mutex_unlock(&intel_dp->psr.lock); @@ -4721,3 +4736,30 @@ return (pr_support & DP_PANEL_REPLAY_SUPPORT) && !(pr_cap & DP_PANEL_REPLAY_ASYNC_VIDEO_TIMING_NOT_SUPPORTED_IN_PR); } + +/* + * intel_psr_panel_replay_exit - exit Panel Replay during frontbuffer activity + * + * Exits PR on frontbuffer flush and arms dc3co_work with a 50 ms + * delay. Each subsequent flush cancels and rearms the timer. When it + * finally fires, tgl_dc3co_disable_work re-activates PR. + */ +void intel_psr_panel_replay_exit(struct intel_display *display) +{ + struct intel_encoder *encoder; + + for_each_intel_encoder_with_psr(display->drm, encoder) { + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + + mutex_lock(&intel_dp->psr.lock); + if (intel_dp->psr.panel_replay_enabled && + intel_dp->psr.sel_update_enabled) { + if (intel_dp->psr.active) + intel_psr_exit(intel_dp); + mod_delayed_work(display->wq.unordered, + &intel_dp->psr.dc3co_work, + msecs_to_jiffies(50)); + } + mutex_unlock(&intel_dp->psr.lock); + } +} diff -ruN --no-dereference -x .git a/drivers/gpu/drm/i915/display/intel_psr.h b/drivers/gpu/drm/i915/display/intel_psr.h --- a/drivers/gpu/drm/i915/display/intel_psr.h 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/i915/display/intel_psr.h 2026-10-13 14:33:30.000000000 +0200 @@ -40,6 +40,7 @@ void intel_psr_invalidate(struct intel_display *display, unsigned frontbuffer_bits, enum fb_op_origin origin); +void intel_psr_panel_replay_exit(struct intel_display *display); void intel_psr_flush(struct intel_display *display, unsigned frontbuffer_bits, enum fb_op_origin origin); diff -ruN --no-dereference -x .git a/drivers/gpu/drm/i915/display/intel_quirks.c b/drivers/gpu/drm/i915/display/intel_quirks.c --- a/drivers/gpu/drm/i915/display/intel_quirks.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/i915/display/intel_quirks.c 2026-10-13 14:33:30.000000000 +0200 @@ -66,6 +66,18 @@ drm_info(display->drm, "Applying Increase DDI Disabled quirk\n"); } +/* + * In some cases, the firmware might not set the lane count to 4 (for example, + * when booting in some dual GPU Macs with the dGPU as the default GPU), this + * quirk is used to force it as otherwise it might not be possible to compute a + * valid link configuration. + */ +static void quirk_ddi_a_force_4_lanes(struct intel_display *display) +{ + intel_set_quirk(display, QUIRK_DDI_A_FORCE_4_LANES); + drm_info(display->drm, "Applying DDI A Forced 4 Lanes quirk\n"); +} + static void quirk_no_pps_backlight_power_hook(struct intel_display *display) { intel_set_quirk(display, QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK); @@ -262,6 +274,9 @@ /* Xiaomi Book Pro 14 2026 */ { 0xb081, 0x1d72, 0x2424, quirk_disable_psr2 }, + + /* Apple MacBookPro15,1 */ + { 0x3e9b, 0x106b, 0x0176, quirk_ddi_a_force_4_lanes }, }; static const struct intel_dpcd_quirk intel_dpcd_quirks[] = { diff -ruN --no-dereference -x .git a/drivers/gpu/drm/i915/display/intel_quirks.h b/drivers/gpu/drm/i915/display/intel_quirks.h --- a/drivers/gpu/drm/i915/display/intel_quirks.h 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/i915/display/intel_quirks.h 2026-10-13 14:33:30.000000000 +0200 @@ -23,6 +23,7 @@ QUIRK_EDP_LIMIT_RATE_HBR2, QUIRK_DISABLE_EDP_PANEL_REPLAY, QUIRK_DISABLE_PSR2, + QUIRK_DDI_A_FORCE_4_LANES, }; void intel_init_quirks(struct intel_display *display); diff -ruN --no-dereference -x .git a/drivers/gpu/drm/i915/gt/intel_rc6.c b/drivers/gpu/drm/i915/gt/intel_rc6.c --- a/drivers/gpu/drm/i915/gt/intel_rc6.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/i915/gt/intel_rc6.c 2026-10-13 14:33:30.000000000 +0200 @@ -2,9 +2,9 @@ /* * Copyright © 2019 Intel Corporation */ - #include #include +#include #include #include @@ -499,6 +499,17 @@ if (!HAS_RC6(i915)) return false; + if (dmi_match(DMI_PRODUCT_SKU, "IBP1XI08MK1") || + dmi_match(DMI_PRODUCT_SKU, "IBP14I08MK2") || + dmi_match(DMI_PRODUCT_SKU, "IBP1XI08MK2") || + dmi_match(DMI_PRODUCT_SKU, "IBP16I08MK2") || + dmi_match(DMI_PRODUCT_SKU, "OMNIA08IMK1") || + dmi_match(DMI_PRODUCT_SKU, "OMNIA08IMK2")) { + drm_notice(&i915->drm, + "RC6 disabled by quirk\n"); + return false; + } + if (intel_vgpu_active(i915)) return false; diff -ruN --no-dereference -x .git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c 2026-10-13 14:33:30.000000000 +0200 @@ -188,6 +188,11 @@ man->func = &nouveau_vram_manager; + man->cg = drmm_cgroup_register_region(drm->dev, "vram", + drm->gem.vram_available); + if (IS_ERR(man->cg)) + return PTR_ERR(man->cg); + ttm_resource_manager_init(man, &drm->ttm.bdev, drm->gem.vram_available >> PAGE_SHIFT); ttm_set_driver_manager(&drm->ttm.bdev, TTM_PL_VRAM, man); diff -ruN --no-dereference -x .git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c --- a/drivers/gpu/drm/ttm/ttm_bo.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/ttm/ttm_bo.c 2026-10-13 14:33:30.000000000 +0200 @@ -488,6 +488,118 @@ return ret; } +struct ttm_bo_alloc_state { + /** @charge_pool: The memory pool the resource is charged to */ + struct dmem_cgroup_pool_state *charge_pool; + /** @limit_pool: Which pool limit we should test against */ + struct dmem_cgroup_pool_state *limit_pool; + /** @in_evict: Whether we are currently evicting buffers */ + bool in_evict; + /** @may_try_low: If only unprotected BOs, i.e. BOs whose cgroup + * is exceeding its dmem low/min protection, should be considered for eviction + */ + bool may_try_low; +}; + +/** + * ttm_bo_alloc_at_place - Attempt allocating a BO's backing store in a place + * + * @bo: The buffer to allocate the backing store of + * @place: The place to attempt allocation in + * @ctx: ttm_operation_ctx associated with this allocation + * @force_space: If we should evict buffers to force space + * @res: On allocation success, the resulting struct ttm_resource. + * @alloc_state: Object holding allocation state such as charged cgroups. + * + * Returns: + * -EBUSY: No space available, but allocation should be retried with ttm_bo_evict_alloc. + * -ENOSPC: No space available, allocation should not be retried. + * -ERESTARTSYS: An interruptible sleep was interrupted by a signal. + * + */ +static int ttm_bo_alloc_at_place(struct ttm_buffer_object *bo, + const struct ttm_place *place, + bool force_space, + struct ttm_resource **res, + struct ttm_bo_alloc_state *alloc_state) +{ + bool may_evict; + int ret; + + may_evict = !alloc_state->in_evict && force_space && + place->mem_type != TTM_PL_SYSTEM; + if (!alloc_state->charge_pool) { + ret = ttm_resource_try_charge(bo, place, &alloc_state->charge_pool, + force_space ? &alloc_state->limit_pool + : NULL); + if (ret) { + /* + * -EAGAIN means the charge failed, which we treat + * like an allocation failure. Therefore, return an + * error code indicating the allocation failed - + * either -EBUSY if the allocation should be + * retried with eviction, or -ENOSPC if there should + * be no second attempt. + */ + if (!alloc_state->in_evict) + alloc_state->may_try_low = may_evict; + if (ret == -EAGAIN) + ret = may_evict ? -EBUSY : -ENOSPC; + return ret; + } + } + + /* + * cgroup protection plays a special role in eviction. + * Conceptually, protection of memory via the dmem cgroup controller + * entitles the protected cgroup to use a certain amount of memory. + * There are two types of protection - the 'low' limit is a + * "best-effort" protection, whereas the 'min' limit provides a hard + * guarantee that memory within the cgroup's allowance will not be + * evicted under any circumstance. + * + * To faithfully model this concept in TTM, we also need to take cgroup + * protection into account when allocating. When allocation in one + * place fails, TTM will default to trying other places first before + * evicting. + * If the allocation is covered by dmem cgroup protection, however, + * this prevents the allocation from using the memory it is "entitled" + * to. To make sure unprotected allocations cannot push new protected + * allocations out of places they are "entitled" to use, we should + * evict buffers not covered by any cgroup protection, if this + * allocation is covered by cgroup protection. + * + * Buffers covered by 'min' protection are a special case - the 'min' + * limit is a stronger guarantee than 'low', and thus buffers protected + * by 'low' but not 'min' should also be considered for eviction. + * Buffers protected by 'min' will never be considered for eviction + * anyway, so the regular eviction path should be triggered here. + * Buffers protected by 'low' but not 'min' will take a special + * eviction path that only evicts buffers covered by neither 'low' or + * 'min' protections. + */ + if (!alloc_state->in_evict) { + may_evict |= dmem_cgroup_below_min(NULL, alloc_state->charge_pool); + alloc_state->may_try_low = may_evict; + + may_evict |= dmem_cgroup_below_low(NULL, alloc_state->charge_pool); + } + + ret = ttm_resource_alloc(bo, place, res, alloc_state->charge_pool); + if (ret) { + if (ret == -ENOSPC && may_evict) + ret = -EBUSY; + return ret; + } + + /* + * Ownership of charge_pool has been transferred to the TTM resource, + * don't make the caller think we still hold a reference to it. + */ + alloc_state->charge_pool = NULL; + return 0; +} + /** * struct ttm_bo_evict_walk - Parameters for the evict walk. */ @@ -503,22 +615,61 @@ /** @evicted: Number of successful evictions. */ unsigned long evicted; - /** @limit_pool: Which pool limit we should test against */ - struct dmem_cgroup_pool_state *limit_pool; /** @try_low: Whether we should attempt to evict BO's with low watermark threshold */ bool try_low; /** @hit_low: If we cannot evict a bo when @try_low is false (first pass) */ bool hit_low; + + /** @alloc_state: State associated with the allocation attempt. */ + struct ttm_bo_alloc_state *alloc_state; }; static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) { struct ttm_bo_evict_walk *evict_walk = container_of(walk, typeof(*evict_walk), walk); + struct dmem_cgroup_pool_state *limit_pool, *ancestor = NULL; + bool evict_valuable; s64 lret; - if (!dmem_cgroup_state_evict_valuable(evict_walk->limit_pool, bo->resource->css, - evict_walk->try_low, &evict_walk->hit_low)) + /* + * If may_try_low is not set, then we're trying to evict unprotected + * buffers in favor of a protected allocation for charge_pool. Explicitly skip + * buffers belonging to the same cgroup here - that cgroup is definitely protected, + * even though dmem_cgroup_state_evict_valuable would allow the eviction because a + * cgroup is always allowed to evict from itself even if it is protected. + */ + if (!evict_walk->alloc_state->may_try_low && + bo->resource->css == evict_walk->alloc_state->charge_pool) + return 0; + + limit_pool = evict_walk->alloc_state->limit_pool; + /* + * If there is no explicit limit pool, find the root of the shared subtree between + * evictor and evictee. This is important so that recursive protection rules can + * apply properly: Recursive protection distributes cgroup protection afforded + * to a parent cgroup but not used explicitly by a child cgroup between all child + * cgroups (see docs of effective_protection in mm/page_counter.c). However, when + * direct siblings compete for memory, siblings that were explicitly protected + * should get prioritized over siblings that weren't. This only happens correctly + * when the root of the shared subtree is passed to + * dmem_cgroup_state_evict_valuable. Otherwise, the effective-protection + * calculation cannot distinguish direct siblings from unrelated subtrees and the + * calculated protection ends up wrong. + */ + if (!limit_pool) { + ancestor = dmem_cgroup_get_common_ancestor(bo->resource->css, + evict_walk->alloc_state->charge_pool); + limit_pool = ancestor; + } + + evict_valuable = dmem_cgroup_state_evict_valuable(limit_pool, bo->resource->css, + evict_walk->try_low, + &evict_walk->hit_low); + if (ancestor) + dmem_cgroup_pool_state_put(ancestor); + + if (!evict_valuable) return 0; if (bo->pin_count || !bo->bdev->funcs->eviction_valuable(bo, evict_walk->place)) @@ -537,8 +688,10 @@ evict_walk->evicted++; if (evict_walk->res) - lret = ttm_resource_alloc(evict_walk->evictor, evict_walk->place, - evict_walk->res, NULL); + lret = ttm_bo_alloc_at_place(evict_walk->evictor, + evict_walk->place, false, + evict_walk->res, + evict_walk->alloc_state); if (lret == 0) return 1; out: @@ -560,7 +713,7 @@ struct ttm_operation_ctx *ctx, struct ww_acquire_ctx *ticket, struct ttm_resource **res, - struct dmem_cgroup_pool_state *limit_pool) + struct ttm_bo_alloc_state *state) { struct ttm_bo_evict_walk evict_walk = { .walk = { @@ -573,15 +726,21 @@ .place = place, .evictor = evictor, .res = res, - .limit_pool = limit_pool, + .alloc_state = state, }; s64 lret; + state->in_evict = true; + evict_walk.walk.arg.trylock_only = true; lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); - /* One more attempt if we hit low limit? */ - if (!lret && evict_walk.hit_low) { + /* If we failed to find enough BOs to evict, but we skipped over + * some BOs because they were covered by dmem low protection, retry + * evicting these protected BOs too, except if we're told not to + * consider protected BOs at all. + */ + if (!lret && evict_walk.hit_low && state->may_try_low) { evict_walk.try_low = true; lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); } @@ -602,11 +761,13 @@ } while (!lret && evict_walk.evicted); /* We hit the low limit? Try once more */ - if (!lret && evict_walk.hit_low && !evict_walk.try_low) { + if (!lret && evict_walk.hit_low && !evict_walk.try_low && + state->may_try_low) { evict_walk.try_low = true; goto retry; } out: + state->in_evict = false; if (lret < 0) return lret; if (lret == 0) @@ -724,9 +885,8 @@ for (i = 0; i < placement->num_placement; ++i) { const struct ttm_place *place = &placement->placement[i]; - struct dmem_cgroup_pool_state *limit_pool = NULL; + struct ttm_bo_alloc_state alloc_state = {}; struct ttm_resource_manager *man; - bool may_evict; man = ttm_manager_type(bdev, place->mem_type); if (!man || !ttm_resource_manager_used(man)) @@ -736,25 +896,30 @@ TTM_PL_FLAG_FALLBACK)) continue; - may_evict = (force_space && place->mem_type != TTM_PL_SYSTEM); - ret = ttm_resource_alloc(bo, place, res, force_space ? &limit_pool : NULL); - if (ret) { - if (ret != -ENOSPC) { - dmem_cgroup_pool_state_put(limit_pool); - return ret; - } - if (!may_evict) { - dmem_cgroup_pool_state_put(limit_pool); - continue; - } + ret = ttm_bo_alloc_at_place(bo, place, force_space, res, + &alloc_state); + if (ret == -ENOSPC) { + dmem_cgroup_uncharge(alloc_state.charge_pool, bo->base.size); + dmem_cgroup_pool_state_put(alloc_state.limit_pool); + continue; + } else if (ret == -EBUSY) { ret = ttm_bo_evict_alloc(bdev, man, place, bo, ctx, - ticket, res, limit_pool); - dmem_cgroup_pool_state_put(limit_pool); - if (ret == -EBUSY) - continue; - if (ret) + ticket, res, &alloc_state); + + dmem_cgroup_pool_state_put(alloc_state.limit_pool); + + if (ret) { + dmem_cgroup_uncharge(alloc_state.charge_pool, + bo->base.size); + if (ret == -EBUSY) + continue; return ret; + } + } else if (ret) { + dmem_cgroup_uncharge(alloc_state.charge_pool, bo->base.size); + dmem_cgroup_pool_state_put(alloc_state.limit_pool); + return ret; } ret = ttm_bo_add_pipelined_eviction_fences(bo, man, ctx->no_wait_gpu); diff -ruN --no-dereference -x .git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c --- a/drivers/gpu/drm/ttm/ttm_resource.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/ttm/ttm_resource.c 2026-10-13 14:33:30.000000000 +0200 @@ -386,33 +386,52 @@ } EXPORT_SYMBOL(ttm_resource_fini); +/** + * ttm_resource_try_charge - charge a resource manager's cgroup pool + * @bo: buffer for which an allocation should be charged + * @place: where the allocation is attempted to be placed + * @ret_pool: on charge success, the pool that was charged + * @ret_limit_pool: on charge failure, the pool responsible for the failure + * + * Should be used to charge cgroups before attempting resource allocation. + * When charging succeeds, the value of ret_pool should be passed to + * ttm_resource_alloc. + * + * Returns: 0 on charge success, negative errno on failure. + */ +int ttm_resource_try_charge(struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct dmem_cgroup_pool_state **ret_pool, + struct dmem_cgroup_pool_state **ret_limit_pool) +{ + struct ttm_resource_manager *man = + ttm_manager_type(bo->bdev, place->mem_type); + + if (!man->cg) { + *ret_pool = NULL; + if (ret_limit_pool) + *ret_limit_pool = NULL; + return 0; + } + + return dmem_cgroup_try_charge(man->cg, bo->base.size, ret_pool, + ret_limit_pool); +} + int ttm_resource_alloc(struct ttm_buffer_object *bo, const struct ttm_place *place, struct ttm_resource **res_ptr, - struct dmem_cgroup_pool_state **ret_limit_pool) + struct dmem_cgroup_pool_state *charge_pool) { struct ttm_resource_manager *man = ttm_manager_type(bo->bdev, place->mem_type); - struct dmem_cgroup_pool_state *pool = NULL; int ret; - if (man->cg) { - ret = dmem_cgroup_try_charge(man->cg, bo->base.size, &pool, ret_limit_pool); - if (ret) { - if (ret == -EAGAIN) - ret = -ENOSPC; - return ret; - } - } - ret = man->func->alloc(man, bo, place, res_ptr); - if (ret) { - if (pool) - dmem_cgroup_uncharge(pool, bo->base.size); + if (ret) return ret; - } - (*res_ptr)->css = pool; + (*res_ptr)->css = charge_pool; spin_lock(&bo->bdev->lru_lock); ttm_resource_add_bulk_move(*res_ptr, bo); diff -ruN --no-dereference -x .git a/drivers/gpu/drm/xe/display/xe_display_bo.c b/drivers/gpu/drm/xe/display/xe_display_bo.c --- a/drivers/gpu/drm/xe/display/xe_display_bo.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/drm/xe/display/xe_display_bo.c 2026-10-13 14:33:30.000000000 +0200 @@ -147,33 +147,12 @@ struct xe_device *xe = to_xe_device(drm); struct xe_bo *obj; - obj = ERR_PTR(-ENODEV); - - if (xe_display_bo_fbdev_prefer_stolen(xe, size)) { - obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), - size, - ttm_bo_type_kernel, - XE_BO_FLAG_FORCE_WC | - XE_BO_FLAG_STOLEN | - XE_BO_FLAG_GGTT, - false); - if (!IS_ERR(obj)) - drm_info(&xe->drm, "Allocated fbdev into stolen\n"); - else - drm_info(&xe->drm, "Allocated fbdev into stolen failed: %li\n", PTR_ERR(obj)); - } else { - drm_info(&xe->drm, "Allocating fbdev: Stolen memory not preferred.\n"); - } - - if (IS_ERR(obj)) { - obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), size, - ttm_bo_type_kernel, - XE_BO_FLAG_FORCE_WC | - XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) | - XE_BO_FLAG_GGTT, - false); - } - + obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), size, + ttm_bo_type_kernel, + XE_BO_FLAG_FORCE_WC | + XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) | + XE_BO_FLAG_GGTT, + false); if (IS_ERR(obj)) { drm_err(&xe->drm, "failed to allocate framebuffer (%pe)\n", obj); return ERR_PTR(-ENOMEM); diff -ruN --no-dereference -x .git a/drivers/gpu/vga/vga_switcheroo.c b/drivers/gpu/vga/vga_switcheroo.c --- a/drivers/gpu/vga/vga_switcheroo.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/gpu/vga/vga_switcheroo.c 2026-10-13 14:33:30.000000000 +0200 @@ -31,6 +31,7 @@ #define pr_fmt(fmt) "vga_switcheroo: " fmt #include +#include #include #include #include @@ -436,12 +437,8 @@ bool vga_switcheroo_client_probe_defer(struct pci_dev *pdev) { if (pci_is_display(pdev)) { - /* - * apple-gmux is needed on pre-retina MacBook Pro - * to probe the panel if pdev is the inactive GPU. - */ - if (apple_gmux_present() && pdev != vga_default_device() && - !vgasr_priv.handler_flags) + if (apple_gmux_present() && !vgasr_priv.handler_flags && + (pdev != vga_default_device() || dmi_check_system(t2_mac_tbl))) return true; } diff -ruN --no-dereference -x .git a/drivers/hid/Kconfig b/drivers/hid/Kconfig --- a/drivers/hid/Kconfig 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/hid/Kconfig 2026-10-13 14:33:30.000000000 +0200 @@ -743,7 +743,8 @@ Support for the Apple Magic Mouse/Trackpad multi-touch. Say Y here if you want support for the multi-touch features of the - Apple Wireless "Magic" Mouse and the Apple Wireless "Magic" Trackpad. + Apple Wireless "Magic" Mouse, the Apple Wireless "Magic" Trackpad and + force touch Trackpads in Macbooks starting from 2015. config HID_MALTRON tristate "Maltron L90 keyboard" @@ -1495,4 +1496,8 @@ source "drivers/hid/usbhid/Kconfig" +source "drivers/hid/spi-hid/Kconfig" + +source "drivers/hid/dockchannel-hid/Kconfig" + endif # HID_SUPPORT diff -ruN --no-dereference -x .git a/drivers/hid/Makefile b/drivers/hid/Makefile --- a/drivers/hid/Makefile 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/hid/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -179,6 +179,12 @@ obj-$(CONFIG_AMD_SFH_HID) += amd-sfh-hid/ +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid/ + +obj-$(CONFIG_SPI_HID_APPLE_CORE) += spi-hid/ + +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid/ + obj-$(CONFIG_SURFACE_HID_CORE) += surface-hid/ obj-$(CONFIG_INTEL_THC_HID) += intel-thc-hid/ diff -ruN --no-dereference -x .git a/drivers/hid/dockchannel-hid/Kconfig b/drivers/hid/dockchannel-hid/Kconfig --- a/drivers/hid/dockchannel-hid/Kconfig 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/hid/dockchannel-hid/Kconfig 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +menu "DockChannel HID support" + depends on APPLE_DOCKCHANNEL + +config HID_DOCKCHANNEL + tristate "HID over DockChannel transport layer for Apple Silicon SoCs" + depends on APPLE_DOCKCHANNEL && INPUT && OF && HID + help + Say Y here if you use an M2 or later Apple Silicon based laptop. + The keyboard and touchpad are HID based devices connected via the + proprietary DockChannel interface. + +endmenu diff -ruN --no-dereference -x .git a/drivers/hid/dockchannel-hid/Makefile b/drivers/hid/dockchannel-hid/Makefile --- a/drivers/hid/dockchannel-hid/Makefile 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/hid/dockchannel-hid/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# +# Makefile for DockChannel HID transport drivers +# + +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid.o diff -ruN --no-dereference -x .git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,1213 @@ +/* + * SPDX-License-Identifier: GPL-2.0 OR MIT + * + * Apple DockChannel HID transport driver + * + * Copyright The Asahi Linux Contributors + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../hid-ids.h" + +#define COMMAND_TIMEOUT_MS 1000 +#define START_TIMEOUT_MS 2000 + +#define MAX_INTERFACES 16 + +/* Data + checksum */ +#define MAX_PKT_SIZE (0xffff + 4) + +#define DCHID_CHANNEL_CMD 0x11 +#define DCHID_CHANNEL_REPORT 0x12 + +struct dchid_hdr { + u8 hdr_len; + u8 channel; + u16 length; + u8 seq; + u8 iface; + u16 pad; +} __packed; + +#define IFACE_COMM 0 + +#define FLAGS_GROUP GENMASK(7, 6) +#define FLAGS_REQ GENMASK(5, 0) + +#define REQ_SET_REPORT 0 +#define REQ_GET_REPORT 1 + +struct dchid_subhdr { + u8 flags; + u8 unk; + u16 length; + u32 retcode; +} __packed; + +#define EVENT_GPIO_CMD 0xa0 +#define EVENT_INIT 0xf0 +#define EVENT_READY 0xf1 + +struct dchid_init_hdr { + u8 type; + u8 unk1; + u8 unk2; + u8 iface; + char name[16]; + u8 more_packets; + u8 unkpad; +} __packed; + +#define INIT_HID_DESCRIPTOR 0 +#define INIT_GPIO_REQUEST 1 +#define INIT_TERMINATOR 2 +#define INIT_PRODUCT_NAME 7 + +#define CMD_RESET_INTERFACE 0x40 +#define CMD_SEND_FIRMWARE 0x95 +#define CMD_ENABLE_INTERFACE 0xb4 +#define CMD_ACK_GPIO_CMD 0xa1 + +struct dchid_init_block_hdr { + u16 type; + u16 length; +} __packed; + +#define MAX_GPIO_NAME 32 + +struct dchid_gpio_request { + u16 unk; + u16 id; + char name[MAX_GPIO_NAME]; +} __packed; + +struct dchid_gpio_cmd { + u8 type; + u8 iface; + u8 gpio; + u8 unk; + u8 cmd; +} __packed; + +struct dchid_gpio_ack { + u8 type; + u32 retcode; + u8 cmd[]; +} __packed; + +#define STM_REPORT_ID 0x10 +#define STM_REPORT_SERIAL 0x11 +#define STM_REPORT_KEYBTYPE 0x14 + +struct dchid_stm_id { + u8 unk; + u16 vendor_id; + u16 product_id; + u16 version_number; + u8 unk2; + u8 unk3; + u8 keyboard_type; + u8 serial_length; + /* Serial follows, but we grab it with a different report. */ +} __packed; + +#define FW_MAGIC 0x46444948 +#define FW_VER 1 + +struct fw_header { + u32 magic; + u32 version; + u32 hdr_length; + u32 data_length; + u32 iface_offset; +} __packed; + +struct dchid_work { + struct work_struct work; + struct dchid_iface *iface; + + struct dchid_hdr hdr; + u8 data[]; +}; + +struct dchid_iface { + struct dockchannel_hid *dchid; + struct hid_device *hid; + struct workqueue_struct *wq; + + bool creating; + struct work_struct create_work; + + int index; + const char *name; + const struct device_node *of_node; + + uint8_t tx_seq; + bool deferred; + bool starting; + bool open; + struct completion ready; + + void *hid_desc; + size_t hid_desc_len; + + struct gpio_desc *gpio; + char gpio_name[MAX_GPIO_NAME]; + int gpio_id; + + struct mutex out_mutex; + u32 out_flags; + int out_report; + u32 retcode; + void *resp_buf; + size_t resp_size; + struct completion out_complete; + + u32 keyboard_layout_id; +}; + +struct dockchannel_hid { + struct device *dev; + struct dockchannel *dc; + struct device_link *helper_link; + + bool id_ready; + struct dchid_stm_id device_id; + char serial[64]; + + struct dchid_iface *comm; + struct dchid_iface *ifaces[MAX_INTERFACES]; + + u8 pkt_buf[MAX_PKT_SIZE]; + + /* Workqueue to asynchronously create HID devices */ + struct workqueue_struct *new_iface_wq; +}; + +static ssize_t apple_layout_id_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct dchid_iface *iface = hdev->driver_data; + + return scnprintf(buf, PAGE_SIZE, "%d\n", iface->keyboard_layout_id); +} + +static DEVICE_ATTR_RO(apple_layout_id); + +static struct dchid_iface * +dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) +{ + struct dchid_iface *iface; + + if (index >= MAX_INTERFACES) { + dev_err(dchid->dev, "Interface index %d out of range\n", index); + return NULL; + } + + if (dchid->ifaces[index]) + return dchid->ifaces[index]; + + iface = devm_kzalloc(dchid->dev, sizeof(struct dchid_iface), GFP_KERNEL); + if (!iface) + return NULL; + + iface->index = index; + iface->name = devm_kstrdup(dchid->dev, name, GFP_KERNEL); + iface->dchid = dchid; + iface->out_report= -1; + init_completion(&iface->out_complete); + init_completion(&iface->ready); + mutex_init(&iface->out_mutex); + iface->wq = alloc_ordered_workqueue("dchid-%s", WQ_MEM_RECLAIM, iface->name); + if (!iface->wq) + return NULL; + + /* Comm is not a HID subdevice */ + if (!strcmp(name, "comm")) { + dchid->ifaces[index] = iface; + return iface; + } + + iface->of_node = of_get_child_by_name(dchid->dev->of_node, name); + if (!iface->of_node) { + dev_warn(dchid->dev, "No OF node for subdevice %s, ignoring.", name); + return NULL; + } + + dchid->ifaces[index] = iface; + return iface; +} + +static u32 dchid_checksum(void *p, size_t length) +{ + u32 sum = 0; + + while (length >= 4) { + sum += get_unaligned_le32(p); + p += 4; + length -= 4; + } + + WARN_ON_ONCE(length); + return sum; +} + +static int dchid_send(struct dchid_iface *iface, u32 flags, void *msg, size_t size) +{ + u32 checksum = 0xffffffff; + size_t wsize = round_down(size, 4); + size_t tsize = size - wsize; + int ret; + struct { + struct dchid_hdr hdr; + struct dchid_subhdr sub; + } __packed h; + + memset(&h, 0, sizeof(h)); + h.hdr.hdr_len = sizeof(h.hdr); + h.hdr.channel = DCHID_CHANNEL_CMD; + h.hdr.length = round_up(size, 4) + sizeof(h.sub); + h.hdr.seq = iface->tx_seq; + h.hdr.iface = iface->index; + h.sub.flags = flags; + h.sub.length = size; + + ret = dockchannel_send(iface->dchid->dc, &h, sizeof(h)); + if (ret < 0) + return ret; + checksum -= dchid_checksum(&h, sizeof(h)); + + ret = dockchannel_send(iface->dchid->dc, msg, wsize); + if (ret < 0) + return ret; + checksum -= dchid_checksum(msg, wsize); + + if (tsize) { + u8 tail[4] = {0, 0, 0, 0}; + + memcpy(tail, msg + wsize, tsize); + ret = dockchannel_send(iface->dchid->dc, tail, sizeof(tail)); + if (ret < 0) + return ret; + checksum -= dchid_checksum(tail, sizeof(tail)); + } + + ret = dockchannel_send(iface->dchid->dc, &checksum, sizeof(checksum)); + if (ret < 0) + return ret; + + return 0; +} + +static int dchid_cmd(struct dchid_iface *iface, u32 type, u32 req, + void *data, size_t size, void *resp_buf, size_t resp_size) +{ + int ret; + int report_id = *(u8*)data; + + mutex_lock(&iface->out_mutex); + + WARN_ON(iface->out_report != -1); + iface->out_report = report_id; + iface->out_flags = FIELD_PREP(FLAGS_GROUP, type) | FIELD_PREP(FLAGS_REQ, req); + iface->resp_buf = resp_buf; + iface->resp_size = resp_size; + reinit_completion(&iface->out_complete); + + ret = dchid_send(iface, iface->out_flags, data, size); + if (ret < 0) + goto done; + + if (!wait_for_completion_timeout(&iface->out_complete, msecs_to_jiffies(COMMAND_TIMEOUT_MS))) { + dev_err(iface->dchid->dev, "output report 0x%x to iface %d (%s) timed out\n", + report_id, iface->index, iface->name); + ret = -ETIMEDOUT; + goto done; + } + + ret = iface->resp_size; + if (iface->retcode) { + dev_err(iface->dchid->dev, + "output report 0x%x to iface %d (%s) failed with err 0x%x\n", + report_id, iface->index, iface->name, iface->retcode); + ret = -EIO; + } + +done: + iface->tx_seq++; + iface->out_report = -1; + iface->out_flags = 0; + iface->resp_buf = NULL; + iface->resp_size = 0; + mutex_unlock(&iface->out_mutex); + return ret; +} + +static int dchid_comm_cmd(struct dockchannel_hid *dchid, void *cmd, size_t size) +{ + return dchid_cmd(dchid->comm, HID_FEATURE_REPORT, REQ_SET_REPORT, cmd, size, NULL, 0); +} + +static int dchid_enable_interface(struct dchid_iface *iface) +{ + u8 msg[] = { CMD_ENABLE_INTERFACE, iface->index }; + + return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); +} + +static int dchid_reset_interface(struct dchid_iface *iface, int state) +{ + u8 msg[] = { CMD_RESET_INTERFACE, 1, iface->index, state }; + + return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); +} + +static int dchid_send_firmware(struct dchid_iface *iface, void *firmware, size_t size) +{ + struct { + u8 cmd; + u8 unk1; + u8 unk2; + u8 iface; + u64 addr; + u32 size; + } __packed msg = { + .cmd = CMD_SEND_FIRMWARE, + .unk1 = 2, + .unk2 = 0, + .iface = iface->index, + .size = size, + }; + dma_addr_t addr; + void *buf = dmam_alloc_coherent(iface->dchid->dev, size, &addr, GFP_KERNEL); + + if (IS_ERR_OR_NULL(buf)) + return buf ? PTR_ERR(buf) : -ENOMEM; + + msg.addr = addr; + memcpy(buf, firmware, size); + wmb(); + + return dchid_comm_cmd(iface->dchid, &msg, sizeof(msg)); +} + +static int dchid_get_firmware(struct dchid_iface *iface, void **firmware, size_t *size) +{ + int ret; + const char *fw_name; + const struct firmware *fw; + struct fw_header *hdr; + u8 *fw_data; + + ret = of_property_read_string(iface->of_node, "firmware-name", &fw_name); + if (ret) { + /* Firmware is only for some devices */ + *firmware = NULL; + *size = 0; + return 0; + } + + ret = request_firmware(&fw, fw_name, iface->dchid->dev); + if (ret) + return ret; + + hdr = (struct fw_header *)fw->data; + + if (hdr->magic != FW_MAGIC || hdr->version != FW_VER || + hdr->hdr_length < sizeof(*hdr) || hdr->hdr_length > fw->size || + (hdr->hdr_length + (size_t)hdr->data_length) > fw->size || + hdr->iface_offset >= hdr->data_length) { + dev_warn(iface->dchid->dev, "%s: invalid firmware header\n", + fw_name); + ret = -EINVAL; + goto done; + } + + fw_data = devm_kmemdup(iface->dchid->dev, fw->data + hdr->hdr_length, + hdr->data_length, GFP_KERNEL); + if (!fw_data) { + ret = -ENOMEM; + goto done; + } + + if (hdr->iface_offset) + fw_data[hdr->iface_offset] = iface->index; + + *firmware = fw_data; + *size = hdr->data_length; + +done: + release_firmware(fw); + return ret; +} + +static int dchid_request_gpio(struct dchid_iface *iface) +{ + char prop_name[MAX_GPIO_NAME + 16]; + + if (iface->gpio) + return 0; + + dev_info(iface->dchid->dev, "Requesting GPIO %s#%d: %s\n", + iface->name, iface->gpio_id, iface->gpio_name); + + snprintf(prop_name, sizeof(prop_name), "apple,%s", iface->gpio_name); + + iface->gpio = devm_gpiod_get_index(iface->dchid->dev, prop_name, 0, GPIOD_OUT_LOW); + + if (IS_ERR_OR_NULL(iface->gpio)) { + dev_err(iface->dchid->dev, "Failed to request GPIO %s-gpios\n", prop_name); + iface->gpio = NULL; + return -1; + } + + return 0; +} + +static int dchid_start_interface(struct dchid_iface *iface) +{ + void *fw; + size_t size; + int ret; + + if (iface->starting) { + dev_warn(iface->dchid->dev, "Interface %s is already starting", iface->name); + return -EINPROGRESS; + } + + dev_info(iface->dchid->dev, "Starting interface %s\n", iface->name); + + iface->starting = true; + + /* Look to see if we need firmware */ + ret = dchid_get_firmware(iface, &fw, &size); + if (ret < 0) + goto err; + + /* If we need a GPIO, make sure we have it. */ + if (iface->gpio_id) { + ret = dchid_request_gpio(iface); + if (ret < 0) + goto err; + } + + /* Only multi-touch has firmware */ + if (fw && size) { + + /* Send firmware to the device */ + dev_info(iface->dchid->dev, "Sending firmware for %s\n", iface->name); + ret = dchid_send_firmware(iface, fw, size); + if (ret < 0) { + dev_err(iface->dchid->dev, "Failed to send %s firmwareS", iface->name); + goto err; + } + + /* After loading firmware, multi-touch needs a reset */ + dev_info(iface->dchid->dev, "Resetting %s\n", iface->name); + dchid_reset_interface(iface, 0); + dchid_reset_interface(iface, 2); + } + + return 0; + +err: + iface->starting = false; + return ret; +} + +static int dchid_start(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + if (iface->keyboard_layout_id) { + int ret = device_create_file(&hdev->dev, &dev_attr_apple_layout_id); + if (ret) { + dev_warn(iface->dchid->dev, "Failed to create apple_layout_id: %d", ret); + iface->keyboard_layout_id = 0; + } + } + + return 0; +}; + +static void dchid_stop(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + if (iface->keyboard_layout_id) + device_remove_file(&hdev->dev, &dev_attr_apple_layout_id); +} + +static int dchid_open(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + int ret; + + if (!completion_done(&iface->ready)) { + ret = dchid_start_interface(iface); + if (ret < 0) + return ret; + + if (!wait_for_completion_timeout(&iface->ready, msecs_to_jiffies(START_TIMEOUT_MS))) { + dev_err(iface->dchid->dev, "iface %s start timed out\n", iface->name); + return -ETIMEDOUT; + } + } + + iface->open = true; + return 0; +} + +static void dchid_close(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + iface->open = false; +} + +static int dchid_parse(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + return hid_parse_report(hdev, iface->hid_desc, iface->hid_desc_len); +} + +/* Note: buf excludes report number! For ease of fetching strings/etc. */ +static int dchid_get_report_cmd(struct dchid_iface *iface, u8 reportnum, void *buf, size_t len) +{ + int ret = dchid_cmd(iface, HID_FEATURE_REPORT, REQ_GET_REPORT, &reportnum, 1, buf, len); + + return ret <= 0 ? ret : ret - 1; +} + +/* Note: buf includes report number! */ +static int dchid_set_report(struct dchid_iface *iface, void *buf, size_t len) +{ + return dchid_cmd(iface, HID_OUTPUT_REPORT, REQ_SET_REPORT, buf, len, NULL, 0); +} + +static int dchid_raw_request(struct hid_device *hdev, + unsigned char reportnum, __u8 *buf, size_t len, + unsigned char rtype, int reqtype) +{ + struct dchid_iface *iface = hdev->driver_data; + + switch (reqtype) { + case HID_REQ_GET_REPORT: + buf[0] = reportnum; + return dchid_cmd(iface, rtype, REQ_GET_REPORT, &reportnum, 1, buf + 1, len - 1); + case HID_REQ_SET_REPORT: + return dchid_set_report(iface, buf, len); + default: + return -EIO; + } + + return 0; +} + +static struct hid_ll_driver dchid_ll = { + .start = &dchid_start, + .stop = &dchid_stop, + .open = &dchid_open, + .close = &dchid_close, + .parse = &dchid_parse, + .raw_request = &dchid_raw_request, +}; + +static void dchid_create_interface_work(struct work_struct *ws) +{ + struct dchid_iface *iface = container_of(ws, struct dchid_iface, create_work); + struct dockchannel_hid *dchid = iface->dchid; + struct hid_device *hid; + int ret; + + if (iface->hid) { + dev_warn(dchid->dev, "Interface %s already created!\n", + iface->name); + return; + } + + dev_info(dchid->dev, "New interface %s\n", iface->name); + + /* Start the interface. This is not the entire init process, as firmware is loaded later on device open. */ + ret = dchid_enable_interface(iface); + if (ret < 0) { + dev_warn(dchid->dev, "Failed to enable %s: %d\n", iface->name, ret); + return; + } + + iface->deferred = false; + + hid = hid_allocate_device(); + if (IS_ERR(hid)) + return; + + snprintf(hid->name, sizeof(hid->name), "Apple MTP %s", iface->name); + snprintf(hid->phys, sizeof(hid->phys), "%s.%d (%s)", + dev_name(dchid->dev), iface->index, iface->name); + strscpy(hid->uniq, dchid->serial, sizeof(hid->uniq)); + + hid->ll_driver = &dchid_ll; + hid->bus = BUS_HOST; + hid->vendor = dchid->device_id.vendor_id; + hid->product = dchid->device_id.product_id; + hid->version = dchid->device_id.version_number; + hid->type = HID_TYPE_OTHER; + if (!strcmp(iface->name, "multi-touch")) { + hid->type = HID_TYPE_SPI_MOUSE; + } else if (!strcmp(iface->name, "keyboard")) { + u32 country_code = 0; + + hid->type = HID_TYPE_SPI_KEYBOARD; + + /* + * We have to get the country code from the device tree, since the + * device provides no reliable way to get this info. + */ + if (!of_property_read_u32(iface->of_node, "hid-country-code", &country_code)) + hid->country = country_code; + + of_property_read_u32(iface->of_node, "apple,keyboard-layout-id", + &iface->keyboard_layout_id); + } + + hid->dev.parent = iface->dchid->dev; + hid->driver_data = iface; + + iface->hid = hid; + + ret = hid_add_device(hid); + if (ret < 0) { + iface->hid = NULL; + hid_destroy_device(hid); + dev_warn(iface->dchid->dev, "Failed to register hid device %s", iface->name); + } +} + +static int dchid_create_interface(struct dchid_iface *iface) +{ + if (iface->creating) + return -EBUSY; + + iface->creating = true; + INIT_WORK(&iface->create_work, dchid_create_interface_work); + return queue_work(iface->dchid->new_iface_wq, &iface->create_work); +} + +static void dchid_handle_descriptor(struct dchid_iface *iface, void *hid_desc, size_t desc_len) +{ + if (iface->hid) { + dev_warn(iface->dchid->dev, "Tried to initialize already started interface %s!\n", + iface->name); + return; + } + + iface->hid_desc = devm_kmemdup(iface->dchid->dev, hid_desc, desc_len, GFP_KERNEL); + if (!iface->hid_desc) + return; + + iface->hid_desc_len = desc_len; +} + +static void dchid_handle_ready(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_iface *iface; + u8 *pkt = data; + u8 index; + int i, ret; + + if (length < 2) { + dev_err(dchid->dev, "Bad length for ready message: %zu\n", length); + return; + } + + index = pkt[1]; + + if (index >= MAX_INTERFACES) { + dev_err(dchid->dev, "Got ready notification for bad iface %d\n", index); + return; + } + + iface = dchid->ifaces[index]; + if (!iface) { + dev_err(dchid->dev, "Got ready notification for unknown iface %d\n", index); + return; + } + + dev_info(dchid->dev, "Interface %s is now ready\n", iface->name); + complete_all(&iface->ready); + + /* When STM is ready, grab global device info */ + if (!strcmp(iface->name, "stm")) { + ret = dchid_get_report_cmd(iface, STM_REPORT_ID, &dchid->device_id, + sizeof(dchid->device_id)); + if (ret < sizeof(dchid->device_id)) { + dev_warn(iface->dchid->dev, "Failed to get device ID from STM!\n"); + /* Fake it and keep going. Things might still work... */ + memset(&dchid->device_id, 0, sizeof(dchid->device_id)); + dchid->device_id.vendor_id = HOST_VENDOR_ID_APPLE; + } + ret = dchid_get_report_cmd(iface, STM_REPORT_SERIAL, dchid->serial, + sizeof(dchid->serial) - 1); + if (ret < 0) { + dev_warn(iface->dchid->dev, "Failed to get serial from STM!\n"); + dchid->serial[0] = 0; + } + + dchid->id_ready = true; + for (i = 0; i < MAX_INTERFACES; i++) { + if (!dchid->ifaces[i] || !dchid->ifaces[i]->deferred) + continue; + dchid_create_interface(dchid->ifaces[i]); + } + } +} + +static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_init_hdr *hdr = data; + struct dchid_iface *iface; + struct dchid_init_block_hdr *blk; + + if (length < sizeof(*hdr)) + return; + + iface = dchid_get_interface(dchid, hdr->iface, hdr->name); + if (!iface) + return; + + data += sizeof(*hdr); + length -= sizeof(*hdr); + + while (length >= sizeof(*blk)) { + blk = data; + data += sizeof(*blk); + length -= sizeof(*blk); + + if (blk->length > length) + break; + + switch (blk->type) { + case INIT_HID_DESCRIPTOR: + dchid_handle_descriptor(iface, data, blk->length); + break; + + case INIT_GPIO_REQUEST: { + struct dchid_gpio_request *req = data; + + if (sizeof(*req) > length) + break; + + if (iface->gpio_id) { + dev_err(dchid->dev, + "Cannot request more than one GPIO per interface!\n"); + break; + } + + strscpy(iface->gpio_name, req->name, MAX_GPIO_NAME); + iface->gpio_id = req->id; + break; + } + + case INIT_TERMINATOR: + break; + + case INIT_PRODUCT_NAME: { + char *product = data; + + if (product[blk->length - 1] != 0) { + dev_warn(dchid->dev, "Unterminated product name for %s\n", + iface->name); + } else { + dev_info(dchid->dev, "Product name for %s: %s\n", + iface->name, product); + } + break; + } + + default: + dev_warn(dchid->dev, "Unknown init packet %d for %s\n", + blk->type, iface->name); + break; + } + + data += blk->length; + length -= blk->length; + + if (blk->type == INIT_TERMINATOR) + break; + } + + if (hdr->more_packets) + return; + + /* We need to enable STM first, since it'll give us the device IDs */ + if (iface->dchid->id_ready || !strcmp(iface->name, "stm")) { + dchid_create_interface(iface); + } else { + iface->deferred = true; + } +} + +static void dchid_handle_gpio(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_gpio_cmd *cmd = data; + struct dchid_iface *iface; + u32 retcode = 0xe000f00d; /* Give it a random Apple-style error code */ + struct dchid_gpio_ack *ack; + + if (length < sizeof(*cmd)) + return; + + if (cmd->iface >= MAX_INTERFACES || !(iface = dchid->ifaces[cmd->iface])) { + dev_err(dchid->dev, "Got GPIO command for bad inteface %d\n", cmd->iface); + goto err; + } + + if (dchid_request_gpio(iface) < 0) + goto err; + + if (!iface->gpio || cmd->gpio != iface->gpio_id) { + dev_err(dchid->dev, "Got GPIO command for bad GPIO %s#%d\n", + iface->name, cmd->gpio); + goto err; + } + + dev_info(dchid->dev, "GPIO command: %s#%d: %d\n", iface->name, cmd->gpio, cmd->cmd); + + switch (cmd->cmd) { + case 3: + /* Pulse. */ + gpiod_set_value_cansleep(iface->gpio, 1); + msleep(10); /* Random guess... */ + gpiod_set_value_cansleep(iface->gpio, 0); + retcode = 0; + break; + default: + dev_err(dchid->dev, "Unknown GPIO command %d\n", cmd->cmd ); + break; + } + +err: + /* Ack it */ + ack = kzalloc(sizeof(*ack) + length, GFP_KERNEL); + if (!ack) + return; + + ack->type = CMD_ACK_GPIO_CMD; + ack->retcode = retcode; + memcpy(ack->cmd, data, length); + + if (dchid_comm_cmd(dchid, ack, sizeof(*ack) + length) < 0) + dev_err(dchid->dev, "Failed to ACK GPIO command\n"); + + kfree(ack); +} + +static void dchid_handle_event(struct dockchannel_hid *dchid, void *data, size_t length) +{ + u8 *p = data; + switch (*p) { + case EVENT_INIT: + dchid_handle_init(dchid, data, length); + break; + case EVENT_READY: + dchid_handle_ready(dchid, data, length); + break; + case EVENT_GPIO_CMD: + dchid_handle_gpio(dchid, data, length); + break; + } +} + +static void dchid_handle_report(struct dchid_iface *iface, void *data, size_t length) +{ + struct dockchannel_hid *dchid = iface->dchid; + + if (!iface->hid) { + dev_warn(dchid->dev, "Report received but %s is not initialized!\n", iface->name); + return; + } + + if (!iface->open) + return; + + hid_input_report(iface->hid, HID_INPUT_REPORT, data, length, 1); +} + +static void dchid_packet_work(struct work_struct *ws) +{ + struct dchid_work *work = container_of(ws, struct dchid_work, work); + struct dchid_subhdr *shdr = (void *)work->data; + struct dockchannel_hid *dchid = work->iface->dchid; + int type = FIELD_GET(FLAGS_GROUP, shdr->flags); + u8 *payload = work->data + sizeof(*shdr); + + if (shdr->length + sizeof(*shdr) > work->hdr.length) { + dev_err(dchid->dev, "Bad sub header length (%hu > %zu)\n", + shdr->length, work->hdr.length - sizeof(*shdr)); + return; + } + + switch (type) { + case HID_INPUT_REPORT: + if (work->hdr.iface == IFACE_COMM) + dchid_handle_event(dchid, payload, shdr->length); + else + dchid_handle_report(work->iface, payload, shdr->length); + break; + default: + dev_err(dchid->dev, "Received unknown packet type %d\n", type); + break; + } + + kfree(work); +} + +static void dchid_handle_ack(struct dchid_iface *iface, struct dchid_hdr *hdr, void *data) +{ + struct dchid_subhdr *shdr = (void *)data; + u8 *payload = data + sizeof(*shdr); + + if (shdr->length + sizeof(*shdr) > hdr->length) { + dev_err(iface->dchid->dev, "Bad sub header length (%hu > %zu)\n", + shdr->length, hdr->length - sizeof(*shdr)); + return; + } + if (shdr->flags != iface->out_flags) { + dev_err(iface->dchid->dev, + "Received unexpected flags 0x%x on ACK channel (expFected 0x%x)\n", + shdr->flags, iface->out_flags); + return; + } + + if (shdr->length < 1) { + dev_err(iface->dchid->dev, "Received length 0 output report ack\n"); + return; + } + if (iface->tx_seq != hdr->seq) { + dev_err(iface->dchid->dev, "Received ACK with bad seq (expected %d, got %d)\n", + iface->tx_seq, hdr->seq); + return; + } + if (iface->out_report != payload[0]) { + dev_err(iface->dchid->dev, "Received ACK with bad report (expected %d, got %d\n", + iface->out_report, payload[0]); + return; + } + + if (iface->resp_buf && iface->resp_size) + memcpy(iface->resp_buf, payload + 1, min((size_t)shdr->length - 1, iface->resp_size)); + + iface->resp_size = shdr->length; + iface->out_report = -1; + iface->retcode = shdr->retcode; + complete(&iface->out_complete); +} + +static void dchid_handle_packet(void *cookie, size_t avail) +{ + struct dockchannel_hid *dchid = cookie; + struct dchid_hdr hdr; + struct dchid_work *work; + struct dchid_iface *iface; + u32 checksum; + + if (dockchannel_recv(dchid->dc, &hdr, sizeof(hdr)) != sizeof(hdr)) { + dev_err(dchid->dev, "Read failed (header)\n"); + return; + } + + if (hdr.hdr_len != sizeof(hdr)) { + dev_err(dchid->dev, "Bad header length %d\n", hdr.hdr_len); + goto done; + } + + if (dockchannel_recv(dchid->dc, dchid->pkt_buf, hdr.length + 4) != (hdr.length + 4)) { + dev_err(dchid->dev, "Read failed (body)\n"); + goto done; + } + + checksum = dchid_checksum(&hdr, sizeof(hdr)); + checksum += dchid_checksum(dchid->pkt_buf, hdr.length + 4); + + if (checksum != 0xffffffff) { + dev_err(dchid->dev, "Checksum mismatch (iface %d): 0x%08x != 0xffffffff\n", + hdr.iface, checksum); + goto done; + } + + + if (hdr.iface >= MAX_INTERFACES) { + dev_err(dchid->dev, "Bad iface %d\n", hdr.iface); + } + + iface = dchid->ifaces[hdr.iface]; + + if (!iface) { + dev_err(dchid->dev, "Received packet for uninitialized iface %d\n", hdr.iface); + goto done; + } + + switch (hdr.channel) { + case DCHID_CHANNEL_CMD: + dchid_handle_ack(iface, &hdr, dchid->pkt_buf); + goto done; + case DCHID_CHANNEL_REPORT: + break; + default: + dev_warn(dchid->dev, "Unknown channel 0x%x, treating as report...\n", + hdr.channel); + break; + } + + work = kzalloc(sizeof(*work) + hdr.length, GFP_KERNEL); + if (!work) + return; + + work->hdr = hdr; + work->iface = iface; + memcpy(work->data, dchid->pkt_buf, hdr.length); + INIT_WORK(&work->work, dchid_packet_work); + + queue_work(iface->wq, &work->work); + +done: + dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); +} + +static int dockchannel_hid_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel_hid *dchid; + struct device_node *child, *helper; + struct platform_device *helper_pdev; + struct property *prop; + int ret; + + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) + return ret; + + dchid = devm_kzalloc(dev, sizeof(*dchid), GFP_KERNEL); + if (!dchid) { + return -ENOMEM; + } + + dchid->dev = dev; + + /* + * First make sure all the GPIOs are available, in cased we need to defer. + * This is necessary because MTP will request them by name later, and by then + * it's too late to defer the probe. + */ + + for_each_child_of_node(dev->of_node, child) { + for_each_property_of_node(child, prop) { + size_t len = strlen(prop->name); + struct gpio_desc *gpio; + + if (len < 12 || strncmp("apple,", prop->name, 6) || + strcmp("-gpios", prop->name + len - 6)) + continue; + + gpio = fwnode_gpiod_get_index(&child->fwnode, prop->name, 0, GPIOD_ASIS, + prop->name); + if (IS_ERR_OR_NULL(gpio)) { + if (PTR_ERR(gpio) == -EPROBE_DEFER) { + of_node_put(child); + return -EPROBE_DEFER; + } + } else { + gpiod_put(gpio); + } + } + } + + /* + * Make sure we also have the MTP coprocessor available, and + * defer probe if the helper hasn't probed yet. + */ + helper = of_parse_phandle(dev->of_node, "apple,helper-cpu", 0); + if (!helper) { + dev_err(dev, "Missing apple,helper-cpu property"); + return -EINVAL; + } + + helper_pdev = of_find_device_by_node(helper); + of_node_put(helper); + if (!helper_pdev) { + dev_err(dev, "Failed to find helper device"); + return -EINVAL; + } + + dchid->helper_link = device_link_add(dev, &helper_pdev->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + put_device(&helper_pdev->dev); + if (!dchid->helper_link) { + dev_err(dev, "Failed to link to helper device"); + return -EINVAL; + } + + if (dchid->helper_link->supplier->links.status != DL_DEV_DRIVER_BOUND) + return -EPROBE_DEFER; + + /* Now it is safe to begin initializing */ + dchid->dc = dockchannel_init(pdev); + if (IS_ERR_OR_NULL(dchid->dc)) { + return PTR_ERR(dchid->dc); + } + dchid->new_iface_wq = alloc_workqueue("dchid-new", WQ_MEM_RECLAIM, 0); + if (!dchid->new_iface_wq) + return -ENOMEM; + + dchid->comm = dchid_get_interface(dchid, IFACE_COMM, "comm"); + if (!dchid->comm) { + dev_err(dchid->dev, "Failed to initialize comm interface"); + return -EIO; + } + + dev_info(dchid->dev, "Initialized, awaiting packets\n"); + dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); + + return 0; +} + +static void dockchannel_hid_remove(struct platform_device *pdev) +{ + BUG_ON(1); +} + +static const struct of_device_id dockchannel_hid_of_match[] = { + { .compatible = "apple,dockchannel-hid" }, + {}, +}; +MODULE_DEVICE_TABLE(of, dockchannel_hid_of_match); +MODULE_FIRMWARE("apple/tpmtfw-*.bin"); + +static struct platform_driver dockchannel_hid_driver = { + .driver = { + .name = "dockchannel-hid", + .of_match_table = dockchannel_hid_of_match, + }, + .probe = dockchannel_hid_probe, + .remove = dockchannel_hid_remove, +}; +module_platform_driver(dockchannel_hid_driver); + +MODULE_DESCRIPTION("Apple DockChannel HID transport driver"); +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); diff -ruN --no-dereference -x .git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c --- a/drivers/hid/hid-apple.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/hid/hid-apple.c 2026-10-13 14:33:30.000000000 +0200 @@ -30,7 +30,7 @@ #include "hid-ids.h" #define APPLE_RDESC_JIS BIT(0) -/* BIT(1) reserved, was: APPLE_IGNORE_MOUSE */ +#define APPLE_IGNORE_MOUSE BIT(1) #define APPLE_HAS_FN BIT(2) /* BIT(3) reserved, was: APPLE_HIDDEV */ #define APPLE_ISO_TILDE_QUIRK BIT(4) @@ -54,10 +54,16 @@ #define APPLE_MAGIC_REPORT_ID_POWER 3 #define APPLE_MAGIC_REPORT_ID_BRIGHTNESS 1 +// DO NOT UPSTREAM: +// temporary Fn key mode until xkeyboard-config has keyboard layouts with media +// key mappings. At that point auto mode can drop function key mappings and this +// mode can be dropped. +#define FKEYS_IGNORE 5 + static unsigned int fnmode = 3; module_param(fnmode, uint, 0644); MODULE_PARM_DESC(fnmode, "Mode of fn key on Apple keyboards (0 = disabled, " - "1 = fkeyslast, 2 = fkeysfirst, [3] = auto, 4 = fkeysdisabled)"); + "1 = fkeyslast, 2 = fkeysfirst, [3] = auto, 4 = fkeysdisabled, 5 = fkeysignore))"); static int iso_layout = -1; module_param(iso_layout, int, 0644); @@ -280,6 +286,16 @@ { } }; +static const struct apple_key_translation apple_fn_keys_minimal[] = { + { KEY_BACKSPACE, KEY_DELETE }, + { KEY_ENTER, KEY_INSERT }, + { KEY_UP, KEY_PAGEUP }, + { KEY_DOWN, KEY_PAGEDOWN }, + { KEY_LEFT, KEY_HOME }, + { KEY_RIGHT, KEY_END }, + { } +}; + static const struct apple_key_translation powerbook_fn_keys[] = { { KEY_BACKSPACE, KEY_DELETE }, { KEY_F1, KEY_BRIGHTNESSDOWN, APPLE_FLAG_FKEY }, @@ -440,6 +456,8 @@ real_fnmode = 2; else real_fnmode = 1; + } else if (fnmode == FKEYS_IGNORE) { + real_fnmode = 2; } else { real_fnmode = fnmode; } @@ -480,6 +498,23 @@ asc->fn_on = !!value; if (real_fnmode) { + switch (hid->bus) { + case BUS_HOST: + case BUS_SPI: + switch (hid->product) { + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020: + case HOST_DEVICE_ID_APPLE_MACBOOK_PRO13_2022: + table = macbookpro_dedicated_esc_fn_keys; + break; + default: + if (fnmode == FKEYS_IGNORE) + table = apple_fn_keys_minimal; + else + table = magic_keyboard_2021_and_2024_fn_keys; + break; + } + break; + default: switch (hid->product) { case USB_DEVICE_ID_APPLE_ALU_WIRELESS_ANSI: case USB_DEVICE_ID_APPLE_ALU_WIRELESS_ISO: @@ -528,6 +563,7 @@ else table = apple_fn_keys; } + } trans = apple_find_translation(table, code); @@ -711,6 +747,7 @@ /* Enable all needed keys */ apple_setup_key_translation(input, apple_fn_keys); + apple_setup_key_translation(input, apple_fn_keys_minimal); apple_setup_key_translation(input, powerbook_fn_keys); apple_setup_key_translation(input, powerbook_numlock_keys); apple_setup_key_translation(input, apple_iso_keyboard); @@ -955,6 +992,18 @@ struct apple_sc *asc; int ret; + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && id->vendor == SPI_VENDOR_ID_APPLE && + hdev->type != HID_TYPE_SPI_KEYBOARD) + return -ENODEV; + + if (quirks & APPLE_IGNORE_MOUSE && hdev->type == HID_TYPE_USBMOUSE) + return -ENODEV; + + // key remapping will happen in xkeyboard-config so ignore + // APPLE_ISO_TILDE_QUIRK + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && fnmode == FKEYS_IGNORE) + quirks &= ~APPLE_ISO_TILDE_QUIRK; + asc = devm_kzalloc(&hdev->dev, sizeof(*asc), GFP_KERNEL); if (asc == NULL) { hid_err(hdev, "can't alloc apple descriptor\n"); @@ -1180,27 +1229,31 @@ { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_JIS), .driver_data = APPLE_HAS_FN | APPLE_RDESC_JIS }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K), - .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK | + APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132), .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK | - APPLE_DISABLE_FKEYS }, + APPLE_DISABLE_FKEYS | APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680), .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK | - APPLE_DISABLE_FKEYS }, + APPLE_DISABLE_FKEYS | APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT), .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK | - APPLE_DISABLE_FKEYS }, + APPLE_DISABLE_FKEYS | APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213), .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK | - APPLE_DISABLE_FKEYS }, + APPLE_DISABLE_FKEYS | APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K), - .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_DISABLE_FKEYS }, + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_DISABLE_FKEYS | + APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223), - .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_DISABLE_FKEYS }, + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_DISABLE_FKEYS | + APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K), - .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F), - .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_DISABLE_FKEYS }, + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_DISABLE_FKEYS | + APPLE_IGNORE_MOUSE }, { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2009_ANSI), .driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN }, { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2009_ISO), @@ -1212,6 +1265,8 @@ .driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER1_TP_ONLY), .driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN }, + { HID_SPI_DEVICE(SPI_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, // TODO: remove APPLE_ISO_TILDE_QUIRK { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2021), .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_RDESC_BATTERY }, { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2021), @@ -1224,6 +1279,8 @@ .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_RDESC_BATTERY }, { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_NUMPAD_2021), .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, // TODO: remove APPLE_ISO_TILDE_QUIRK { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2024), .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_RDESC_BATTERY }, { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2024), diff -ruN --no-dereference -x .git a/drivers/hid/hid-appletb-kbd.c b/drivers/hid/hid-appletb-kbd.c --- a/drivers/hid/hid-appletb-kbd.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/hid/hid-appletb-kbd.c 2026-10-13 14:33:30.000000000 +0200 @@ -56,6 +56,12 @@ module_param_named(idle_timeout, appletb_tb_idle_timeout, int, 0644); MODULE_PARM_DESC(idle_timeout, "Idle timeout in sec"); +static int appletb_tb_double_press_switch_layers; +module_param_named(double_press_switch_time, + appletb_tb_double_press_switch_layers, int, 0644); +MODULE_PARM_DESC(double_press_switch_time, "Time in ms within which if fn key is double " + "pressed will switch layers"); + struct appletb_kbd { struct hid_field *mode_field; struct input_handler inp_handler; @@ -68,6 +74,7 @@ bool has_turned_off; u8 saved_mode; u8 current_mode; + unsigned long last_fn_press; }; static const struct key_entry appletb_kbd_keymap[] = { @@ -243,6 +250,18 @@ return kbd->current_mode == APPLETB_KBD_MODE_OFF; } +static u8 appletb_switch_mode(u8 mode) +{ + switch (mode) { + case APPLETB_KBD_MODE_SPCL: + return APPLETB_KBD_MODE_FN; + case APPLETB_KBD_MODE_FN: + return APPLETB_KBD_MODE_SPCL; + default: + return mode; + } +} + static void appletb_kbd_inp_event(struct input_handle *handle, unsigned int type, unsigned int code, int value) { @@ -250,15 +269,42 @@ reset_inactivity_timer(kbd); - if (type == EV_KEY && code == KEY_FN && appletb_tb_fn_toggle && - (kbd->current_mode == APPLETB_KBD_MODE_SPCL || - kbd->current_mode == APPLETB_KBD_MODE_FN)) { + if (type == EV_KEY && code == KEY_FN && + (kbd->current_mode == APPLETB_KBD_MODE_SPCL || + kbd->current_mode == APPLETB_KBD_MODE_FN)) { + if (value == 1) { - kbd->saved_mode = kbd->current_mode; - appletb_kbd_set_mode(kbd, kbd->current_mode == APPLETB_KBD_MODE_SPCL - ? APPLETB_KBD_MODE_FN : APPLETB_KBD_MODE_SPCL); + if (appletb_tb_double_press_switch_layers) { + unsigned long now = jiffies; + + if (time_before(now, kbd->last_fn_press + + msecs_to_jiffies(appletb_tb_double_press_switch_layers))) { + + appletb_tb_def_mode = + appletb_switch_mode( + appletb_tb_def_mode); + + appletb_kbd_set_mode(kbd, + appletb_tb_def_mode); + + kbd->saved_mode = appletb_tb_def_mode; + kbd->last_fn_press = 0; + + return; + } + + kbd->last_fn_press = now; + } + if (appletb_tb_fn_toggle) { + kbd->saved_mode = kbd->current_mode; + + appletb_kbd_set_mode(kbd, + appletb_switch_mode(kbd->current_mode)); + } + } else if (value == 0) { - if (kbd->saved_mode != kbd->current_mode) + if (appletb_tb_fn_toggle && + kbd->saved_mode != kbd->current_mode) appletb_kbd_set_mode(kbd, kbd->saved_mode); } } diff -ruN --no-dereference -x .git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c --- a/drivers/hid/hid-asus.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/hid/hid-asus.c 2026-10-13 14:33:30.000000000 +0200 @@ -1480,12 +1480,14 @@ return ret; } - for (int r = 0; r < ARRAY_SIZE(asus_report_id_init); r++) { - if (asus_has_report_id(hdev, asus_report_id_init[r])) { - ret = asus_kbd_init(hdev, asus_report_id_init[r]); - if (ret < 0) - hid_warn(hdev, "Failed to initialize 0x%x: %d.\n", - asus_report_id_init[r], ret); + if (!drvdata->tp) { + for (int r = 0; r < ARRAY_SIZE(asus_report_id_init); r++) { + if (asus_has_report_id(hdev, asus_report_id_init[r])) { + ret = asus_kbd_init(hdev, asus_report_id_init[r]); + if (ret < 0) + hid_warn(hdev, "Failed to initialize 0x%x: %d.\n", + asus_report_id_init[r], ret); + } } } diff -ruN --no-dereference -x .git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c --- a/drivers/hid/hid-core.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/hid/hid-core.c 2026-10-13 14:33:30.000000000 +0200 @@ -477,7 +477,10 @@ case HID_GLOBAL_ITEM_TAG_REPORT_SIZE: parser->global.report_size = item_udata(item); - if (parser->global.report_size > 256) { + /* Arbitrary maximum. Some Apple devices have 16384 here. + * This * HID_MAX_USAGES must fit in a signed integer. + */ + if (parser->global.report_size > 16384) { hid_err(parser->device, "invalid report_size %d\n", parser->global.report_size); return -1; @@ -2380,6 +2383,12 @@ case BUS_SDW: bus = "SOUNDWIRE"; break; + case BUS_SPI: + bus = "SPI"; + break; + case BUS_HOST: + bus = "HOST"; + break; case BUS_VIRTUAL: bus = "VIRTUAL"; break; diff -ruN --no-dereference -x .git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h --- a/drivers/hid/hid-ids.h 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/hid/hid-ids.h 2026-10-13 14:33:30.000000000 +0200 @@ -96,6 +96,8 @@ #define USB_VENDOR_ID_APPLE 0x05ac #define BT_VENDOR_ID_APPLE 0x004c +#define SPI_VENDOR_ID_APPLE 0x05ac +#define HOST_VENDOR_ID_APPLE 0x05ac #define USB_DEVICE_ID_APPLE_MIGHTYMOUSE 0x0304 #define USB_DEVICE_ID_APPLE_MAGICMOUSE 0x030d #define USB_DEVICE_ID_APPLE_MAGICMOUSE2 0x0269 @@ -200,6 +202,14 @@ #define USB_DEVICE_ID_APPLE_IRCONTROL5 0x8243 #define USB_DEVICE_ID_APPLE_TOUCHBAR_BACKLIGHT 0x8102 #define USB_DEVICE_ID_APPLE_TOUCHBAR_DISPLAY 0x8302 +#define SPI_DEVICE_ID_APPLE_MACBOOK_AIR_2020 0x0281 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020 0x0341 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO14_2021 0x0342 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO16_2021 0x0343 +#define HOST_DEVICE_ID_APPLE_MACBOOK_AIR13_2022 0x0351 +#define HOST_DEVICE_ID_APPLE_MACBOOK_PRO14_2023 0x0352 +#define HOST_DEVICE_ID_APPLE_MACBOOK_PRO16_2023 0x0353 +#define HOST_DEVICE_ID_APPLE_MACBOOK_PRO13_2022 0x0354 #define USB_VENDOR_ID_ASETEK 0x2433 #define USB_DEVICE_ID_ASETEK_INVICTA 0xf300 diff -ruN --no-dereference -x .git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c --- a/drivers/hid/hid-magicmouse.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/hid/hid-magicmouse.c 2026-10-13 14:33:30.000000000 +0200 @@ -60,8 +60,14 @@ #define MOUSE_REPORT_ID 0x29 #define MOUSE2_REPORT_ID 0x12 #define DOUBLE_REPORT_ID 0xf7 +#define SPI_REPORT_ID 0x02 +#define SPI_RESET_REPORT_ID 0x60 +#define MTP_REPORT_ID 0x75 +#define SENSOR_DIMENSIONS_REPORT_ID 0xd9 #define USB_BATTERY_TIMEOUT_SEC 60 +#define MAX_CONTACTS 16 + /* These definitions are not precise, but they're close enough. (Bits * 0x03 seem to indicate the aspect ratio of the touch, bits 0x70 seem * to be some kind of bit mask -- 0x20 may be a near-field reading, @@ -112,30 +118,156 @@ #define TRACKPAD2_RES_Y \ ((TRACKPAD2_MAX_Y - TRACKPAD2_MIN_Y) / (TRACKPAD2_DIMENSION_Y / 100)) +#define J140K_TP_DIMENSION_X (float)12100 +#define J140K_TP_MIN_X -5318 +#define J140K_TP_MAX_X 5787 +#define J140K_TP_RES_X \ + ((J140K_TP_MAX_X - J140K_TP_MIN_X) / (J140K_TP_DIMENSION_X / 100)) +#define J140K_TP_DIMENSION_Y (float)8200 +#define J140K_TP_MIN_Y -157 +#define J140K_TP_MAX_Y 7102 +#define J140K_TP_RES_Y \ + ((J140K_TP_MAX_Y - J140K_TP_MIN_Y) / (J140K_TP_DIMENSION_Y / 100)) + +#define J132_TP_DIMENSION_X (float)13500 +#define J132_TP_MIN_X -6243 +#define J132_TP_MAX_X 6749 +#define J132_TP_RES_X \ + ((J132_TP_MAX_X - J132_TP_MIN_X) / (J132_TP_DIMENSION_X / 100)) +#define J132_TP_DIMENSION_Y (float)8400 +#define J132_TP_MIN_Y -170 +#define J132_TP_MAX_Y 7685 +#define J132_TP_RES_Y \ + ((J132_TP_MAX_Y - J132_TP_MIN_Y) / (J132_TP_DIMENSION_Y / 100)) + +#define J680_TP_DIMENSION_X (float)16000 +#define J680_TP_MIN_X -7456 +#define J680_TP_MAX_X 7976 +#define J680_TP_RES_X \ + ((J680_TP_MAX_X - J680_TP_MIN_X) / (J680_TP_DIMENSION_X / 100)) +#define J680_TP_DIMENSION_Y (float)10000 +#define J680_TP_MIN_Y -163 +#define J680_TP_MAX_Y 9283 +#define J680_TP_RES_Y \ + ((J680_TP_MAX_Y - J680_TP_MIN_Y) / (J680_TP_DIMENSION_Y / 100)) + +#define J680_ALT_TP_DIMENSION_X (float)16000 +#define J680_ALT_TP_MIN_X -7456 +#define J680_ALT_TP_MAX_X 7976 +#define J680_ALT_TP_RES_X \ + ((J680_ALT_TP_MAX_X - J680_ALT_TP_MIN_X) / (J680_ALT_TP_DIMENSION_X / 100)) +#define J680_ALT_TP_DIMENSION_Y (float)10000 +#define J680_ALT_TP_MIN_Y -163 +#define J680_ALT_TP_MAX_Y 9283 +#define J680_ALT_TP_RES_Y \ + ((J680_ALT_TP_MAX_Y - J680_ALT_TP_MIN_Y) / (J680_ALT_TP_DIMENSION_Y / 100)) + +#define J213_TP_DIMENSION_X (float)13500 +#define J213_TP_MIN_X -6243 +#define J213_TP_MAX_X 6749 +#define J213_TP_RES_X \ + ((J213_TP_MAX_X - J213_TP_MIN_X) / (J213_TP_DIMENSION_X / 100)) +#define J213_TP_DIMENSION_Y (float)8400 +#define J213_TP_MIN_Y -170 +#define J213_TP_MAX_Y 7685 +#define J213_TP_RES_Y \ + ((J213_TP_MAX_Y - J213_TP_MIN_Y) / (J213_TP_DIMENSION_Y / 100)) + +#define J214K_TP_DIMENSION_X (float)13200 +#define J214K_TP_MIN_X -6046 +#define J214K_TP_MAX_X 6536 +#define J214K_TP_RES_X \ + ((J214K_TP_MAX_X - J214K_TP_MIN_X) / (J214K_TP_DIMENSION_X / 100)) +#define J214K_TP_DIMENSION_Y (float)8200 +#define J214K_TP_MIN_Y -164 +#define J214K_TP_MAX_Y 7439 +#define J214K_TP_RES_Y \ + ((J214K_TP_MAX_Y - J214K_TP_MIN_Y) / (J214K_TP_DIMENSION_Y / 100)) + +#define J223_TP_DIMENSION_X (float)13200 +#define J223_TP_MIN_X -6046 +#define J223_TP_MAX_X 6536 +#define J223_TP_RES_X \ + ((J223_TP_MAX_X - J223_TP_MIN_X) / (J223_TP_DIMENSION_X / 100)) +#define J223_TP_DIMENSION_Y (float)8200 +#define J223_TP_MIN_Y -164 +#define J223_TP_MAX_Y 7439 +#define J223_TP_RES_Y \ + ((J223_TP_MAX_Y - J223_TP_MIN_Y) / (J223_TP_DIMENSION_Y / 100)) + +#define J230K_TP_DIMENSION_X (float)12100 +#define J230K_TP_MIN_X -5318 +#define J230K_TP_MAX_X 5787 +#define J230K_TP_RES_X \ + ((J230K_TP_MAX_X - J230K_TP_MIN_X) / (J230K_TP_DIMENSION_X / 100)) +#define J230K_TP_DIMENSION_Y (float)8200 +#define J230K_TP_MIN_Y -157 +#define J230K_TP_MAX_Y 7102 +#define J230K_TP_RES_Y \ + ((J230K_TP_MAX_Y - J230K_TP_MIN_Y) / (J230K_TP_DIMENSION_Y / 100)) + +#define J152F_TP_DIMENSION_X (float)16000 +#define J152F_TP_MIN_X -7456 +#define J152F_TP_MAX_X 7976 +#define J152F_TP_RES_X \ + ((J152F_TP_MAX_X - J152F_TP_MIN_X) / (J152F_TP_DIMENSION_X / 100)) +#define J152F_TP_DIMENSION_Y (float)10000 +#define J152F_TP_MIN_Y -163 +#define J152F_TP_MAX_Y 9283 +#define J152F_TP_RES_Y \ + ((J152F_TP_MAX_Y - J152F_TP_MIN_Y) / (J152F_TP_DIMENSION_Y / 100)) + +/* These are fallback values, since the real values will be queried from the device. */ +#define J314_TP_DIMENSION_X (float)13000 +#define J314_TP_MIN_X -5900 +#define J314_TP_MAX_X 6500 +#define J314_TP_RES_X \ + ((J314_TP_MAX_X - J314_TP_MIN_X) / (J314_TP_DIMENSION_X / 100)) +#define J314_TP_DIMENSION_Y (float)8100 +#define J314_TP_MIN_Y -200 +#define J314_TP_MAX_Y 7400 +#define J314_TP_RES_Y \ + ((J314_TP_MAX_Y - J314_TP_MIN_Y) / (J314_TP_DIMENSION_Y / 100)) + +#define T2_TOUCHPAD_ENTRY(model) \ + { USB_DEVICE_ID_APPLE_WELLSPRINGT2_##model, model##_TP_MIN_X, model##_TP_MIN_Y, \ +model##_TP_MAX_X, model##_TP_MAX_Y, model##_TP_RES_X, model##_TP_RES_Y } + +#define INTERNAL_TP_MAX_FINGER_ORIENTATION 16384 + +struct magicmouse_input_ops { + int (*raw_event)(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size); + int (*setup_input)(struct input_dev *input, struct hid_device *hdev); +}; + /** * struct magicmouse_sc - Tracks Magic Mouse-specific data. * @input: Input device through which we report events. * @quirks: Currently unused. + * @query_dimensions: Whether to query and update dimensions on first open * @ntouches: Number of touches in most recent touch report. * @scroll_accel: Number of consecutive scroll motions. * @scroll_jiffies: Time of last scroll motion. + * @pos: multi touch position data of the last report. * @touches: Most recent data for a touch, indexed by tracking ID. * @tracking_ids: Mapping of current touch input data to @touches. * @hdev: Pointer to the underlying HID device. * @work: Workqueue to handle initialization retry for quirky devices. * @battery_timer: Timer for obtaining battery level information. + * @input_ops: Input ops based on device type. */ struct magicmouse_sc { struct input_dev *input; unsigned long quirks; + bool query_dimensions; int ntouches; int scroll_accel; unsigned long scroll_jiffies; + struct input_mt_pos pos[MAX_CONTACTS]; struct { - short x; - short y; short scroll_x; short scroll_y; short scroll_x_hr; @@ -143,14 +275,112 @@ u8 size; bool scroll_x_active; bool scroll_y_active; - } touches[16]; - int tracking_ids[16]; + } touches[MAX_CONTACTS]; + int tracking_ids[MAX_CONTACTS]; struct hid_device *hdev; struct delayed_work work; struct timer_list battery_timer; + struct magicmouse_input_ops input_ops; }; +static int magicmouse_enable_multitouch(struct hid_device *hdev); + +static inline int le16_to_int(__le16 x) +{ + return (signed short)le16_to_cpu(x); +} + +static int magicmouse_open(struct input_dev *dev) +{ + struct hid_device *hdev = input_get_drvdata(dev); + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + int ret; + + ret = hid_hw_open(hdev); + if (ret) + return ret; + + /* + * Some devices repond with 'invalid report id' when feature + * report switching it into multitouch mode is sent to it. + * + * This results in -EIO from the _raw low-level transport callback, + * but there seems to be no other way of switching the mode. + * Thus the super-ugly hacky success check below. + * + * MTP devices do not need this. + */ + if (hdev->bus != BUS_HOST) { + ret = magicmouse_enable_multitouch(hdev); + if (ret != -EIO && ret < 0) { + hid_err(hdev, "unable to request touch data (%d)\n", ret); + return ret; + } + if (ret == -EIO && (hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2 || + hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2_USBC)) { + schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + } + } + + /* + * For Apple Silicon trackpads, we want to query the dimensions on + * device open. This is because doing so requires the firmware, but + * we don't want to force a firmware load until the device is opened + * for the first time. So do that here and update the input properties + * just in time before userspace queries them. + */ + if (msc->query_dimensions) { + struct input_dev *input = msc->input; + u8 buf[32]; + struct { + __le32 width; + __le32 height; + __le16 min_x; + __le16 min_y; + __le16 max_x; + __le16 max_y; + } dim; + uint32_t x_span, y_span; + + ret = hid_hw_raw_request(hdev, SENSOR_DIMENSIONS_REPORT_ID, buf, sizeof(buf), HID_FEATURE_REPORT, HID_REQ_GET_REPORT); + if (ret < (int)(1 + sizeof(dim))) { + hid_err(hdev, "unable to request dimensions (%d)\n", ret); + return ret; + } + + memcpy(&dim, buf + 1, sizeof(dim)); + + /* finger position */ + input_set_abs_params(input, ABS_MT_POSITION_X, + le16_to_int(dim.min_x), le16_to_int(dim.max_x), 0, 0); + /* Y axis is inverted */ + input_set_abs_params(input, ABS_MT_POSITION_Y, + -le16_to_int(dim.max_y), -le16_to_int(dim.min_y), 0, 0); + x_span = le16_to_int(dim.max_x) - le16_to_int(dim.min_x); + y_span = le16_to_int(dim.max_y) - le16_to_int(dim.min_y); + + /* X/Y resolution */ + input_abs_set_res(input, ABS_MT_POSITION_X, 100 * x_span / le32_to_cpu(dim.width) ); + input_abs_set_res(input, ABS_MT_POSITION_Y, 100 * y_span / le32_to_cpu(dim.height) ); + + /* copy info, as input_mt_init_slots() does */ + dev->absinfo[ABS_X] = dev->absinfo[ABS_MT_POSITION_X]; + dev->absinfo[ABS_Y] = dev->absinfo[ABS_MT_POSITION_Y]; + + msc->query_dimensions = false; + } + + return 0; +} + +static void magicmouse_close(struct input_dev *dev) +{ + struct hid_device *hdev = input_get_drvdata(dev); + + hid_hw_close(hdev); +} + static int magicmouse_firm_touch(struct magicmouse_sc *msc) { int touch = -1; @@ -192,7 +422,7 @@ } else if (last_state != 0) { state = last_state; } else if ((id = magicmouse_firm_touch(msc)) >= 0) { - int x = msc->touches[id].x; + int x = msc->pos[id].x; if (x < middle_button_start) state = 1; else if (x > middle_button_stop) @@ -256,8 +486,8 @@ /* Store tracking ID and other fields. */ msc->tracking_ids[raw_id] = id; - msc->touches[id].x = x; - msc->touches[id].y = y; + msc->pos[id].x = x; + msc->pos[id].y = y; msc->touches[id].size = size; /* If requested, emulate a scroll wheel by detecting small @@ -383,7 +613,7 @@ } } -static int __magicmouse_raw_event(struct hid_device *hdev, +static int __magicmouse_raw_event_usb(struct hid_device *hdev, struct hid_report *report, u8 *data, int size, bool nested) { struct magicmouse_sc *msc = hid_get_drvdata(hdev); @@ -515,8 +745,8 @@ return 0; } - __magicmouse_raw_event(hdev, report, data + 2, data[1], true); - __magicmouse_raw_event(hdev, report, data + 2 + data[1], + __magicmouse_raw_event_usb(hdev, report, data + 2, data[1], true); + __magicmouse_raw_event_usb(hdev, report, data + 2 + data[1], size - 2 - data[1], true); return 0; default: @@ -543,10 +773,192 @@ return 1; } -static int magicmouse_raw_event(struct hid_device *hdev, +static int magicmouse_raw_event_usb(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + return __magicmouse_raw_event_usb(hdev, report, data, size, false); +} + +/** + * struct tp_finger - single trackpad finger structure, le16-aligned + * + * @unknown1: unknown + * @unknown2: unknown + * @abs_x: absolute x coordinate + * @abs_y: absolute y coordinate + * @rel_x: relative x coordinate + * @rel_y: relative y coordinate + * @tool_major: tool area, major axis + * @tool_minor: tool area, minor axis + * @orientation: 16384 when point, else 15 bit angle + * @touch_major: touch area, major axis + * @touch_minor: touch area, minor axis + * @unused: zeros + * @pressure: pressure on forcetouch touchpad + * @multi: one finger: varies, more fingers: constant + */ +struct tp_finger { + __le16 unknown1; + __le16 unknown2; + __le16 abs_x; + __le16 abs_y; + __le16 rel_x; + __le16 rel_y; + __le16 tool_major; + __le16 tool_minor; + __le16 orientation; + __le16 touch_major; + __le16 touch_minor; + __le16 unused[2]; + __le16 pressure; + __le16 multi; +} __attribute__((packed, aligned(2))); + +/** + * struct vendor trackpad report + * + * @num_fingers: the number of fingers being reported in @fingers + * @buttons: same as HID buttons + */ +struct tp_header { + // HID vendor part, up to 1751 bytes + u8 unknown[22]; + u8 num_fingers; + u8 buttons; + u8 unknown3[14]; +}; + +/** + * struct standard HID mouse report + * + * @report_id: reportid + * @buttons: HID Usage Buttons 3 1-bit reports + */ +struct tp_mouse_report { + // HID mouse report + u8 report_id; + u8 buttons; + u8 rel_x; + u8 rel_y; + u8 padding[4]; +}; + +static void report_finger_data(struct input_dev *input, int slot, + const struct input_mt_pos *pos, + const struct tp_finger *f) +{ + input_mt_slot(input, slot); + input_mt_report_slot_state(input, MT_TOOL_FINGER, true); + + input_report_abs(input, ABS_MT_TOUCH_MAJOR, + le16_to_int(f->touch_major) << 1); + input_report_abs(input, ABS_MT_TOUCH_MINOR, + le16_to_int(f->touch_minor) << 1); + input_report_abs(input, ABS_MT_WIDTH_MAJOR, + le16_to_int(f->tool_major) << 1); + input_report_abs(input, ABS_MT_WIDTH_MINOR, + le16_to_int(f->tool_minor) << 1); + input_report_abs(input, ABS_MT_ORIENTATION, + INTERNAL_TP_MAX_FINGER_ORIENTATION - le16_to_int(f->orientation)); + input_report_abs(input, ABS_MT_PRESSURE, le16_to_int(f->pressure)); + input_report_abs(input, ABS_MT_POSITION_X, pos->x); + input_report_abs(input, ABS_MT_POSITION_Y, pos->y); +} + +static int magicmouse_raw_event_mtp(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + struct input_dev *input = msc->input; + struct tp_header *tp_hdr; + struct tp_finger *f; + int i, n; + u32 npoints; + const size_t hdr_sz = sizeof(struct tp_header); + const size_t touch_sz = sizeof(struct tp_finger); + u8 map_contacs[MAX_CONTACTS]; + + // hid_warn(hdev, "%s\n", __func__); + // print_hex_dump_debug("appleft ev: ", DUMP_PREFIX_OFFSET, 16, 1, data, + // size, false); + + /* Expect 46 bytes of prefix, and N * 30 bytes of touch data. */ + if (size < hdr_sz || ((size - hdr_sz) % touch_sz) != 0) + return 0; + + tp_hdr = (struct tp_header *)data; + + npoints = (size - hdr_sz) / touch_sz; + if (npoints < tp_hdr->num_fingers || npoints > MAX_CONTACTS) { + hid_warn(hdev, + "unexpected number of touches (%u) for " + "report\n", + npoints); + return 0; + } + + n = 0; + for (i = 0; i < tp_hdr->num_fingers; i++) { + f = (struct tp_finger *)(data + hdr_sz + i * touch_sz); + if (le16_to_int(f->touch_major) == 0) + continue; + + hid_dbg(hdev, "ev x:%04x y:%04x\n", le16_to_int(f->abs_x), + le16_to_int(f->abs_y)); + msc->pos[n].x = le16_to_int(f->abs_x); + msc->pos[n].y = -le16_to_int(f->abs_y); + map_contacs[n] = i; + n++; + } + + input_mt_assign_slots(input, msc->tracking_ids, msc->pos, n, 0); + + for (i = 0; i < n; i++) { + int idx = map_contacs[i]; + f = (struct tp_finger *)(data + hdr_sz + idx * touch_sz); + report_finger_data(input, msc->tracking_ids[i], &msc->pos[i], f); + } + + input_mt_sync_frame(input); + input_report_key(input, BTN_MOUSE, tp_hdr->buttons & 1); + + input_sync(input); + return 1; +} + +static int magicmouse_raw_event_spi(struct hid_device *hdev, struct hid_report *report, u8 *data, int size) { - return __magicmouse_raw_event(hdev, report, data, size, false); + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + const size_t hdr_sz = sizeof(struct tp_mouse_report); + + if (!size) + return 0; + + if (data[0] == SPI_RESET_REPORT_ID) { + hid_info(hdev, "Touch controller was reset, re-enabling touch mode\n"); + schedule_delayed_work(&msc->work, msecs_to_jiffies(10)); + return 1; + } + + if (data[0] != SPI_REPORT_ID || size < hdr_sz) + return 0; + + return magicmouse_raw_event_mtp(hdev, report, data + hdr_sz, size - hdr_sz); +} + +static int magicmouse_raw_event_t2(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + const size_t hdr_sz = sizeof(struct tp_mouse_report); + + if (!size) + return 0; + + if (data[0] != TRACKPAD2_USB_REPORT_ID || size < hdr_sz) + return 0; + + return magicmouse_raw_event_mtp(hdev, report, data + hdr_sz, size - hdr_sz); } static int magicmouse_event(struct hid_device *hdev, struct hid_field *field, @@ -567,7 +979,24 @@ return 0; } -static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hdev) +static int magicmouse_raw_event(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + return msc->input_ops.raw_event(hdev, report, data, size); +} + +static int magicmouse_setup_input(struct input_dev *input, + struct hid_device *hdev) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + return msc->input_ops.setup_input(input, hdev); +} + +static int magicmouse_setup_input_usb(struct input_dev *input, + struct hid_device *hdev) { int error; int mt_flags = 0; @@ -646,7 +1075,7 @@ __set_bit(EV_ABS, input->evbit); - error = input_mt_init_slots(input, 16, mt_flags); + error = input_mt_init_slots(input, MAX_CONTACTS, mt_flags); if (error) return error; input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 255 << 2, @@ -726,6 +1155,171 @@ */ __clear_bit(EV_REP, input->evbit); + /* + * This isn't strictly speaking needed for USB, but enabling MT on + * device open is probably more robust than only doing it once on probe + * even if USB devices are not known to suffer from the SPI reset issue. + */ + input->open = magicmouse_open; + input->close = magicmouse_close; + return 0; +} + +struct magicmouse_t2_properties { + u32 id; + int min_x; + int min_y; + int max_x; + int max_y; + int res_x; + int res_y; +}; + +static const struct magicmouse_t2_properties magicmouse_t2_configs[] = { + T2_TOUCHPAD_ENTRY(J140K), + T2_TOUCHPAD_ENTRY(J132), + T2_TOUCHPAD_ENTRY(J680), + T2_TOUCHPAD_ENTRY(J680_ALT), + T2_TOUCHPAD_ENTRY(J213), + T2_TOUCHPAD_ENTRY(J214K), + T2_TOUCHPAD_ENTRY(J223), + T2_TOUCHPAD_ENTRY(J230K), + T2_TOUCHPAD_ENTRY(J152F), +}; + +static int magicmouse_setup_input_int_tpd(struct input_dev *input, + struct hid_device *hdev, int min_x, int min_y, + int max_x, int max_y, int res_x, int res_y, + bool query_dimensions) +{ + int error; + int mt_flags = 0; + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + __set_bit(INPUT_PROP_BUTTONPAD, input->propbit); + __clear_bit(BTN_0, input->keybit); + __clear_bit(BTN_RIGHT, input->keybit); + __clear_bit(BTN_MIDDLE, input->keybit); + __clear_bit(EV_REL, input->evbit); + __clear_bit(REL_X, input->relbit); + __clear_bit(REL_Y, input->relbit); + + mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK; + + /* finger touch area */ + input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 5000, 0, 0); + input_set_abs_params(input, ABS_MT_TOUCH_MINOR, 0, 5000, 0, 0); + + /* finger approach area */ + input_set_abs_params(input, ABS_MT_WIDTH_MAJOR, 0, 5000, 0, 0); + input_set_abs_params(input, ABS_MT_WIDTH_MINOR, 0, 5000, 0, 0); + + /* Note: Touch Y position from the device is inverted relative + * to how pointer motion is reported (and relative to how USB + * HID recommends the coordinates work). This driver keeps + * the origin at the same position, and just uses the additive + * inverse of the reported Y. + */ + + input_set_abs_params(input, ABS_MT_PRESSURE, 0, 6000, 0, 0); + + /* + * This makes libinput recognize this as a PressurePad and + * stop trying to use pressure for touch size. Pressure unit + * seems to be ~grams on these touchpads. + */ + input_abs_set_res(input, ABS_MT_PRESSURE, 1); + + /* finger orientation */ + input_set_abs_params(input, ABS_MT_ORIENTATION, -INTERNAL_TP_MAX_FINGER_ORIENTATION, + INTERNAL_TP_MAX_FINGER_ORIENTATION, 0, 0); + + /* finger position */ + input_set_abs_params(input, ABS_MT_POSITION_X, min_x, max_x, 0, 0); + /* Y axis is inverted */ + input_set_abs_params(input, ABS_MT_POSITION_Y, -max_y, -min_y, 0, 0); + + /* X/Y resolution */ + input_abs_set_res(input, ABS_MT_POSITION_X, res_x); + input_abs_set_res(input, ABS_MT_POSITION_Y, res_y); + + input_set_events_per_packet(input, 60); + + /* touchpad button */ + input_set_capability(input, EV_KEY, BTN_MOUSE); + + /* + * hid-input may mark device as using autorepeat, but the trackpad does + * not actually want it. + */ + __clear_bit(EV_REP, input->evbit); + + error = input_mt_init_slots(input, MAX_CONTACTS, mt_flags); + if (error) + return error; + + /* + * Override the default input->open function to send the MT + * enable every time the device is opened. This ensures it works + * even if we missed a reset event due to the device being closed. + * input->close is overridden for symmetry. + * + * This also takes care of the dimensions query. + */ + input->open = magicmouse_open; + input->close = magicmouse_close; + msc->query_dimensions = query_dimensions; + + return 0; +} + +static int magicmouse_setup_input_mtp(struct input_dev *input, + struct hid_device *hdev) +{ + int ret = magicmouse_setup_input_int_tpd(input, hdev, J314_TP_MIN_X, + J314_TP_MIN_Y, J314_TP_MAX_X, + J314_TP_MAX_Y, J314_TP_RES_X, + J314_TP_RES_Y, true); + if (ret) + return ret; + + return 0; +} + +static int magicmouse_setup_input_spi(struct input_dev *input, + struct hid_device *hdev) +{ + int ret = magicmouse_setup_input_int_tpd(input, hdev, J314_TP_MIN_X, + J314_TP_MIN_Y, J314_TP_MAX_X, + J314_TP_MAX_Y, J314_TP_RES_X, + J314_TP_RES_Y, true); + if (ret) + return ret; + + return 0; +} + +static int magicmouse_setup_input_t2(struct input_dev *input, + struct hid_device *hdev) +{ + int min_x, min_y, max_x, max_y, res_x, res_y; + + for (size_t i = 0; i < ARRAY_SIZE(magicmouse_t2_configs); i++) { + if (magicmouse_t2_configs[i].id == hdev->product) { + min_x = magicmouse_t2_configs[i].min_x; + min_y = magicmouse_t2_configs[i].min_y; + max_x = magicmouse_t2_configs[i].max_x; + max_y = magicmouse_t2_configs[i].max_y; + res_x = magicmouse_t2_configs[i].res_x; + res_y = magicmouse_t2_configs[i].res_y; + } + } + + int ret = magicmouse_setup_input_int_tpd(input, hdev, min_x, min_y, + max_x, max_y, res_x, res_y, false); + if (ret) + return ret; + return 0; } @@ -784,14 +1378,27 @@ int feature_size; switch (hdev->product) { + case SPI_DEVICE_ID_APPLE_MACBOOK_AIR_2020: + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020: + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO14_2021: + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO16_2021: case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2: case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F: switch (hdev->vendor) { case BT_VENDOR_ID_APPLE: feature_size = sizeof(feature_mt_trackpad2_bt); feature = feature_mt_trackpad2_bt; break; - default: /* USB_VENDOR_ID_APPLE */ + default: /* USB_VENDOR_ID_APPLE || SPI_VENDOR_ID_APPLE */ feature_size = sizeof(feature_mt_trackpad2_usb); feature = feature_mt_trackpad2_usb; } @@ -897,12 +1504,62 @@ struct hid_report *report; int ret; + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && id->vendor == SPI_VENDOR_ID_APPLE && + hdev->type != HID_TYPE_SPI_MOUSE) + return -ENODEV; + + switch (id->product) { + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F: + if (hdev->type != HID_TYPE_USBMOUSE) + return -ENODEV; + break; + } + msc = devm_kzalloc(&hdev->dev, sizeof(*msc), GFP_KERNEL); if (msc == NULL) { hid_err(hdev, "can't alloc magicmouse descriptor\n"); return -ENOMEM; } + // internal trackpad use a data format use input ops to avoid + // conflicts with the report ID. + switch (id->bus) { + case BUS_HOST: + msc->input_ops.raw_event = magicmouse_raw_event_mtp; + msc->input_ops.setup_input = magicmouse_setup_input_mtp; + break; + case BUS_SPI: + msc->input_ops.raw_event = magicmouse_raw_event_spi; + msc->input_ops.setup_input = magicmouse_setup_input_spi; + break; + default: + switch (id->product) { + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F: + msc->input_ops.raw_event = magicmouse_raw_event_t2; + msc->input_ops.setup_input = magicmouse_setup_input_t2; + break; + default: + msc->input_ops.raw_event = magicmouse_raw_event_usb; + msc->input_ops.setup_input = magicmouse_setup_input_usb; + } + } + msc->scroll_accel = SCROLL_ACCEL_DEFAULT; msc->hdev = hdev; INIT_DEFERRABLE_WORK(&msc->work, magicmouse_enable_mt_work); @@ -961,6 +1618,15 @@ break; case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2: case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F: switch (id->vendor) { case BT_VENDOR_ID_APPLE: report = hid_register_report(hdev, HID_INPUT_REPORT, @@ -971,11 +1637,20 @@ TRACKPAD2_USB_REPORT_ID, 0); } break; - default: /* USB_DEVICE_ID_APPLE_MAGICTRACKPAD */ - report = hid_register_report(hdev, HID_INPUT_REPORT, - TRACKPAD_REPORT_ID, 0); - report = hid_register_report(hdev, HID_INPUT_REPORT, - DOUBLE_REPORT_ID, 0); + default: + switch (id->bus) { + case BUS_HOST: + report = hid_register_report(hdev, HID_INPUT_REPORT, MTP_REPORT_ID, 0); + break; + case BUS_SPI: + report = hid_register_report(hdev, HID_INPUT_REPORT, SPI_REPORT_ID, 0); + break; + default: /* USB_DEVICE_ID_APPLE_MAGICTRACKPAD */ + report = hid_register_report(hdev, HID_INPUT_REPORT, + TRACKPAD_REPORT_ID, 0); + report = hid_register_report(hdev, HID_INPUT_REPORT, + DOUBLE_REPORT_ID, 0); + } } if (!report) { @@ -985,22 +1660,14 @@ } report->size = 6; - /* - * Some devices repond with 'invalid report id' when feature - * report switching it into multitouch mode is sent to it. - * - * This results in -EIO from the _raw low-level transport callback, - * but there seems to be no other way of switching the mode. - * Thus the super-ugly hacky success check below. - */ - ret = magicmouse_enable_multitouch(hdev); - if (ret != -EIO && ret < 0) { - hid_err(hdev, "unable to request touch data (%d)\n", ret); - goto err_stop_hw; - } - if (ret == -EIO && (id->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2 || - id->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2_USBC)) { - schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + /* MTP devices do not need the MT enable, this is handled by the MTP driver */ + if (id->bus == BUS_HOST) + return 0; + + /* SPI devices need to watch for reset events to re-send the MT enable */ + if (id->bus == BUS_SPI) { + report = hid_register_report(hdev, HID_INPUT_REPORT, SPI_RESET_REPORT_ID, 0); + report->size = 2; } /* @@ -1046,7 +1713,7 @@ * enable report. Only the HID_TYPE_USBMOUSE interface accepts it, and * it must be deferred. Sending it inline here is too early. */ - if (msc && hdev->type == HID_TYPE_USBMOUSE) + if (msc && (hdev->type == HID_TYPE_USBMOUSE || hdev->bus == BUS_SPI)) schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); return 0; @@ -1102,6 +1769,28 @@ USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC), .driver_data = 0 }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F), .driver_data = 0 }, + { HID_SPI_DEVICE(SPI_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = 0 }, + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, + HID_ANY_ID), .driver_data = 0 }, { } }; MODULE_DEVICE_TABLE(hid, magic_mice); diff -ruN --no-dereference -x .git a/drivers/hid/spi-hid/Kconfig b/drivers/hid/spi-hid/Kconfig --- a/drivers/hid/spi-hid/Kconfig 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/hid/spi-hid/Kconfig 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0-only +menu "SPI HID support" + depends on SPI + +config SPI_HID_APPLE_OF + tristate "HID over SPI transport layer for Apple Silicon SoCs" + depends on INPUT && OF + select SPI_HID_APPLE_CORE + help + Say Y here if you use Apple Silicon based laptop. The keyboard and + touchpad are HID based devices connected via SPI. + + If unsure, say N. + + This support is also available as a module. If so, the module + will be called spi-hid-apple-of. It will also build/depend on the + module spi-hid-apple. + +endmenu + +config SPI_HID_APPLE_CORE + tristate + select HID + select CRC16 diff -ruN --no-dereference -x .git a/drivers/hid/spi-hid/Makefile b/drivers/hid/spi-hid/Makefile --- a/drivers/hid/spi-hid/Makefile 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/hid/spi-hid/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for SPI HID tarnsport drivers +# + +obj-$(CONFIG_SPI_HID_APPLE_CORE) += spi-hid-apple.o + +spi-hid-apple-objs = spi-hid-apple-core.o + +obj-$(CONFIG_SPI_HID_APPLE_OF) += spi-hid-apple-of.o diff -ruN --no-dereference -x .git a/drivers/hid/spi-hid/spi-hid-apple-core.c b/drivers/hid/spi-hid/spi-hid-apple-core.c --- a/drivers/hid/spi-hid/spi-hid-apple-core.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/hid/spi-hid/spi-hid-apple-core.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,1194 @@ +/* + * SPDX-License-Identifier: GPL-2.0 + * + * Apple SPI HID transport driver + * + * Copyright (C) The Asahi Linux Contributors + * + * Based on: drivers/input/applespi.c + * + * MacBook (Pro) SPI keyboard and touchpad driver + * + * Copyright (c) 2015-2018 Federico Lorenzi + * Copyright (c) 2017-2018 Ronald Tschalär + * + */ + +//#define DEBUG 2 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "spi-hid-apple.h" + +#define SPIHID_DEF_WAIT msecs_to_jiffies(1000) + +#define SPIHID_MAX_INPUT_REPORT_SIZE 0x800 + +/* support only keyboard, trackpad and management dev for now */ +#define SPIHID_MAX_DEVICES 3 + +#define SPIHID_DEVICE_ID_MNGT 0x0 +#define SPIHID_DEVICE_ID_KBD 0x1 +#define SPIHID_DEVICE_ID_TP 0x2 +#define SPIHID_DEVICE_ID_INFO 0xd0 + +#define SPIHID_READ_PACKET 0x20 +#define SPIHID_WRITE_PACKET 0x40 + +#define SPIHID_DESC_MAX 512 + +#define SPIHID_SET_LEDS 0x0151 /* caps lock */ + +#define SPI_RW_CHG_DELAY_US 200 /* 'Inter Stage Us'? */ + +static const u8 spi_hid_apple_booted[4] = { 0xa0, 0x80, 0x00, 0x00 }; +static const u8 spi_hid_apple_status_ok[4] = { 0xac, 0x27, 0x68, 0xd5 }; + +struct spihid_interface { + struct hid_device *hid; + u8 *hid_desc; + u32 hid_desc_len; + u32 id; + unsigned country; + u32 max_control_report_len; + u32 max_input_report_len; + u32 max_output_report_len; + u8 name[32]; + u8 reply_buf[SPIHID_DESC_MAX]; + u32 reply_len; + bool ready; +}; + +struct spihid_input_report { + u8 *buf; + u32 length; + u32 offset; + u8 device; + u8 flags; +}; + +struct spihid_apple { + struct spi_device *spidev; + + struct spihid_apple_ops *ops; + + struct spihid_interface mngt; + struct spihid_interface kbd; + struct spihid_interface tp; + + wait_queue_head_t wait; + struct mutex tx_lock; //< protects against concurrent SPI writes + + struct spi_message rx_msg; + struct spi_message tx_msg; + struct spi_transfer rx_transfer; + struct spi_transfer tx_transfer; + struct spi_transfer status_transfer; + + u8 *rx_buf; + u8 *tx_buf; + u8 *status_buf; + + u8 vendor[32]; + u8 product[64]; + u8 serial[32]; + + u32 num_devices; + + u32 vendor_id; + u32 product_id; + u32 version_number; + + u8 msg_id; + + /* fragmented HID report */ + struct spihid_input_report report; + + /* state tracking flags */ + bool status_booted; + +#ifdef IRQ_WAKE_SUPPORT + bool irq_wake_enabled; +#endif +}; + +/** + * struct spihid_msg_hdr - common header of protocol messages. + * + * Each message begins with fixed header, followed by a message-type specific + * payload, and ends with a 16-bit crc. Because of the varying lengths of the + * payload, the crc is defined at the end of each payload struct, rather than + * in this struct. + * + * @unknown0: request type? output, input (0x10), feature, protocol + * @unknown1: maybe report id? + * @unknown2: mostly zero, in info request maybe device num + * @id: incremented on each message, rolls over after 255; there is a + * separate counter for each message type. + * @rsplen: response length (the exact nature of this field is quite + * speculative). On a request/write this is often the same as + * @length, though in some cases it has been seen to be much larger + * (e.g. 0x400); on a response/read this the same as on the + * request; for reads that are not responses it is 0. + * @length: length of the remainder of the data in the whole message + * structure (after re-assembly in case of being split over + * multiple spi-packets), minus the trailing crc. The total size + * of a message is therefore @length + 10. + */ + +struct spihid_msg_hdr { + u8 unknown0; + u8 unknown1; + u8 unknown2; + u8 id; + __le16 rsplen; + __le16 length; +}; + +/** + * struct spihid_transfer_packet - a complete spi packet; always 256 bytes. This carries + * the (parts of the) message in the data. But note that this does not + * necessarily contain a complete message, as in some cases (e.g. many + * fingers pressed) the message is split over multiple packets (see the + * @offset, @remain, and @length fields). In general the data parts in + * spihid_transfer_packet's are concatenated until @remaining is 0, and the + * result is an message. + * + * @flags: 0x40 = write (to device), 0x20 = read (from device); note that + * the response to a write still has 0x40. + * @device: 1 = keyboard, 2 = touchpad + * @offset: specifies the offset of this packet's data in the complete + * message; i.e. > 0 indicates this is a continuation packet (in + * the second packet for a message split over multiple packets + * this would then be the same as the @length in the first packet) + * @remain: number of message bytes remaining in subsequents packets (in + * the first packet of a message split over two packets this would + * then be the same as the @length in the second packet) + * @length: length of the valid data in the @data in this packet + * @data: all or part of a message + * @crc16: crc over this whole structure minus this @crc16 field. This + * covers just this packet, even on multi-packet messages (in + * contrast to the crc in the message). + */ +struct spihid_transfer_packet { + u8 flags; + u8 device; + __le16 offset; + __le16 remain; + __le16 length; + u8 data[246]; + __le16 crc16; +}; + +/* + * how HID is mapped onto the protocol is not fully clear. This are the known + * reports/request: + * + * pkt.flags pkt.dev? msg.u0 msg.u1 msg.u2 + * info 0x40 0xd0 0x20 0x01 0xd0 + * + * info mngt: 0x40 0xd0 0x20 0x10 0x00 + * info kbd: 0x40 0xd0 0x20 0x10 0x01 + * info tp: 0x40 0xd0 0x20 0x10 0x02 + * + * desc kbd: 0x40 0xd0 0x20 0x10 0x01 + * desc trackpad: 0x40 0xd0 0x20 0x10 0x02 + * + * mt mode: 0x40 0x02 0x52 0x02 0x00 set protocol? + * capslock led 0x40 0x01 0x51 0x01 0x00 output report + * + * report kbd: 0x20 0x01 0x10 0x01 0x00 input report + * report tp: 0x20 0x02 0x10 0x02 0x00 input report + * + */ + + +static int spihid_apple_request(struct spihid_apple *spihid, u8 target, u8 unk0, + u8 unk1, u8 unk2, u16 resp_len, u8 *buf, + size_t len) +{ + struct spihid_transfer_packet *pkt; + struct spihid_msg_hdr *hdr; + u16 crc; + int err; + + /* know reports are small enoug to fit in a single packet */ + if (len > sizeof(pkt->data) - sizeof(*hdr) - sizeof(__le16)) + return -EINVAL; + + err = mutex_lock_interruptible(&spihid->tx_lock); + if (err < 0) + return err; + + pkt = (struct spihid_transfer_packet *)spihid->tx_buf; + + memset(pkt, 0, sizeof(*pkt)); + pkt->flags = SPIHID_WRITE_PACKET; + pkt->device = target; + pkt->length = cpu_to_le16(sizeof(*hdr) + len + sizeof(__le16)); + + hdr = (struct spihid_msg_hdr *)&pkt->data[0]; + hdr->unknown0 = unk0; + hdr->unknown1 = unk1; + hdr->unknown2 = unk2; + hdr->id = spihid->msg_id++; + hdr->rsplen = cpu_to_le16(resp_len); + hdr->length = cpu_to_le16(len); + + if (len) + memcpy(pkt->data + sizeof(*hdr), buf, len); + crc = crc16(0, &pkt->data[0], sizeof(*hdr) + len); + put_unaligned_le16(crc, pkt->data + sizeof(*hdr) + len); + + pkt->crc16 = cpu_to_le16(crc16(0, spihid->tx_buf, + offsetof(struct spihid_transfer_packet, crc16))); + + memset(spihid->status_buf, 0, sizeof(spi_hid_apple_status_ok)); + + err = spi_sync(spihid->spidev, &spihid->tx_msg); + + if (memcmp(spihid->status_buf, spi_hid_apple_status_ok, + sizeof(spi_hid_apple_status_ok))) { + u8 *b = spihid->status_buf; + dev_warn_ratelimited(&spihid->spidev->dev, "status message " + "mismatch: %02x %02x %02x %02x\n", + b[0], b[1], b[2], b[3]); + } + mutex_unlock(&spihid->tx_lock); + if (err < 0) + return err; + + return (int)len; +} + +static struct spihid_apple *spihid_get_data(struct spihid_interface *idev) +{ + switch (idev->id) { + case SPIHID_DEVICE_ID_KBD: + return container_of(idev, struct spihid_apple, kbd); + case SPIHID_DEVICE_ID_TP: + return container_of(idev, struct spihid_apple, tp); + default: + return NULL; + } +} + +static int apple_ll_start(struct hid_device *hdev) +{ + /* no-op SPI transport is already setup */ + return 0; +}; + +static void apple_ll_stop(struct hid_device *hdev) +{ + /* no-op, devices will be desstroyed on driver destruction */ +} + +static int apple_ll_open(struct hid_device *hdev) +{ + struct spihid_apple *spihid; + struct spihid_interface *idev = hdev->driver_data; + + if (idev->hid_desc_len == 0) { + spihid = spihid_get_data(idev); + dev_warn(&spihid->spidev->dev, + "HID descriptor missing for dev %u", idev->id); + } else + idev->ready = true; + + return 0; +} + +static void apple_ll_close(struct hid_device *hdev) +{ + struct spihid_interface *idev = hdev->driver_data; + idev->ready = false; +} + +static int apple_ll_parse(struct hid_device *hdev) +{ + struct spihid_interface *idev = hdev->driver_data; + + return hid_parse_report(hdev, idev->hid_desc, idev->hid_desc_len); +} + +static int apple_ll_raw_request(struct hid_device *hdev, + unsigned char reportnum, __u8 *buf, size_t len, + unsigned char rtype, int reqtype) +{ + struct spihid_interface *idev = hdev->driver_data; + struct spihid_apple *spihid = spihid_get_data(idev); + int ret; + + dev_dbg(&spihid->spidev->dev, + "apple_ll_raw_request: device:%u reportnum:%hhu rtype:%hhu", + idev->id, reportnum, rtype); + + switch (reqtype) { + case HID_REQ_GET_REPORT: + if (rtype != HID_FEATURE_REPORT) + return -EINVAL; + + idev->reply_len = 0; + ret = spihid_apple_request(spihid, idev->id, 0x32, reportnum, 0x00, len, NULL, 0); + if (ret < 0) + return ret; + + ret = wait_event_interruptible_timeout(spihid->wait, idev->reply_len, + SPIHID_DEF_WAIT); + if (ret == 0) + ret = -ETIMEDOUT; + if (ret < 0) { + dev_err(&spihid->spidev->dev, "waiting for get report failed: %d", ret); + return ret; + } + memcpy(buf, idev->reply_buf, max_t(size_t, len, idev->reply_len)); + return idev->reply_len; + + case HID_REQ_SET_REPORT: + if (buf[0] != reportnum) + return -EINVAL; + if (reportnum != idev->id) { + dev_warn(&spihid->spidev->dev, + "device:%u reportnum:" + "%hhu mismatch", + idev->id, reportnum); + return -EINVAL; + } + return spihid_apple_request(spihid, idev->id, 0x52, reportnum, 0x00, 2, buf, len); + default: + return -EIO; + } +} + +static int apple_ll_output_report(struct hid_device *hdev, __u8 *buf, + size_t len) +{ + struct spihid_interface *idev = hdev->driver_data; + struct spihid_apple *spihid = spihid_get_data(idev); + if (!spihid) + return -1; + + dev_dbg(&spihid->spidev->dev, + "apple_ll_output_report: device:%u len:%zu:", + idev->id, len); + // second idev->id should maybe be buf[0]? + return spihid_apple_request(spihid, idev->id, 0x51, idev->id, 0x00, 0, buf, len); +} + +static struct hid_ll_driver apple_hid_ll = { + .start = &apple_ll_start, + .stop = &apple_ll_stop, + .open = &apple_ll_open, + .close = &apple_ll_close, + .parse = &apple_ll_parse, + .raw_request = &apple_ll_raw_request, + .output_report = &apple_ll_output_report, + .max_buffer_size = SPIHID_MAX_INPUT_REPORT_SIZE, +}; + +static struct spihid_interface *spihid_get_iface(struct spihid_apple *spihid, + u32 iface) +{ + switch (iface) { + case SPIHID_DEVICE_ID_MNGT: + return &spihid->mngt; + case SPIHID_DEVICE_ID_KBD: + return &spihid->kbd; + case SPIHID_DEVICE_ID_TP: + return &spihid->tp; + default: + return NULL; + } +} + +static int spihid_verify_msg(struct spihid_apple *spihid, u8 *buf, size_t len) +{ + u16 msg_crc, crc; + struct device *dev = &spihid->spidev->dev; + + crc = crc16(0, buf, len - sizeof(__le16)); + msg_crc = get_unaligned_le16(buf + len - sizeof(__le16)); + if (crc != msg_crc) { + dev_warn_ratelimited(dev, "Read message crc mismatch\n"); + return 0; + } + return 1; +} + +static bool spihid_status_report(struct spihid_apple *spihid, u8 *pl, + size_t len) +{ + struct device *dev = &spihid->spidev->dev; + dev_dbg(dev, "%s: len: %zu", __func__, len); + if (len == 5 && pl[0] == 0xe0) + return true; + + return false; +} + +static bool spihid_process_input_report(struct spihid_apple *spihid, u32 device, + struct spihid_msg_hdr *hdr, u8 *payload, + size_t len) +{ + //dev_dbg(&spihid>spidev->dev, "input report: req:%hx iface:%u ", hdr->unknown0, device); + if (hdr->unknown0 != 0x10) + return false; + + /* HID device as well but Vendor usage only, handle it internally for now */ + if (device == 0) { + if (hdr->unknown1 == 0xe0) { + return spihid_status_report(spihid, payload, len); + } + } else if (device < SPIHID_MAX_DEVICES) { + struct spihid_interface *iface = + spihid_get_iface(spihid, device); + if (iface && iface->hid && iface->ready) { + hid_input_report(iface->hid, HID_INPUT_REPORT, payload, + len, 1); + return true; + } + } else + dev_dbg(&spihid->spidev->dev, + "unexpected iface:%u for input report", device); + + return false; +} + +struct spihid_device_info { + __le16 u0[2]; + __le16 num_devices; + __le16 vendor_id; + __le16 product_id; + __le16 version_number; + __le16 vendor_str[2]; //< offset and string length + __le16 product_str[2]; //< offset and string length + __le16 serial_str[2]; //< offset and string length +}; + +static bool spihid_process_device_info(struct spihid_apple *spihid, u32 iface, + u8 *payload, size_t len) +{ + struct device *dev = &spihid->spidev->dev; + + if (iface != SPIHID_DEVICE_ID_INFO) + return false; + + if (spihid->vendor_id == 0 && + len >= sizeof(struct spihid_device_info)) { + struct spihid_device_info *info = + (struct spihid_device_info *)payload; + u16 voff, vlen, poff, plen, soff, slen; + u32 num_devices; + + num_devices = __le16_to_cpu(info->num_devices); + + if (num_devices < SPIHID_MAX_DEVICES) { + dev_err(dev, + "Device info reports %u devices, expecting at least 3", + num_devices); + return false; + } + spihid->num_devices = num_devices; + + if (spihid->num_devices > SPIHID_MAX_DEVICES) { + dev_info( + dev, + "limiting the number of devices to mngt, kbd and mouse"); + spihid->num_devices = SPIHID_MAX_DEVICES; + } + + spihid->vendor_id = __le16_to_cpu(info->vendor_id); + spihid->product_id = __le16_to_cpu(info->product_id); + spihid->version_number = __le16_to_cpu(info->version_number); + + voff = __le16_to_cpu(info->vendor_str[0]); + vlen = __le16_to_cpu(info->vendor_str[1]); + + if (voff < len && vlen <= len - voff && + vlen < sizeof(spihid->vendor)) { + memcpy(spihid->vendor, payload + voff, vlen); + spihid->vendor[vlen] = '\0'; + } + + poff = __le16_to_cpu(info->product_str[0]); + plen = __le16_to_cpu(info->product_str[1]); + + if (poff < len && plen <= len - poff && + plen < sizeof(spihid->product)) { + memcpy(spihid->product, payload + poff, plen); + spihid->product[plen] = '\0'; + } + + soff = __le16_to_cpu(info->serial_str[0]); + slen = __le16_to_cpu(info->serial_str[1]); + + if (soff < len && slen <= len - soff && + slen < sizeof(spihid->serial)) { + memcpy(spihid->vendor, payload + soff, slen); + spihid->serial[slen] = '\0'; + } + + wake_up_interruptible(&spihid->wait); + } + return true; +} + +struct spihid_iface_info { + u8 u_0; + u8 interface_num; + u8 u_2; + u8 u_3; + u8 u_4; + u8 country_code; + __le16 max_input_report_len; + __le16 max_output_report_len; + __le16 max_control_report_len; + __le16 name_offset; + __le16 name_length; +}; + +static bool spihid_process_iface_info(struct spihid_apple *spihid, u32 num, + u8 *payload, size_t len) +{ + struct spihid_iface_info *info; + struct spihid_interface *iface = spihid_get_iface(spihid, num); + u32 name_off, name_len; + + if (!iface) + return false; + + if (!iface->max_input_report_len) { + if (len < sizeof(*info)) + return false; + + info = (struct spihid_iface_info *)payload; + + iface->max_input_report_len = + le16_to_cpu(info->max_input_report_len); + iface->max_output_report_len = + le16_to_cpu(info->max_output_report_len); + iface->max_control_report_len = + le16_to_cpu(info->max_control_report_len); + iface->country = info->country_code; + + name_off = le16_to_cpu(info->name_offset); + name_len = le16_to_cpu(info->name_length); + + if (name_off < len && name_len <= len - name_off && + name_len < sizeof(iface->name)) { + memcpy(iface->name, payload + name_off, name_len); + iface->name[name_len] = '\0'; + } + + dev_dbg(&spihid->spidev->dev, "Info for %s, country code: 0x%x", + iface->name, iface->country); + + wake_up_interruptible(&spihid->wait); + } + + return true; +} + +static int spihid_register_hid_device(struct spihid_apple *spihid, + struct spihid_interface *idev, u8 device); + +static bool spihid_process_iface_hid_report_desc(struct spihid_apple *spihid, + u32 num, u8 *payload, + size_t len) +{ + struct spihid_interface *iface = spihid_get_iface(spihid, num); + + if (!iface) + return false; + + if (iface->hid_desc_len == 0) { + if (len > SPIHID_DESC_MAX) + return false; + memcpy(iface->hid_desc, payload, len); + iface->hid_desc_len = len; + + /* do not register the mngt iface as HID device */ + if (num > 0) + spihid_register_hid_device(spihid, iface, num); + + wake_up_interruptible(&spihid->wait); + } + return true; +} + +static bool spihid_process_iface_get_report(struct spihid_apple *spihid, + u32 device, u8 report, + u8 *payload, size_t len) +{ + struct spihid_interface *iface = spihid_get_iface(spihid, device); + + if (!iface) + return false; + + if (len > sizeof(iface->reply_buf) || len < 1) + return false; + + memcpy(iface->reply_buf, payload, len); + iface->reply_len = len; + + wake_up_interruptible(&spihid->wait); + + return true; +} + +static bool spihid_process_response(struct spihid_apple *spihid, u32 device, + struct spihid_msg_hdr *hdr, u8 *payload, + size_t len) +{ + if (hdr->unknown0 == 0x20) { + switch (hdr->unknown1) { + case 0x01: + return spihid_process_device_info(spihid, hdr->unknown2, + payload, len); + case 0x02: + return spihid_process_iface_info(spihid, hdr->unknown2, + payload, len); + case 0x10: + return spihid_process_iface_hid_report_desc( + spihid, hdr->unknown2, payload, len); + default: + break; + } + } + + if (hdr->unknown0 == 0x32) { + return spihid_process_iface_get_report(spihid, device, hdr->unknown1, payload, len); + } + + return false; +} + +static void spihid_process_message(struct spihid_apple *spihid, u8 *data, + size_t length, u8 device, u8 flags) +{ + struct device *dev = &spihid->spidev->dev; + struct spihid_msg_hdr *hdr; + bool handled = false; + size_t payload_len; + u8 *payload; + + if (!spihid_verify_msg(spihid, data, length)) + return; + + hdr = (struct spihid_msg_hdr *)data; + payload_len = le16_to_cpu(hdr->length); + + if (payload_len == 0 || + (payload_len + sizeof(struct spihid_msg_hdr) + 2) > length) + return; + + payload = data + sizeof(struct spihid_msg_hdr); + + switch (flags) { + case SPIHID_READ_PACKET: + handled = spihid_process_input_report(spihid, device, hdr, + payload, payload_len); + break; + case SPIHID_WRITE_PACKET: + handled = spihid_process_response(spihid, device, hdr, payload, + payload_len); + break; + default: + break; + } + +#if defined(DEBUG) && DEBUG > 1 + { + dev_dbg(dev, + "R msg: req:%02hhx rep:%02hhx dev:%02hhx id:%hu len:%hu\n", + hdr->unknown0, hdr->unknown1, hdr->unknown2, hdr->id, + hdr->length); + print_hex_dump_debug("spihid msg: ", DUMP_PREFIX_OFFSET, 16, 1, + payload, le16_to_cpu(hdr->length), true); + } +#else + if (!handled) { + dev_dbg(dev, + "R unhandled msg: req:%02hhx rep:%02hhx dev:%02hhx id:%hu len:%hu\n", + hdr->unknown0, hdr->unknown1, hdr->unknown2, hdr->id, + hdr->length); + print_hex_dump_debug("spihid msg: ", DUMP_PREFIX_OFFSET, 16, 1, + payload, le16_to_cpu(hdr->length), true); + } +#endif +} + +static void spihid_assemble_message(struct spihid_apple *spihid, + struct spihid_transfer_packet *pkt) +{ + size_t length, offset, remain; + struct device *dev = &spihid->spidev->dev; + struct spihid_input_report *rep = &spihid->report; + + length = le16_to_cpu(pkt->length); + remain = le16_to_cpu(pkt->remain); + offset = le16_to_cpu(pkt->offset); + + if (offset + length + remain > U16_MAX) { + return; + } + + if (pkt->device != rep->device || pkt->flags != rep->flags || + offset != rep->offset) { + rep->device = 0; + rep->flags = 0; + rep->offset = 0; + rep->length = 0; + } + + if (offset == 0) { + if (rep->offset != 0) { + dev_warn(dev, "incomplete report off:%u len:%u", + rep->offset, rep->length); + } + memcpy(rep->buf, pkt->data, length); + rep->offset = length; + rep->length = length + remain; + rep->device = pkt->device; + rep->flags = pkt->flags; + } else if (offset == rep->offset) { + if (offset + length + remain != rep->length) { + dev_warn(dev, "incomplete report off:%u len:%u", + rep->offset, rep->length); + return; + } + memcpy(rep->buf + offset, pkt->data, length); + rep->offset += length; + + if (rep->offset == rep->length) { + spihid_process_message(spihid, rep->buf, rep->length, + rep->device, rep->flags); + rep->device = 0; + rep->flags = 0; + rep->offset = 0; + rep->length = 0; + } + } +} + +static void spihid_process_read(struct spihid_apple *spihid) +{ + u16 crc; + size_t length; + struct device *dev = &spihid->spidev->dev; + struct spihid_transfer_packet *pkt; + + pkt = (struct spihid_transfer_packet *)spihid->rx_buf; + + /* check transfer packet crc */ + crc = crc16(0, spihid->rx_buf, + offsetof(struct spihid_transfer_packet, crc16)); + if (crc != le16_to_cpu(pkt->crc16)) { + dev_warn_ratelimited(dev, "Read package crc mismatch\n"); + return; + } + + length = le16_to_cpu(pkt->length); + + if (length < sizeof(struct spihid_msg_hdr) + 2) { + if (length == sizeof(spi_hid_apple_booted) && + !memcmp(pkt->data, spi_hid_apple_booted, length)) { + if (!spihid->status_booted) { + spihid->status_booted = true; + wake_up_interruptible(&spihid->wait); + } + } else { + dev_info(dev, "R short packet: len:%zu\n", length); + print_hex_dump(KERN_INFO, "spihid pkt:", + DUMP_PREFIX_OFFSET, 16, 1, pkt->data, + length, false); + } + return; + } + +#if defined(DEBUG) && DEBUG > 1 + dev_dbg(dev, + "R pkt: flags:%02hhx dev:%02hhx off:%hu remain:%hu, len:%zu\n", + pkt->flags, pkt->device, pkt->offset, pkt->remain, length); +#if defined(DEBUG) && DEBUG > 2 + print_hex_dump_debug("spihid pkt: ", DUMP_PREFIX_OFFSET, 16, 1, + spihid->rx_buf, + sizeof(struct spihid_transfer_packet), true); +#endif +#endif + + if (length > sizeof(pkt->data)) { + dev_warn_ratelimited(dev, "Invalid pkt len:%zu", length); + return; + } + + /* short message */ + if (pkt->offset == 0 && pkt->remain == 0) { + spihid_process_message(spihid, pkt->data, length, pkt->device, + pkt->flags); + } else { + spihid_assemble_message(spihid, pkt); + } +} + +static void spihid_read_packet_sync(struct spihid_apple *spihid) +{ + int err; + + err = spi_sync(spihid->spidev, &spihid->rx_msg); + if (!err) { + spihid_process_read(spihid); + } else { + dev_warn(&spihid->spidev->dev, "RX failed: %d\n", err); + } +} + +irqreturn_t spihid_apple_core_irq(int irq, void *data) +{ + struct spi_device *spi = data; + struct spihid_apple *spihid = spi_get_drvdata(spi); + + spihid_read_packet_sync(spihid); + + return IRQ_HANDLED; +} +EXPORT_SYMBOL_GPL(spihid_apple_core_irq); + +static void spihid_apple_setup_spi_msgs(struct spihid_apple *spihid) +{ + memset(&spihid->rx_transfer, 0, sizeof(spihid->rx_transfer)); + + spihid->rx_transfer.rx_buf = spihid->rx_buf; + spihid->rx_transfer.len = sizeof(struct spihid_transfer_packet); + + spi_message_init(&spihid->rx_msg); + spi_message_add_tail(&spihid->rx_transfer, &spihid->rx_msg); + + memset(&spihid->tx_transfer, 0, sizeof(spihid->rx_transfer)); + memset(&spihid->status_transfer, 0, sizeof(spihid->status_transfer)); + + spihid->tx_transfer.tx_buf = spihid->tx_buf; + spihid->tx_transfer.len = sizeof(struct spihid_transfer_packet); + spihid->tx_transfer.delay.unit = SPI_DELAY_UNIT_USECS; + spihid->tx_transfer.delay.value = SPI_RW_CHG_DELAY_US; + + spihid->status_transfer.rx_buf = spihid->status_buf; + spihid->status_transfer.len = sizeof(spi_hid_apple_status_ok); + + spi_message_init(&spihid->tx_msg); + spi_message_add_tail(&spihid->tx_transfer, &spihid->tx_msg); + spi_message_add_tail(&spihid->status_transfer, &spihid->tx_msg); +} + +static int spihid_apple_setup_spi(struct spihid_apple *spihid) +{ + spihid_apple_setup_spi_msgs(spihid); + + return spihid->ops->power_on(spihid->ops); +} + +static int spihid_register_hid_device(struct spihid_apple *spihid, + struct spihid_interface *iface, u8 device) +{ + int ret; + char *suffix; + struct hid_device *hid; + + iface->id = device; + + hid = hid_allocate_device(); + if (IS_ERR(hid)) + return PTR_ERR(hid); + + /* + * Use 'Apple SPI Keyboard' and 'Apple SPI Trackpad' as input device + * names. The device names need to be distinct since at least Kwin uses + * the tripple Vendor ID, Product ID, Name to identify devices. + */ + snprintf(hid->name, sizeof(hid->name), "Apple SPI %s", iface->name); + // strip ' / Boot' suffix from the name + suffix = strstr(hid->name, " / Boot"); + if (suffix) + suffix[0] = '\0'; + snprintf(hid->phys, sizeof(hid->phys), "%s (%hhx)", + dev_name(&spihid->spidev->dev), device); + strscpy(hid->uniq, spihid->serial, sizeof(hid->uniq)); + + hid->ll_driver = &apple_hid_ll; + hid->bus = BUS_SPI; + hid->vendor = spihid->vendor_id; + hid->product = spihid->product_id; + hid->version = spihid->version_number; + + if (device == SPIHID_DEVICE_ID_KBD) + hid->type = HID_TYPE_SPI_KEYBOARD; + else if (device == SPIHID_DEVICE_ID_TP) + hid->type = HID_TYPE_SPI_MOUSE; + + hid->country = iface->country; + hid->dev.parent = &spihid->spidev->dev; + hid->driver_data = iface; + + ret = hid_add_device(hid); + if (ret < 0) { + hid_destroy_device(hid); + dev_warn(&spihid->spidev->dev, + "Failed to register hid device %hhu", device); + return ret; + } + + iface->hid = hid; + + return 0; +} + +static void spihid_destroy_hid_device(struct spihid_interface *iface) +{ + if (iface->hid) { + hid_destroy_device(iface->hid); + iface->hid = NULL; + } + iface->ready = false; +} + +int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops) +{ + struct device *dev = &spi->dev; + struct spihid_apple *spihid; + int err, i; + + if (!ops || !ops->power_on || !ops->power_off || !ops->enable_irq || !ops->disable_irq) + return -EINVAL; + + spihid = devm_kzalloc(dev, sizeof(*spihid), GFP_KERNEL); + if (!spihid) + return -ENOMEM; + + spihid->ops = ops; + spihid->spidev = spi; + + // init spi + spi_set_drvdata(spi, spihid); + + /* + * allocate SPI buffers + * Overallocate the receice buffer since it passed directly into + * hid_input_report / hid_report_raw_event. The later expects the buffer + * to be HID_MAX_BUFFER_SIZE (16k) or hid_ll_driver.max_buffer_size if + * set. + */ + spihid->rx_buf = devm_kmalloc( + &spi->dev, SPIHID_MAX_INPUT_REPORT_SIZE, GFP_KERNEL); + spihid->tx_buf = devm_kmalloc( + &spi->dev, sizeof(struct spihid_transfer_packet), GFP_KERNEL); + spihid->status_buf = devm_kmalloc( + &spi->dev, sizeof(spi_hid_apple_status_ok), GFP_KERNEL); + + if (!spihid->rx_buf || !spihid->tx_buf || !spihid->status_buf) + return -ENOMEM; + + spihid->report.buf = + devm_kmalloc(dev, SPIHID_MAX_INPUT_REPORT_SIZE, GFP_KERNEL); + + spihid->kbd.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); + spihid->tp.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); + + if (!spihid->report.buf || !spihid->kbd.hid_desc || + !spihid->tp.hid_desc) + return -ENOMEM; + + init_waitqueue_head(&spihid->wait); + + mutex_init(&spihid->tx_lock); + + /* Init spi transfer buffers and power device on */ + err = spihid_apple_setup_spi(spihid); + if (err < 0) + goto error; + + /* enable HID irq */ + spihid->ops->enable_irq(spihid->ops); + + // wait for boot message + err = wait_event_interruptible_timeout(spihid->wait, + spihid->status_booted, + msecs_to_jiffies(1000)); + if (err == 0) + err = -ENODEV; + if (err < 0) { + dev_err(dev, "waiting for device boot failed: %d", err); + goto error; + } + + /* request device information */ + dev_dbg(dev, "request device info"); + spihid_apple_request(spihid, 0xd0, 0x20, 0x01, 0xd0, 0, NULL, 0); + err = wait_event_interruptible_timeout(spihid->wait, spihid->vendor_id, + SPIHID_DEF_WAIT); + if (err == 0) + err = -ENODEV; + if (err < 0) { + dev_err(dev, "waiting for device info failed: %d", err); + goto error; + } + + /* request interface information */ + for (i = 0; i < spihid->num_devices; i++) { + struct spihid_interface *iface = spihid_get_iface(spihid, i); + if (!iface) + continue; + dev_dbg(dev, "request interface info 0x%02x", i); + spihid_apple_request(spihid, 0xd0, 0x20, 0x02, i, + SPIHID_DESC_MAX, NULL, 0); + err = wait_event_interruptible_timeout( + spihid->wait, iface->max_input_report_len, + SPIHID_DEF_WAIT); + } + + /* request HID report descriptors */ + for (i = 1; i < spihid->num_devices; i++) { + struct spihid_interface *iface = spihid_get_iface(spihid, i); + if (!iface) + continue; + dev_dbg(dev, "request hid report desc 0x%02x", i); + spihid_apple_request(spihid, 0xd0, 0x20, 0x10, i, + SPIHID_DESC_MAX, NULL, 0); + wait_event_interruptible_timeout( + spihid->wait, iface->hid_desc_len, SPIHID_DEF_WAIT); + } + + return 0; +error: + return err; +} +EXPORT_SYMBOL_GPL(spihid_apple_core_probe); + +void spihid_apple_core_remove(struct spi_device *spi) +{ + struct spihid_apple *spihid = spi_get_drvdata(spi); + + /* destroy input devices */ + + spihid_destroy_hid_device(&spihid->tp); + spihid_destroy_hid_device(&spihid->kbd); + + /* disable irq */ + spihid->ops->disable_irq(spihid->ops); + + /* power SPI device down */ + spihid->ops->power_off(spihid->ops); +} +EXPORT_SYMBOL_GPL(spihid_apple_core_remove); + +void spihid_apple_core_shutdown(struct spi_device *spi) +{ + struct spihid_apple *spihid = spi_get_drvdata(spi); + + /* disable irq */ + spihid->ops->disable_irq(spihid->ops); + + /* power SPI device down */ + spihid->ops->power_off(spihid->ops); +} +EXPORT_SYMBOL_GPL(spihid_apple_core_shutdown); + +#ifdef CONFIG_PM_SLEEP +static int spihid_apple_core_suspend(struct device *dev) +{ + int ret; +#ifdef IRQ_WAKE_SUPPORT + int wake_status; +#endif + struct spihid_apple *spihid = spi_get_drvdata(to_spi_device(dev)); + + if (spihid->tp.hid) { + ret = hid_driver_suspend(spihid->tp.hid, PMSG_SUSPEND); + if (ret < 0) + return ret; + } + + if (spihid->kbd.hid) { + ret = hid_driver_suspend(spihid->kbd.hid, PMSG_SUSPEND); + if (ret < 0) { + if (spihid->tp.hid) + hid_driver_resume(spihid->tp.hid); + return ret; + } + } + + /* Save some power */ + spihid->ops->disable_irq(spihid->ops); + +#ifdef IRQ_WAKE_SUPPORT + if (device_may_wakeup(dev)) { + wake_status = spihid->ops->enable_irq_wake(spihid->ops); + if (!wake_status) + spihid->irq_wake_enabled = true; + else + dev_warn(dev, "Failed to enable irq wake: %d\n", + wake_status); + } else { + spihid->ops->power_off(spihid->ops); + } +#else + spihid->ops->power_off(spihid->ops); +#endif + + return 0; +} + +static int spihid_apple_core_resume(struct device *dev) +{ + int ret_tp = 0, ret_kbd = 0; + struct spihid_apple *spihid = spi_get_drvdata(to_spi_device(dev)); +#ifdef IRQ_WAKE_SUPPORT + int wake_status; + + if (!device_may_wakeup(dev)) { + spihid->ops->power_on(spihid->ops); + } else if (spihid->irq_wake_enabled) { + wake_status = spihid->ops->disable_irq_wake(spihid->ops); + if (!wake_status) + spihid->irq_wake_enabled = false; + else + dev_warn(dev, "Failed to disable irq wake: %d\n", + wake_status); + } +#endif + + spihid->ops->enable_irq(spihid->ops); + spihid->ops->power_on(spihid->ops); + + if (spihid->tp.hid) + ret_tp = hid_driver_reset_resume(spihid->tp.hid); + if (spihid->kbd.hid) + ret_kbd = hid_driver_reset_resume(spihid->kbd.hid); + + if (ret_tp < 0) + return ret_tp; + + return ret_kbd; +} +#endif + +const struct dev_pm_ops spihid_apple_core_pm = { + SET_SYSTEM_SLEEP_PM_OPS(spihid_apple_core_suspend, + spihid_apple_core_resume) +}; +EXPORT_SYMBOL_GPL(spihid_apple_core_pm); + +MODULE_DESCRIPTION("Apple SPI HID transport driver"); +MODULE_AUTHOR("Janne Grunau "); +MODULE_LICENSE("GPL"); diff -ruN --no-dereference -x .git a/drivers/hid/spi-hid/spi-hid-apple-of.c b/drivers/hid/spi-hid/spi-hid-apple-of.c --- a/drivers/hid/spi-hid/spi-hid-apple-of.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/hid/spi-hid/spi-hid-apple-of.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,153 @@ +/* + * SPDX-License-Identifier: GPL-2.0 + * + * Apple SPI HID transport driver - Open Firmware + * + * Copyright (C) The Asahi Linux Contributors + */ + +#include +#include +#include +#include + +#include "spi-hid-apple.h" + + +struct spihid_apple_of { + struct spihid_apple_ops ops; + + struct gpio_desc *enable_gpio; + int irq; +}; + +static int spihid_apple_of_power_on(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + /* reset the controller on boot */ + gpiod_direction_output(sh_of->enable_gpio, 1); + msleep(5); + gpiod_direction_output(sh_of->enable_gpio, 0); + msleep(5); + /* turn SPI device on */ + gpiod_direction_output(sh_of->enable_gpio, 1); + msleep(50); + + return 0; +} + +static int spihid_apple_of_power_off(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + /* turn SPI device off */ + gpiod_direction_output(sh_of->enable_gpio, 0); + + return 0; +} + +static int spihid_apple_of_enable_irq(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + enable_irq(sh_of->irq); + + return 0; +} + +static int spihid_apple_of_disable_irq(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + disable_irq(sh_of->irq); + + return 0; +} + +static int spihid_apple_of_enable_irq_wake(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + return enable_irq_wake(sh_of->irq); +} + +static int spihid_apple_of_disable_irq_wake(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + return disable_irq_wake(sh_of->irq); +} + +static int spihid_apple_of_probe(struct spi_device *spi) +{ + struct device *dev = &spi->dev; + struct spihid_apple_of *spihid_of; + int err; + + spihid_of = devm_kzalloc(dev, sizeof(*spihid_of), GFP_KERNEL); + if (!spihid_of) + return -ENOMEM; + + spihid_of->ops.power_on = spihid_apple_of_power_on; + spihid_of->ops.power_off = spihid_apple_of_power_off; + spihid_of->ops.enable_irq = spihid_apple_of_enable_irq; + spihid_of->ops.disable_irq = spihid_apple_of_disable_irq; + spihid_of->ops.enable_irq_wake = spihid_apple_of_enable_irq_wake; + spihid_of->ops.disable_irq_wake = spihid_apple_of_disable_irq_wake; + + spihid_of->enable_gpio = devm_gpiod_get_index(dev, "spien", 0, 0); + if (IS_ERR(spihid_of->enable_gpio)) { + err = PTR_ERR(spihid_of->enable_gpio); + dev_err(dev, "failed to get 'spien' gpio pin: %d", err); + return err; + } + + spihid_of->irq = of_irq_get(dev->of_node, 0); + if (spihid_of->irq < 0) { + err = spihid_of->irq; + dev_err(dev, "failed to get 'extended-irq': %d", err); + return err; + } + err = devm_request_threaded_irq(dev, spihid_of->irq, NULL, + spihid_apple_core_irq, IRQF_ONESHOT | IRQF_NO_AUTOEN, + "spi-hid-apple-irq", spi); + if (err < 0) { + dev_err(dev, "failed to request extended-irq %d: %d", + spihid_of->irq, err); + return err; + } + + return spihid_apple_core_probe(spi, &spihid_of->ops); +} + +static const struct of_device_id spihid_apple_of_match[] = { + { .compatible = "apple,spi-hid-transport" }, + {}, +}; +MODULE_DEVICE_TABLE(of, spihid_apple_of_match); + +static struct spi_device_id spihid_apple_of_id[] = { + { "spi-hid-transport", 0 }, + {} +}; +MODULE_DEVICE_TABLE(spi, spihid_apple_of_id); + +static struct spi_driver spihid_apple_of_driver = { + .driver = { + .name = "spi-hid-apple-of", + .pm = &spihid_apple_core_pm, + .of_match_table = of_match_ptr(spihid_apple_of_match), + }, + + .id_table = spihid_apple_of_id, + .probe = spihid_apple_of_probe, + .remove = spihid_apple_core_remove, + .shutdown = spihid_apple_core_shutdown, +}; + +module_spi_driver(spihid_apple_of_driver); + +MODULE_DESCRIPTION("Apple SPI HID transport driver for OpenFirmware systems"); +MODULE_AUTHOR("Janne Grunau "); +MODULE_LICENSE("GPL"); diff -ruN --no-dereference -x .git a/drivers/hid/spi-hid/spi-hid-apple.h b/drivers/hid/spi-hid/spi-hid-apple.h --- a/drivers/hid/spi-hid/spi-hid-apple.h 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/hid/spi-hid/spi-hid-apple.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ + +#ifndef SPI_HID_APPLE_H +#define SPI_HID_APPLE_H + +#include +#include + +/** + * struct spihid_apple_ops - Ops to control the device from the core driver. + * + * @power_on: reset and power the device on. + * @power_off: power the device off. + * @enable_irq: enable irq or ACPI gpe. + * @disable_irq: disable irq or ACPI gpe. + */ + +struct spihid_apple_ops { + int (*power_on)(struct spihid_apple_ops *ops); + int (*power_off)(struct spihid_apple_ops *ops); + int (*enable_irq)(struct spihid_apple_ops *ops); + int (*disable_irq)(struct spihid_apple_ops *ops); + int (*enable_irq_wake)(struct spihid_apple_ops *ops); + int (*disable_irq_wake)(struct spihid_apple_ops *ops); +}; + +irqreturn_t spihid_apple_core_irq(int irq, void *data); + +int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops); +void spihid_apple_core_remove(struct spi_device *spi); +void spihid_apple_core_shutdown(struct spi_device *spi); + +extern const struct dev_pm_ops spihid_apple_core_pm; + +#endif /* SPI_HID_APPLE_H */ diff -ruN --no-dereference -x .git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig --- a/drivers/hwmon/Kconfig 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/hwmon/Kconfig 2026-10-13 14:33:30.000000000 +0200 @@ -388,6 +388,16 @@ Say Y here if you have an applicable laptop and want to experience the awesome power of applesmc. +config SENSORS_APPLESMC_T2 + tristate "Apple SMC for T2 Macbooks" + depends on INPUT && X86 + select NEW_LEDS + select LEDS_CLASS + help + Duplicate of the applesmc driver patched of T2 Macbooks. A separate + driver is provided instead of patching the available one because of + incompatibilies with non-T2 Macbooks. + config SENSORS_ARCTIC_FAN_CONTROLLER tristate "ARCTIC Fan Controller" depends on USB_HID diff -ruN --no-dereference -x .git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile --- a/drivers/hwmon/Makefile 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/hwmon/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -48,6 +48,7 @@ obj-$(CONFIG_SENSORS_ADT7475) += adt7475.o obj-$(CONFIG_SENSORS_AHT10) += aht10.o obj-$(CONFIG_SENSORS_APPLESMC) += applesmc.o +obj-$(CONFIG_SENSORS_APPLESMC_T2) += applesmc-t2.o obj-$(CONFIG_SENSORS_AQUACOMPUTER_D5NEXT) += aquacomputer_d5next.o obj-$(CONFIG_SENSORS_ARCTIC_FAN_CONTROLLER) += arctic_fan_controller.o obj-$(CONFIG_SENSORS_ARM_SCMI) += scmi-hwmon.o diff -ruN --no-dereference -x .git a/drivers/hwmon/applesmc-t2.c b/drivers/hwmon/applesmc-t2.c --- a/drivers/hwmon/applesmc-t2.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/hwmon/applesmc-t2.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,1917 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * drivers/hwmon/applesmc.c - driver for Apple's SMC (accelerometer, temperature + * sensors, fan control, keyboard backlight control) used in Intel-based Apple + * computers. + * + * Copyright (C) 2007 Nicolas Boichat + * Copyright (C) 2010 Henrik Rydberg + * Copyright (C) 2019 Paul Pawlowski + * + * Based on hdaps.c driver: + * Copyright (C) 2005 Robert Love + * Copyright (C) 2005 Jesper Juhl + * + * Fan control based on smcFanControl: + * Copyright (C) 2006 Hendrik Holtmann + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* data port used by Apple SMC */ +#define APPLESMC_DATA_PORT 0 +/* command/status port used by Apple SMC */ +#define APPLESMC_CMD_PORT 4 + +#define APPLESMC_NR_PORTS 32 /* 0x300-0x31f */ + +#define APPLESMC_IOMEM_KEY_DATA 0 +#define APPLESMC_IOMEM_KEY_STATUS 0x4005 +#define APPLESMC_IOMEM_KEY_NAME 0x78 +#define APPLESMC_IOMEM_KEY_DATA_LEN 0x7D +#define APPLESMC_IOMEM_KEY_SMC_ID 0x7E +#define APPLESMC_IOMEM_KEY_CMD 0x7F +#define APPLESMC_IOMEM_MIN_SIZE 0x4006 + +#define APPLESMC_IOMEM_KEY_TYPE_CODE 0 +#define APPLESMC_IOMEM_KEY_TYPE_DATA_LEN 5 +#define APPLESMC_IOMEM_KEY_TYPE_FLAGS 6 + +#define APPLESMC_MAX_DATA_LENGTH 32 + +/* Apple SMC status bits */ +#define SMC_STATUS_AWAITING_DATA BIT(0) /* SMC has data waiting to be read */ +#define SMC_STATUS_IB_CLOSED BIT(1) /* Will ignore any input */ +#define SMC_STATUS_BUSY BIT(2) /* Command in progress */ + +/* Initial wait is 8us */ +#define APPLESMC_MIN_WAIT 0x0008 + +#define APPLESMC_READ_CMD 0x10 +#define APPLESMC_WRITE_CMD 0x11 +#define APPLESMC_GET_KEY_BY_INDEX_CMD 0x12 +#define APPLESMC_GET_KEY_TYPE_CMD 0x13 + +#define KEY_COUNT_KEY "#KEY" /* r-o ui32 */ + +#define LIGHT_SENSOR_LEFT_KEY "ALV0" /* r-o {alv (6-10 bytes) */ +#define LIGHT_SENSOR_RIGHT_KEY "ALV1" /* r-o {alv (6-10 bytes) */ +#define BACKLIGHT_KEY "LKSB" /* w-o {lkb (2 bytes) */ + +#define CLAMSHELL_KEY "MSLD" /* r-o ui8 (unused) */ + +#define MOTION_SENSOR_X_KEY "MO_X" /* r-o sp78 (2 bytes) */ +#define MOTION_SENSOR_Y_KEY "MO_Y" /* r-o sp78 (2 bytes) */ +#define MOTION_SENSOR_Z_KEY "MO_Z" /* r-o sp78 (2 bytes) */ +#define MOTION_SENSOR_KEY "MOCN" /* r/w ui16 */ + +#define FANS_COUNT "FNum" /* r-o ui8 */ +#define FANS_MANUAL "FS! " /* r-w ui16 */ +#define FAN_ID_FMT "F%dID" /* r-o char[16] */ + +#define TEMP_SENSOR_TYPE "sp78" +#define FLOAT_TYPE "flt " + +/* List of keys used to read/write fan speeds */ +static const char *const fan_speed_fmt[] = { + "F%dAc", /* actual speed */ + "F%dMn", /* minimum speed (rw) */ + "F%dMx", /* maximum speed */ + "F%dSf", /* safe speed - not all models */ + "F%dTg", /* target speed (manual: rw) */ +}; +#define FAN_MANUAL_FMT "F%dMd" + +#define INIT_TIMEOUT_MSECS 5000 /* wait up to 5s for device init ... */ +#define INIT_WAIT_MSECS 50 /* ... in 50ms increments */ + +#define APPLESMC_POLL_INTERVAL 50 /* msecs */ +#define APPLESMC_INPUT_FUZZ 4 /* input event threshold */ +#define APPLESMC_INPUT_FLAT 4 + +#define to_index(attr) (to_sensor_dev_attr(attr)->index & 0xffff) +#define to_option(attr) (to_sensor_dev_attr(attr)->index >> 16) + +/* Dynamic device node attributes */ +struct applesmc_dev_attr { + struct sensor_device_attribute sda; /* hwmon attributes */ + char name[32]; /* room for node file name */ +}; + +/* Dynamic device node group */ +struct applesmc_node_group { + char *format; /* format string */ + void *show; /* show function */ + void *store; /* store function */ + int option; /* function argument */ + struct applesmc_dev_attr *nodes; /* dynamic node array */ +}; + +/* AppleSMC entry - cached register information */ +struct applesmc_entry { + char key[5]; /* four-letter key code */ + u8 valid; /* set when entry is successfully read once */ + u8 len; /* bounded by APPLESMC_MAX_DATA_LENGTH */ + char type[5]; /* four-letter type code */ + u8 flags; /* 0x10: func; 0x40: write; 0x80: read */ +}; + +/* Register lookup and registers common to all SMCs */ +struct applesmc_registers { + struct mutex mutex; /* register read/write mutex */ + unsigned int key_count; /* number of SMC registers */ + unsigned int fan_count; /* number of fans */ + unsigned int temp_count; /* number of temperature registers */ + unsigned int temp_begin; /* temperature lower index bound */ + unsigned int temp_end; /* temperature upper index bound */ + unsigned int index_count; /* size of temperature index array */ + int num_light_sensors; /* number of light sensors */ + bool has_accelerometer; /* has motion sensor */ + bool has_key_backlight; /* has keyboard backlight */ + bool init_complete; /* true when fully initialized */ + struct applesmc_entry *cache; /* cached key entries */ + const char **index; /* temperature key index */ +}; + +struct applesmc_device { + struct acpi_device *dev; + struct device *ldev; + struct applesmc_registers reg; + + bool port_base_set, iomem_base_set; + u16 port_base; + u8 *__iomem iomem_base; + u32 iomem_base_addr, iomem_base_size; + + s16 rest_x; + s16 rest_y; + + u8 backlight_state[2]; + + struct device *hwmon_dev; + struct input_dev *idev; + + /* + * Last index written to key_at_index sysfs file, and value to use for all other + * key_at_index_* sysfs files. + */ + unsigned int key_at_index; + + struct workqueue_struct *backlight_wq; + struct work_struct backlight_work; + struct led_classdev backlight_dev; +}; + +static const int debug; + +/* + * Wait for specific status bits with a mask on the SMC. + * Used before all transactions. + * This does 10 fast loops of 8us then exponentially backs off for a + * minimum total wait of 262ms. Depending on usleep_range this could + * run out past 500ms. + */ + +static int port_wait_status(struct applesmc_device *smc, u8 val, u8 mask) +{ + u8 status; + int us; + int i; + + us = APPLESMC_MIN_WAIT; + for (i = 0; i < 24 ; i++) { + status = inb(smc->port_base + APPLESMC_CMD_PORT); + if ((status & mask) == val) + return 0; + usleep_range(us, us * 2); + if (i > 9) + us <<= 1; + } + return -EIO; +} + +/* port_send_byte - Write to SMC data port. Callers must hold applesmc_lock. */ + +static int port_send_byte(struct applesmc_device *smc, u8 cmd, u16 port) +{ + int status; + + status = port_wait_status(smc, 0, SMC_STATUS_IB_CLOSED); + if (status) + return status; + /* + * This needs to be a separate read looking for bit 0x04 + * after bit 0x02 falls. If consolidated with the wait above + * this extra read may not happen if status returns both + * simultaneously and this would appear to be required. + */ + status = port_wait_status(smc, SMC_STATUS_BUSY, SMC_STATUS_BUSY); + if (status) + return status; + + outb(cmd, smc->port_base + port); + return 0; +} + +/* port_send_command - Write a command to the SMC. Callers must hold applesmc_lock. */ + +static int port_send_command(struct applesmc_device *smc, u8 cmd) +{ + int ret; + + ret = port_wait_status(smc, 0, SMC_STATUS_IB_CLOSED); + if (ret) + return ret; + + outb(cmd, smc->port_base + APPLESMC_CMD_PORT); + return 0; +} + +/* + * Based on logic from the Apple driver. This is issued before any interaction + * If busy is stuck high, issue a read command to reset the SMC state machine. + * If busy is stuck high after the command then the SMC is jammed. + */ + +static int port_smc_sane(struct applesmc_device *smc) +{ + int ret; + + ret = port_wait_status(smc, 0, SMC_STATUS_BUSY); + if (!ret) + return ret; + ret = port_send_command(smc, APPLESMC_READ_CMD); + if (ret) + return ret; + return port_wait_status(smc, 0, SMC_STATUS_BUSY); +} + +static int port_send_argument(struct applesmc_device *smc, const char *key) +{ + int i; + + for (i = 0; i < 4; i++) + if (port_send_byte(smc, key[i], APPLESMC_DATA_PORT)) + return -EIO; + return 0; +} + +static int port_read_smc(struct applesmc_device *smc, u8 cmd, const char *key, + u8 *buffer, u8 len) +{ + u8 status, data = 0; + int i; + int ret; + + ret = port_smc_sane(smc); + if (ret) + return ret; + + if (port_send_command(smc, cmd) || port_send_argument(smc, key)) { + pr_warn("%.4s: read arg fail\n", key); + return -EIO; + } + + /* This has no effect on newer (2012) SMCs */ + if (port_send_byte(smc, len, APPLESMC_DATA_PORT)) { + pr_warn("%.4s: read len fail\n", key); + return -EIO; + } + + for (i = 0; i < len; i++) { + if (port_wait_status(smc, + SMC_STATUS_AWAITING_DATA | SMC_STATUS_BUSY, + SMC_STATUS_AWAITING_DATA | SMC_STATUS_BUSY)) { + pr_warn("%.4s: read data[%d] fail\n", key, i); + return -EIO; + } + buffer[i] = inb(smc->port_base + APPLESMC_DATA_PORT); + } + + /* Read the data port until bit0 is cleared */ + for (i = 0; i < 16; i++) { + udelay(APPLESMC_MIN_WAIT); + status = inb(smc->port_base + APPLESMC_CMD_PORT); + if (!(status & SMC_STATUS_AWAITING_DATA)) + break; + data = inb(smc->port_base + APPLESMC_DATA_PORT); + } + if (i) + pr_warn("flushed %d bytes, last value is: %d\n", i, data); + + return port_wait_status(smc, 0, SMC_STATUS_BUSY); +} + +static int port_write_smc(struct applesmc_device *smc, u8 cmd, const char *key, + const u8 *buffer, u8 len) +{ + int i; + int ret; + + ret = port_smc_sane(smc); + if (ret) + return ret; + + if (port_send_command(smc, cmd) || port_send_argument(smc, key)) { + pr_warn("%s: write arg fail\n", key); + return -EIO; + } + + if (port_send_byte(smc, len, APPLESMC_DATA_PORT)) { + pr_warn("%.4s: write len fail\n", key); + return -EIO; + } + + for (i = 0; i < len; i++) { + if (port_send_byte(smc, buffer[i], APPLESMC_DATA_PORT)) { + pr_warn("%s: write data fail\n", key); + return -EIO; + } + } + + return port_wait_status(smc, 0, SMC_STATUS_BUSY); +} + +static int port_get_smc_key_info(struct applesmc_device *smc, + const char *key, struct applesmc_entry *info) +{ + int ret; + u8 raw[6]; + + ret = port_read_smc(smc, APPLESMC_GET_KEY_TYPE_CMD, key, raw, 6); + if (ret) + return ret; + info->len = raw[0]; + memcpy(info->type, &raw[1], 4); + info->flags = raw[5]; + return 0; +} + + +/* + * MMIO based communication. + * TODO: Use updated mechanism for cmd timeout/retry + */ + +static void iomem_clear_status(struct applesmc_device *smc) +{ + if (ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_STATUS)) + iowrite8(0, smc->iomem_base + APPLESMC_IOMEM_KEY_STATUS); +} + +static int iomem_wait_read(struct applesmc_device *smc) +{ + u8 status; + int us; + int i; + + us = APPLESMC_MIN_WAIT; + for (i = 0; i < 24 ; i++) { + status = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_STATUS); + if (status & 0x20) + return 0; + usleep_range(us, us * 2); + if (i > 9) + us <<= 1; + } + + dev_warn(smc->ldev, "%s... timeout\n", __func__); + return -EIO; +} + +static int iomem_read_smc(struct applesmc_device *smc, u8 cmd, const char *key, + u8 *buffer, u8 len) +{ + u8 err, remote_len; + u32 key_int = *((u32 *) key); + + iomem_clear_status(smc); + iowrite32(key_int, smc->iomem_base + APPLESMC_IOMEM_KEY_NAME); + iowrite8(0, smc->iomem_base + APPLESMC_IOMEM_KEY_SMC_ID); + iowrite8(cmd, smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + + if (iomem_wait_read(smc)) + return -EIO; + + err = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + if (err != 0) { + dev_warn(smc->ldev, "read_smc_mmio(%x %8x/%.4s) failed: %u\n", + cmd, key_int, key, err); + return -EIO; + } + + if (cmd == APPLESMC_READ_CMD) { + remote_len = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_DATA_LEN); + if (remote_len != len) { + dev_warn(smc->ldev, + "read_smc_mmio(%x %8x/%.4s) failed: buffer length mismatch (remote = %u, requested = %u)\n", + cmd, key_int, key, remote_len, len); + return -EINVAL; + } + } else { + remote_len = len; + } + + memcpy_fromio(buffer, smc->iomem_base + APPLESMC_IOMEM_KEY_DATA, + remote_len); + + dev_dbg(smc->ldev, "read_smc_mmio(%x %8x/%.4s): buflen=%u reslen=%u\n", + cmd, key_int, key, len, remote_len); + print_hex_dump_bytes("read_smc_mmio(): ", DUMP_PREFIX_NONE, buffer, remote_len); + return 0; +} + +static int iomem_get_smc_key_type(struct applesmc_device *smc, const char *key, + struct applesmc_entry *e) +{ + u8 err; + u8 cmd = APPLESMC_GET_KEY_TYPE_CMD; + u32 key_int = *((u32 *) key); + + iomem_clear_status(smc); + iowrite32(key_int, smc->iomem_base + APPLESMC_IOMEM_KEY_NAME); + iowrite8(0, smc->iomem_base + APPLESMC_IOMEM_KEY_SMC_ID); + iowrite8(cmd, smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + + if (iomem_wait_read(smc)) + return -EIO; + + err = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + if (err != 0) { + dev_warn(smc->ldev, "get_smc_key_type_mmio(%.4s) failed: %u\n", key, err); + return -EIO; + } + + e->len = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_TYPE_DATA_LEN); + *((uint32_t *) e->type) = ioread32( + smc->iomem_base + APPLESMC_IOMEM_KEY_TYPE_CODE); + e->flags = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_TYPE_FLAGS); + + dev_dbg(smc->ldev, "get_smc_key_type_mmio(%.4s): len=%u type=%.4s flags=%x\n", + key, e->len, e->type, e->flags); + return 0; +} + +static int iomem_write_smc(struct applesmc_device *smc, u8 cmd, const char *key, + const u8 *buffer, u8 len) +{ + u8 err; + u32 key_int = *((u32 *) key); + + iomem_clear_status(smc); + iowrite32(key_int, smc->iomem_base + APPLESMC_IOMEM_KEY_NAME); + memcpy_toio(smc->iomem_base + APPLESMC_IOMEM_KEY_DATA, buffer, len); + iowrite8(len, smc->iomem_base + APPLESMC_IOMEM_KEY_DATA_LEN); + iowrite8(0, smc->iomem_base + APPLESMC_IOMEM_KEY_SMC_ID); + iowrite8(cmd, smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + + if (iomem_wait_read(smc)) + return -EIO; + + err = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + if (err != 0) { + dev_warn(smc->ldev, "write_smc_mmio(%x %.4s) failed: %u\n", cmd, key, err); + print_hex_dump_bytes("write_smc_mmio(): ", DUMP_PREFIX_NONE, buffer, len); + return -EIO; + } + + dev_dbg(smc->ldev, "write_smc_mmio(%x %.4s): buflen=%u\n", cmd, key, len); + print_hex_dump_bytes("write_smc_mmio(): ", DUMP_PREFIX_NONE, buffer, len); + return 0; +} + + +static int read_smc(struct applesmc_device *smc, const char *key, + u8 *buffer, u8 len) +{ + if (smc->iomem_base_set) + return iomem_read_smc(smc, APPLESMC_READ_CMD, key, buffer, len); + else + return port_read_smc(smc, APPLESMC_READ_CMD, key, buffer, len); +} + +static int write_smc(struct applesmc_device *smc, const char *key, + const u8 *buffer, u8 len) +{ + if (smc->iomem_base_set) + return iomem_write_smc(smc, APPLESMC_WRITE_CMD, key, buffer, len); + else + return port_write_smc(smc, APPLESMC_WRITE_CMD, key, buffer, len); +} + +static int get_smc_key_by_index(struct applesmc_device *smc, + unsigned int index, char *key) +{ + __be32 be; + + be = cpu_to_be32(index); + if (smc->iomem_base_set) + return iomem_read_smc(smc, APPLESMC_GET_KEY_BY_INDEX_CMD, + (const char *) &be, (u8 *) key, 4); + else + return port_read_smc(smc, APPLESMC_GET_KEY_BY_INDEX_CMD, + (const char *) &be, (u8 *) key, 4); +} + +static int get_smc_key_info(struct applesmc_device *smc, const char *key, + struct applesmc_entry *info) +{ + if (smc->iomem_base_set) + return iomem_get_smc_key_type(smc, key, info); + else + return port_get_smc_key_info(smc, key, info); +} + +static int read_register_count(struct applesmc_device *smc, + unsigned int *count) +{ + __be32 be; + int ret; + + ret = read_smc(smc, KEY_COUNT_KEY, (u8 *)&be, 4); + if (ret < 0) + return ret; + + *count = be32_to_cpu(be); + return 0; +} + +/* + * Serialized I/O + * + * Returns zero on success or a negative error on failure. + * All functions below are concurrency safe - callers should NOT hold lock. + */ + +static int applesmc_read_entry(struct applesmc_device *smc, + const struct applesmc_entry *entry, u8 *buf, u8 len) +{ + int ret; + + if (entry->len != len) + return -EINVAL; + mutex_lock(&smc->reg.mutex); + ret = read_smc(smc, entry->key, buf, len); + mutex_unlock(&smc->reg.mutex); + + return ret; +} + +static int applesmc_write_entry(struct applesmc_device *smc, + const struct applesmc_entry *entry, const u8 *buf, u8 len) +{ + int ret; + + if (entry->len != len) + return -EINVAL; + mutex_lock(&smc->reg.mutex); + ret = write_smc(smc, entry->key, buf, len); + mutex_unlock(&smc->reg.mutex); + return ret; +} + +static const struct applesmc_entry *applesmc_get_entry_by_index( + struct applesmc_device *smc, int index) +{ + struct applesmc_entry *cache = &smc->reg.cache[index]; + char key[4]; + int ret = 0; + + if (cache->valid) + return cache; + + mutex_lock(&smc->reg.mutex); + + if (cache->valid) + goto out; + ret = get_smc_key_by_index(smc, index, key); + if (ret) + goto out; + memcpy(cache->key, key, 4); + + ret = get_smc_key_info(smc, key, cache); + if (ret) + goto out; + cache->valid = true; + +out: + mutex_unlock(&smc->reg.mutex); + if (ret) + return ERR_PTR(ret); + return cache; +} + +static int applesmc_get_lower_bound(struct applesmc_device *smc, + unsigned int *lo, const char *key) +{ + int begin = 0, end = smc->reg.key_count; + const struct applesmc_entry *entry; + + while (begin != end) { + int middle = begin + (end - begin) / 2; + entry = applesmc_get_entry_by_index(smc, middle); + if (IS_ERR(entry)) { + *lo = 0; + return PTR_ERR(entry); + } + if (strcmp(entry->key, key) < 0) + begin = middle + 1; + else + end = middle; + } + + *lo = begin; + return 0; +} + +static int applesmc_get_upper_bound(struct applesmc_device *smc, + unsigned int *hi, const char *key) +{ + int begin = 0, end = smc->reg.key_count; + const struct applesmc_entry *entry; + + while (begin != end) { + int middle = begin + (end - begin) / 2; + entry = applesmc_get_entry_by_index(smc, middle); + if (IS_ERR(entry)) { + *hi = smc->reg.key_count; + return PTR_ERR(entry); + } + if (strcmp(key, entry->key) < 0) + end = middle; + else + begin = middle + 1; + } + + *hi = begin; + return 0; +} + +static const struct applesmc_entry *applesmc_get_entry_by_key( + struct applesmc_device *smc, const char *key) +{ + int begin, end; + int ret; + + ret = applesmc_get_lower_bound(smc, &begin, key); + if (ret) + return ERR_PTR(ret); + ret = applesmc_get_upper_bound(smc, &end, key); + if (ret) + return ERR_PTR(ret); + if (end - begin != 1) + return ERR_PTR(-EINVAL); + + return applesmc_get_entry_by_index(smc, begin); +} + +static int applesmc_read_key(struct applesmc_device *smc, + const char *key, u8 *buffer, u8 len) +{ + const struct applesmc_entry *entry; + + entry = applesmc_get_entry_by_key(smc, key); + if (IS_ERR(entry)) + return PTR_ERR(entry); + + return applesmc_read_entry(smc, entry, buffer, len); +} + +static int applesmc_write_key(struct applesmc_device *smc, + const char *key, const u8 *buffer, u8 len) +{ + const struct applesmc_entry *entry; + + entry = applesmc_get_entry_by_key(smc, key); + if (IS_ERR(entry)) + return PTR_ERR(entry); + + return applesmc_write_entry(smc, entry, buffer, len); +} + +static int applesmc_has_key(struct applesmc_device *smc, + const char *key, bool *value) +{ + const struct applesmc_entry *entry; + + entry = applesmc_get_entry_by_key(smc, key); + if (IS_ERR(entry) && PTR_ERR(entry) != -EINVAL) + return PTR_ERR(entry); + + *value = !IS_ERR(entry); + return 0; +} + +/* + * applesmc_read_s16 - Read 16-bit signed big endian register + */ +static int applesmc_read_s16(struct applesmc_device *smc, + const char *key, s16 *value) +{ + u8 buffer[2]; + int ret; + + ret = applesmc_read_key(smc, key, buffer, 2); + if (ret) + return ret; + + *value = ((s16)buffer[0] << 8) | buffer[1]; + return 0; +} + +/** + * applesmc_float_to_u32 - Retrieve the integral part of a float. + * This is needed because Apple made fans use float values in the T2. + * The fractional point is not significantly useful though, and the integral + * part can be easily extracted. + */ +static inline u32 applesmc_float_to_u32(u32 d) +{ + u8 sign = (u8) ((d >> 31) & 1); + s32 exp = (s32) ((d >> 23) & 0xff) - 0x7f; + u32 fr = d & ((1u << 23) - 1); + + if (sign || exp < 0) + return 0; + + return (u32) ((1u << exp) + (fr >> (23 - exp))); +} + +/** + * applesmc_u32_to_float - Convert an u32 into a float. + * See applesmc_float_to_u32 for a rationale. + */ +static inline u32 applesmc_u32_to_float(u32 d) +{ + u32 dc = d, bc = 0, exp; + + if (!d) + return 0; + + while (dc >>= 1) + ++bc; + exp = 0x7f + bc; + + return (u32) ((exp << 23) | + ((d << (23 - (exp - 0x7f))) & ((1u << 23) - 1))); +} +/* + * applesmc_device_init - initialize the accelerometer. Can sleep. + */ +static void applesmc_device_init(struct applesmc_device *smc) +{ + int total; + u8 buffer[2]; + + if (!smc->reg.has_accelerometer) + return; + + for (total = INIT_TIMEOUT_MSECS; total > 0; total -= INIT_WAIT_MSECS) { + if (!applesmc_read_key(smc, MOTION_SENSOR_KEY, buffer, 2) && + (buffer[0] != 0x00 || buffer[1] != 0x00)) + return; + buffer[0] = 0xe0; + buffer[1] = 0x00; + applesmc_write_key(smc, MOTION_SENSOR_KEY, buffer, 2); + msleep(INIT_WAIT_MSECS); + } + + pr_warn("failed to init the device\n"); +} + +static int applesmc_init_index(struct applesmc_device *smc, + struct applesmc_registers *s) +{ + const struct applesmc_entry *entry; + unsigned int i; + + if (s->index) + return 0; + + s->index = kcalloc(s->temp_count, sizeof(s->index[0]), GFP_KERNEL); + if (!s->index) + return -ENOMEM; + + for (i = s->temp_begin; i < s->temp_end; i++) { + entry = applesmc_get_entry_by_index(smc, i); + if (IS_ERR(entry)) + continue; + if (strcmp(entry->type, TEMP_SENSOR_TYPE)) + continue; + s->index[s->index_count++] = entry->key; + } + + return 0; +} + +/* + * applesmc_init_smcreg_try - Try to initialize register cache. Idempotent. + */ +static int applesmc_init_smcreg_try(struct applesmc_device *smc) +{ + struct applesmc_registers *s = &smc->reg; + bool left_light_sensor = false, right_light_sensor = false; + unsigned int count; + u8 tmp[1]; + int ret; + + if (s->init_complete) + return 0; + + ret = read_register_count(smc, &count); + if (ret) + return ret; + + if (s->cache && s->key_count != count) { + pr_warn("key count changed from %d to %d\n", + s->key_count, count); + kfree(s->cache); + s->cache = NULL; + } + s->key_count = count; + + if (!s->cache) + s->cache = kzalloc_objs(*s->cache, s->key_count); + if (!s->cache) + return -ENOMEM; + + ret = applesmc_read_key(smc, FANS_COUNT, tmp, 1); + if (ret) + return ret; + s->fan_count = tmp[0]; + if (s->fan_count > 10) + s->fan_count = 10; + + ret = applesmc_get_lower_bound(smc, &s->temp_begin, "T"); + if (ret) + return ret; + ret = applesmc_get_lower_bound(smc, &s->temp_end, "U"); + if (ret) + return ret; + s->temp_count = s->temp_end - s->temp_begin; + + ret = applesmc_init_index(smc, s); + if (ret) + return ret; + + ret = applesmc_has_key(smc, LIGHT_SENSOR_LEFT_KEY, &left_light_sensor); + if (ret) + return ret; + ret = applesmc_has_key(smc, LIGHT_SENSOR_RIGHT_KEY, &right_light_sensor); + if (ret) + return ret; + ret = applesmc_has_key(smc, MOTION_SENSOR_KEY, &s->has_accelerometer); + if (ret) + return ret; + ret = applesmc_has_key(smc, BACKLIGHT_KEY, &s->has_key_backlight); + if (ret) + return ret; + + s->num_light_sensors = left_light_sensor + right_light_sensor; + s->init_complete = true; + + pr_info("key=%d fan=%d temp=%d index=%d acc=%d lux=%d kbd=%d\n", + s->key_count, s->fan_count, s->temp_count, s->index_count, + s->has_accelerometer, + s->num_light_sensors, + s->has_key_backlight); + + return 0; +} + +static void applesmc_destroy_smcreg(struct applesmc_device *smc) +{ + kfree(smc->reg.index); + smc->reg.index = NULL; + kfree(smc->reg.cache); + smc->reg.cache = NULL; + smc->reg.init_complete = false; +} + +/* + * applesmc_init_smcreg - Initialize register cache. + * + * Retries until initialization is successful, or the operation times out. + * + */ +static int applesmc_init_smcreg(struct applesmc_device *smc) +{ + int ms, ret; + + for (ms = 0; ms < INIT_TIMEOUT_MSECS; ms += INIT_WAIT_MSECS) { + ret = applesmc_init_smcreg_try(smc); + if (!ret) { + if (ms) + pr_info("init_smcreg() took %d ms\n", ms); + return 0; + } + msleep(INIT_WAIT_MSECS); + } + + applesmc_destroy_smcreg(smc); + + return ret; +} + +/* Device model stuff */ + +static int applesmc_init_resources(struct applesmc_device *smc); +static void applesmc_free_resources(struct applesmc_device *smc); +static int applesmc_create_modules(struct applesmc_device *smc); +static void applesmc_destroy_modules(struct applesmc_device *smc); + +static int applesmc_add(struct acpi_device *dev) +{ + struct applesmc_device *smc; + int ret; + + smc = kzalloc(sizeof(struct applesmc_device), GFP_KERNEL); + if (!smc) + return -ENOMEM; + smc->dev = dev; + smc->ldev = &dev->dev; + mutex_init(&smc->reg.mutex); + + dev_set_drvdata(&dev->dev, smc); + + ret = applesmc_init_resources(smc); + if (ret) + goto out_mem; + + ret = applesmc_init_smcreg(smc); + if (ret) + goto out_res; + + applesmc_device_init(smc); + + ret = applesmc_create_modules(smc); + if (ret) + goto out_reg; + + return 0; + +out_reg: + applesmc_destroy_smcreg(smc); +out_res: + applesmc_free_resources(smc); +out_mem: + dev_set_drvdata(&dev->dev, NULL); + mutex_destroy(&smc->reg.mutex); + kfree(smc); + + return ret; +} + +static void applesmc_remove(struct acpi_device *dev) +{ + struct applesmc_device *smc = dev_get_drvdata(&dev->dev); + + applesmc_destroy_modules(smc); + applesmc_destroy_smcreg(smc); + applesmc_free_resources(smc); + + mutex_destroy(&smc->reg.mutex); + kfree(smc); + + return; +} + +static acpi_status applesmc_walk_resources(struct acpi_resource *res, + void *data) +{ + struct applesmc_device *smc = data; + + switch (res->type) { + case ACPI_RESOURCE_TYPE_IO: + if (!smc->port_base_set) { + if (res->data.io.address_length < APPLESMC_NR_PORTS) + return AE_ERROR; + smc->port_base = res->data.io.minimum; + smc->port_base_set = true; + } + return AE_OK; + + case ACPI_RESOURCE_TYPE_FIXED_MEMORY32: + if (!smc->iomem_base_set) { + if (res->data.fixed_memory32.address_length < + APPLESMC_IOMEM_MIN_SIZE) { + dev_warn(smc->ldev, "found iomem but it's too small: %u\n", + res->data.fixed_memory32.address_length); + return AE_OK; + } + smc->iomem_base_addr = res->data.fixed_memory32.address; + smc->iomem_base_size = res->data.fixed_memory32.address_length; + smc->iomem_base_set = true; + } + return AE_OK; + + case ACPI_RESOURCE_TYPE_END_TAG: + if (smc->port_base_set) + return AE_OK; + else + return AE_NOT_FOUND; + + default: + return AE_OK; + } +} + +static int applesmc_try_enable_iomem(struct applesmc_device *smc); + +static int applesmc_init_resources(struct applesmc_device *smc) +{ + int ret; + + ret = acpi_walk_resources(smc->dev->handle, METHOD_NAME__CRS, + applesmc_walk_resources, smc); + if (ACPI_FAILURE(ret)) + return -ENXIO; + + if (!request_region(smc->port_base, APPLESMC_NR_PORTS, "applesmc")) + return -ENXIO; + + if (smc->iomem_base_set) { + if (applesmc_try_enable_iomem(smc)) + smc->iomem_base_set = false; + } + + return 0; +} + +static int applesmc_try_enable_iomem(struct applesmc_device *smc) +{ + u8 test_val, ldkn_version; + + dev_dbg(smc->ldev, "Trying to enable iomem based communication\n"); + smc->iomem_base = ioremap(smc->iomem_base_addr, smc->iomem_base_size); + if (!smc->iomem_base) + goto out; + + /* Apple's driver does this check for some reason */ + test_val = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_STATUS); + if (test_val == 0xff) { + dev_warn(smc->ldev, + "iomem enable failed: initial status is 0xff (is %x)\n", + test_val); + goto out_iomem; + } + + if (read_smc(smc, "LDKN", &ldkn_version, 1)) { + dev_warn(smc->ldev, "iomem enable failed: ldkn read failed\n"); + goto out_iomem; + } + + if (ldkn_version < 2) { + dev_warn(smc->ldev, + "iomem enable failed: ldkn version %u is less than minimum (2)\n", + ldkn_version); + goto out_iomem; + } + + return 0; + +out_iomem: + iounmap(smc->iomem_base); + +out: + return -ENXIO; +} + +static void applesmc_free_resources(struct applesmc_device *smc) +{ + if (smc->iomem_base_set) + iounmap(smc->iomem_base); + release_region(smc->port_base, APPLESMC_NR_PORTS); +} + +/* Synchronize device with memorized backlight state */ +static int applesmc_pm_resume(struct device *dev) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + + if (smc->reg.has_key_backlight) + applesmc_write_key(smc, BACKLIGHT_KEY, smc->backlight_state, 2); + + return 0; +} + +/* Reinitialize device on resume from hibernation */ +static int applesmc_pm_restore(struct device *dev) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + + applesmc_device_init(smc); + + return applesmc_pm_resume(dev); +} + +static const struct acpi_device_id applesmc_ids[] = { + {"APP0001", 0}, + {"", 0}, +}; + +static const struct dev_pm_ops applesmc_pm_ops = { + .resume = applesmc_pm_resume, + .restore = applesmc_pm_restore, +}; + +static struct acpi_driver applesmc_driver = { + .name = "applesmc-t2", + .class = "applesmc", + .ids = applesmc_ids, + .ops = { + .add = applesmc_add, + .remove = applesmc_remove + }, + .drv = { + .pm = &applesmc_pm_ops + }, +}; + +/* + * applesmc_calibrate - Set our "resting" values. Callers must + * hold applesmc_lock. + */ +static void applesmc_calibrate(struct applesmc_device *smc) +{ + applesmc_read_s16(smc, MOTION_SENSOR_X_KEY, &smc->rest_x); + applesmc_read_s16(smc, MOTION_SENSOR_Y_KEY, &smc->rest_y); + smc->rest_x = -smc->rest_x; +} + +static void applesmc_idev_poll(struct input_dev *idev) +{ + struct applesmc_device *smc = dev_get_drvdata(&idev->dev); + s16 x, y; + + if (applesmc_read_s16(smc, MOTION_SENSOR_X_KEY, &x)) + return; + if (applesmc_read_s16(smc, MOTION_SENSOR_Y_KEY, &y)) + return; + + x = -x; + input_report_abs(idev, ABS_X, x - smc->rest_x); + input_report_abs(idev, ABS_Y, y - smc->rest_y); + input_sync(idev); +} + +/* Sysfs Files */ + +static ssize_t applesmc_name_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "applesmc\n"); +} + +static ssize_t applesmc_position_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + int ret; + s16 x, y, z; + + ret = applesmc_read_s16(smc, MOTION_SENSOR_X_KEY, &x); + if (ret) + goto out; + ret = applesmc_read_s16(smc, MOTION_SENSOR_Y_KEY, &y); + if (ret) + goto out; + ret = applesmc_read_s16(smc, MOTION_SENSOR_Z_KEY, &z); + if (ret) + goto out; + +out: + if (ret) + return ret; + + return sysfs_emit(buf, "(%d,%d,%d)\n", x, y, z); +} + +static ssize_t applesmc_light_show(struct device *dev, + struct device_attribute *attr, char *sysfsbuf) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + const struct applesmc_entry *entry; + static int data_length; + int ret; + u8 left = 0, right = 0; + u8 buffer[10]; + + if (!data_length) { + entry = applesmc_get_entry_by_key(smc, LIGHT_SENSOR_LEFT_KEY); + if (IS_ERR(entry)) + return PTR_ERR(entry); + if (entry->len > 10) + return -ENXIO; + data_length = entry->len; + pr_info("light sensor data length set to %d\n", data_length); + } + + ret = applesmc_read_key(smc, LIGHT_SENSOR_LEFT_KEY, buffer, data_length); + if (ret) + goto out; + /* newer macbooks report a single 10-bit bigendian value */ + if (data_length == 10) { + left = be16_to_cpu(*(__be16 *)(buffer + 6)) >> 2; + goto out; + } + left = buffer[2]; + + ret = applesmc_read_key(smc, LIGHT_SENSOR_RIGHT_KEY, buffer, data_length); + if (ret) + goto out; + right = buffer[2]; + +out: + if (ret) + return ret; + + return sysfs_emit(sysfsbuf, "(%d,%d)\n", left, right); +} + +/* Displays sensor key as label */ +static ssize_t applesmc_show_sensor_label(struct device *dev, + struct device_attribute *devattr, char *sysfsbuf) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + const char *key = smc->reg.index[to_index(devattr)]; + + return sysfs_emit(sysfsbuf, "%s\n", key); +} + +/* Displays degree Celsius * 1000 */ +static ssize_t applesmc_show_temperature(struct device *dev, + struct device_attribute *devattr, char *sysfsbuf) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + const char *key = smc->reg.index[to_index(devattr)]; + int ret; + s16 value; + int temp; + + ret = applesmc_read_s16(smc, key, &value); + if (ret) + return ret; + + temp = 250 * (value >> 6); + + return sysfs_emit(sysfsbuf, "%d\n", temp); +} + +static ssize_t applesmc_show_fan_speed(struct device *dev, + struct device_attribute *attr, char *sysfsbuf) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + const struct applesmc_entry *entry; + int ret; + unsigned int speed = 0; + char newkey[5]; + u8 buffer[2]; + + scnprintf(newkey, sizeof(newkey), fan_speed_fmt[to_option(attr)], + to_index(attr)); + + entry = applesmc_get_entry_by_key(smc, newkey); + if (IS_ERR(entry)) + return PTR_ERR(entry); + + if (!strcmp(entry->type, FLOAT_TYPE)) { + ret = applesmc_read_entry(smc, entry, (u8 *) &speed, 4); + speed = applesmc_float_to_u32(speed); + } else { + ret = applesmc_read_entry(smc, entry, buffer, 2); + speed = ((buffer[0] << 8 | buffer[1]) >> 2); + } + + if (ret) + return ret; + + return sysfs_emit(sysfsbuf, "%u\n", speed); +} + +static ssize_t applesmc_store_fan_speed(struct device *dev, + struct device_attribute *attr, + const char *sysfsbuf, size_t count) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + const struct applesmc_entry *entry; + int ret; + unsigned long speed; + char newkey[5]; + u8 buffer[2]; + + if (kstrtoul(sysfsbuf, 10, &speed) < 0 || speed >= 0x4000) + return -EINVAL; /* Bigger than a 14-bit value */ + + scnprintf(newkey, sizeof(newkey), fan_speed_fmt[to_option(attr)], + to_index(attr)); + + entry = applesmc_get_entry_by_key(smc, newkey); + if (IS_ERR(entry)) + return PTR_ERR(entry); + + if (!strcmp(entry->type, FLOAT_TYPE)) { + speed = applesmc_u32_to_float(speed); + ret = applesmc_write_entry(smc, entry, (u8 *) &speed, 4); + } else { + buffer[0] = (speed >> 6) & 0xff; + buffer[1] = (speed << 2) & 0xff; + ret = applesmc_write_key(smc, newkey, buffer, 2); + } + + if (ret) + return ret; + else + return count; +} + +static ssize_t applesmc_show_fan_manual(struct device *dev, + struct device_attribute *attr, char *sysfsbuf) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + int ret; + u16 manual = 0; + u8 buffer[2]; + char newkey[5]; + bool has_newkey = false; + + scnprintf(newkey, sizeof(newkey), FAN_MANUAL_FMT, to_index(attr)); + + ret = applesmc_has_key(smc, newkey, &has_newkey); + if (ret) + return ret; + + if (has_newkey) { + ret = applesmc_read_key(smc, newkey, buffer, 1); + manual = buffer[0]; + } else { + ret = applesmc_read_key(smc, FANS_MANUAL, buffer, 2); + manual = ((buffer[0] << 8 | buffer[1]) >> to_index(attr)) & 0x01; + } + + if (ret) + return ret; + + return sysfs_emit(sysfsbuf, "%d\n", manual); +} + +static ssize_t applesmc_store_fan_manual(struct device *dev, + struct device_attribute *attr, + const char *sysfsbuf, size_t count) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + int ret; + u8 buffer[2]; + char newkey[5]; + bool has_newkey = false; + unsigned long input; + u16 val; + + if (kstrtoul(sysfsbuf, 10, &input) < 0) + return -EINVAL; + + scnprintf(newkey, sizeof(newkey), FAN_MANUAL_FMT, to_index(attr)); + + ret = applesmc_has_key(smc, newkey, &has_newkey); + if (ret) + return ret; + + if (has_newkey) { + buffer[0] = input & 1; + ret = applesmc_write_key(smc, newkey, buffer, 1); + } else { + ret = applesmc_read_key(smc, FANS_MANUAL, buffer, 2); + val = (buffer[0] << 8 | buffer[1]); + if (ret) + goto out; + + if (input) + val = val | (0x01 << to_index(attr)); + else + val = val & ~(0x01 << to_index(attr)); + + buffer[0] = (val >> 8) & 0xFF; + buffer[1] = val & 0xFF; + + ret = applesmc_write_key(smc, FANS_MANUAL, buffer, 2); + } + +out: + if (ret) + return ret; + else + return count; +} + +static ssize_t applesmc_show_fan_position(struct device *dev, + struct device_attribute *attr, char *sysfsbuf) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + int ret; + char newkey[5]; + u8 buffer[17]; + + scnprintf(newkey, sizeof(newkey), FAN_ID_FMT, to_index(attr)); + + ret = applesmc_read_key(smc, newkey, buffer, 16); + buffer[16] = 0; + + if (ret) + return ret; + + return sysfs_emit(sysfsbuf, "%s\n", buffer + 4); +} + +static ssize_t applesmc_calibrate_show(struct device *dev, + struct device_attribute *attr, char *sysfsbuf) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + + return sysfs_emit(sysfsbuf, "(%d,%d)\n", smc->rest_x, smc->rest_y); +} + +static ssize_t applesmc_calibrate_store(struct device *dev, + struct device_attribute *attr, const char *sysfsbuf, size_t count) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + + applesmc_calibrate(smc); + + return count; +} + +static void applesmc_backlight_set(struct work_struct *work) +{ + struct applesmc_device *smc = container_of(work, struct applesmc_device, backlight_work); + + applesmc_write_key(smc, BACKLIGHT_KEY, smc->backlight_state, 2); +} + +static void applesmc_brightness_set(struct led_classdev *led_cdev, + enum led_brightness value) +{ + struct applesmc_device *smc = dev_get_drvdata(led_cdev->dev); + int ret; + + smc->backlight_state[0] = value; + ret = queue_work(smc->backlight_wq, &smc->backlight_work); + + if (debug && (!ret)) + dev_dbg(led_cdev->dev, "work was already on the queue.\n"); +} + +static ssize_t applesmc_BCLM_store(struct device *dev, + struct device_attribute *attr, char *sysfsbuf, size_t count) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + u8 val; + + if (kstrtou8(sysfsbuf, 10, &val) < 0) + return -EINVAL; + + if (val < 0 || val > 100) + return -EINVAL; + + if (applesmc_write_key(smc, "BCLM", &val, 1)) + return -ENODEV; + return count; +} + +static ssize_t applesmc_BCLM_show(struct device *dev, + struct device_attribute *attr, char *sysfsbuf) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + u8 val; + + if (applesmc_read_key(smc, "BCLM", &val, 1)) + return -ENODEV; + + return sysfs_emit(sysfsbuf, "%d\n", val); +} + +static ssize_t applesmc_key_count_show(struct device *dev, + struct device_attribute *attr, char *sysfsbuf) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + int ret; + u8 buffer[4]; + u32 count; + + ret = applesmc_read_key(smc, KEY_COUNT_KEY, buffer, 4); + if (ret) + return ret; + + count = ((u32)buffer[0]<<24) + ((u32)buffer[1]<<16) + + ((u32)buffer[2]<<8) + buffer[3]; + return sysfs_emit(sysfsbuf, "%d\n", count); +} + +static ssize_t applesmc_key_at_index_read_show(struct device *dev, + struct device_attribute *attr, char *sysfsbuf) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + const struct applesmc_entry *entry; + int ret; + + entry = applesmc_get_entry_by_index(smc, smc->key_at_index); + if (IS_ERR(entry)) + return PTR_ERR(entry); + ret = applesmc_read_entry(smc, entry, sysfsbuf, entry->len); + if (ret) + return ret; + + return entry->len; +} + +static ssize_t applesmc_key_at_index_data_length_show(struct device *dev, + struct device_attribute *attr, char *sysfsbuf) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + const struct applesmc_entry *entry; + + entry = applesmc_get_entry_by_index(smc, smc->key_at_index); + if (IS_ERR(entry)) + return PTR_ERR(entry); + + return sysfs_emit(sysfsbuf, "%d\n", entry->len); +} + +static ssize_t applesmc_key_at_index_type_show(struct device *dev, + struct device_attribute *attr, char *sysfsbuf) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + const struct applesmc_entry *entry; + + entry = applesmc_get_entry_by_index(smc, smc->key_at_index); + if (IS_ERR(entry)) + return PTR_ERR(entry); + + return sysfs_emit(sysfsbuf, "%s\n", entry->type); +} + +static ssize_t applesmc_key_at_index_name_show(struct device *dev, + struct device_attribute *attr, char *sysfsbuf) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + const struct applesmc_entry *entry; + + entry = applesmc_get_entry_by_index(smc, smc->key_at_index); + if (IS_ERR(entry)) + return PTR_ERR(entry); + + return sysfs_emit(sysfsbuf, "%s\n", entry->key); +} + +static ssize_t applesmc_key_at_index_show(struct device *dev, + struct device_attribute *attr, char *sysfsbuf) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + + return sysfs_emit(sysfsbuf, "%d\n", smc->key_at_index); +} + +static ssize_t applesmc_key_at_index_store(struct device *dev, + struct device_attribute *attr, const char *sysfsbuf, size_t count) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + unsigned long newkey; + + if (kstrtoul(sysfsbuf, 10, &newkey) < 0 + || newkey >= smc->reg.key_count) + return -EINVAL; + + smc->key_at_index = newkey; + return count; +} + +static struct applesmc_node_group info_group[] = { + { "name", applesmc_name_show }, + { "key_count", applesmc_key_count_show }, + { "key_at_index", applesmc_key_at_index_show, applesmc_key_at_index_store }, + { "key_at_index_name", applesmc_key_at_index_name_show }, + { "key_at_index_type", applesmc_key_at_index_type_show }, + { "key_at_index_data_length", applesmc_key_at_index_data_length_show }, + { "key_at_index_data", applesmc_key_at_index_read_show }, + { } +}; + +static struct applesmc_node_group accelerometer_group[] = { + { "position", applesmc_position_show }, + { "calibrate", applesmc_calibrate_show, applesmc_calibrate_store }, + { } +}; + +static struct applesmc_node_group light_sensor_group[] = { + { "light", applesmc_light_show }, + { } +}; + +static struct applesmc_node_group fan_group[] = { + { "fan%d_label", applesmc_show_fan_position }, + { "fan%d_input", applesmc_show_fan_speed, NULL, 0 }, + { "fan%d_min", applesmc_show_fan_speed, applesmc_store_fan_speed, 1 }, + { "fan%d_max", applesmc_show_fan_speed, NULL, 2 }, + { "fan%d_safe", applesmc_show_fan_speed, NULL, 3 }, + { "fan%d_output", applesmc_show_fan_speed, applesmc_store_fan_speed, 4 }, + { "fan%d_manual", applesmc_show_fan_manual, applesmc_store_fan_manual }, + { } +}; + +static struct applesmc_node_group temp_group[] = { + { "temp%d_label", applesmc_show_sensor_label }, + { "temp%d_input", applesmc_show_temperature }, + { } +}; + +static struct applesmc_node_group BCLM_group[] = { + { "battery_charge_limit", applesmc_BCLM_show, applesmc_BCLM_store }, + { } +}; + +/* Module stuff */ + +/* + * applesmc_destroy_nodes - remove files and free associated memory + */ +static void applesmc_destroy_nodes(struct applesmc_device *smc, + struct applesmc_node_group *groups) +{ + struct applesmc_node_group *grp; + struct applesmc_dev_attr *node; + + for (grp = groups; grp->nodes; grp++) { + for (node = grp->nodes; node->sda.dev_attr.attr.name; node++) + sysfs_remove_file(&smc->dev->dev.kobj, + &node->sda.dev_attr.attr); + kfree(grp->nodes); + grp->nodes = NULL; + } +} + +/* + * applesmc_create_nodes - create a two-dimensional group of sysfs files + */ +static int applesmc_create_nodes(struct applesmc_device *smc, + struct applesmc_node_group *groups, int num) +{ + struct applesmc_node_group *grp; + struct applesmc_dev_attr *node; + struct attribute *attr; + int ret, i; + + for (grp = groups; grp->format; grp++) { + grp->nodes = kzalloc_objs(*node, num + 1); + if (!grp->nodes) { + ret = -ENOMEM; + goto out; + } + for (i = 0; i < num; i++) { + node = &grp->nodes[i]; + scnprintf(node->name, sizeof(node->name), grp->format, + i + 1); + node->sda.index = (grp->option << 16) | (i & 0xffff); + node->sda.dev_attr.show = grp->show; + node->sda.dev_attr.store = grp->store; + attr = &node->sda.dev_attr.attr; + sysfs_attr_init(attr); + attr->name = node->name; + attr->mode = 0444 | (grp->store ? 0200 : 0); + ret = sysfs_create_file(&smc->dev->dev.kobj, attr); + if (ret) { + attr->name = NULL; + goto out; + } + } + } + + return 0; +out: + applesmc_destroy_nodes(smc, groups); + return ret; +} + +/* Create accelerometer resources */ +static int applesmc_create_accelerometer(struct applesmc_device *smc) +{ + int ret; + if (!smc->reg.has_accelerometer) + return 0; + + ret = applesmc_create_nodes(smc, accelerometer_group, 1); + if (ret) + goto out; + + smc->idev = input_allocate_device(); + if (!smc->idev) { + ret = -ENOMEM; + goto out_sysfs; + } + + /* initial calibrate for the input device */ + applesmc_calibrate(smc); + + /* initialize the input device */ + smc->idev->name = "applesmc"; + smc->idev->id.bustype = BUS_HOST; + smc->idev->dev.parent = &smc->dev->dev; + input_set_abs_params(smc->idev, ABS_X, + -256, 256, APPLESMC_INPUT_FUZZ, APPLESMC_INPUT_FLAT); + input_set_abs_params(smc->idev, ABS_Y, + -256, 256, APPLESMC_INPUT_FUZZ, APPLESMC_INPUT_FLAT); + + ret = input_setup_polling(smc->idev, applesmc_idev_poll); + if (ret) + goto out_idev; + + input_set_poll_interval(smc->idev, APPLESMC_POLL_INTERVAL); + + ret = input_register_device(smc->idev); + if (ret) + goto out_idev; + + return 0; + +out_idev: + input_free_device(smc->idev); + +out_sysfs: + applesmc_destroy_nodes(smc, accelerometer_group); + +out: + pr_warn("driver init failed (ret=%d)!\n", ret); + return ret; +} + +/* Release all resources used by the accelerometer */ +static void applesmc_release_accelerometer(struct applesmc_device *smc) +{ + if (!smc->reg.has_accelerometer) + return; + input_unregister_device(smc->idev); + applesmc_destroy_nodes(smc, accelerometer_group); +} + +static int applesmc_create_light_sensor(struct applesmc_device *smc) +{ + if (!smc->reg.num_light_sensors) + return 0; + return applesmc_create_nodes(smc, light_sensor_group, 1); +} + +static void applesmc_release_light_sensor(struct applesmc_device *smc) +{ + if (!smc->reg.num_light_sensors) + return; + applesmc_destroy_nodes(smc, light_sensor_group); +} + +static int applesmc_create_key_backlight(struct applesmc_device *smc) +{ + int ret; + + if (!smc->reg.has_key_backlight) + return 0; + smc->backlight_wq = create_singlethread_workqueue("applesmc-led"); + if (!smc->backlight_wq) + return -ENOMEM; + + INIT_WORK(&smc->backlight_work, applesmc_backlight_set); + smc->backlight_dev.name = "smc::kbd_backlight"; + smc->backlight_dev.default_trigger = "nand-disk"; + smc->backlight_dev.brightness_set = applesmc_brightness_set; + ret = led_classdev_register(&smc->dev->dev, &smc->backlight_dev); + if (ret) + destroy_workqueue(smc->backlight_wq); + + return ret; +} + +static void applesmc_release_key_backlight(struct applesmc_device *smc) +{ + if (!smc->reg.has_key_backlight) + return; + led_classdev_unregister(&smc->backlight_dev); + destroy_workqueue(smc->backlight_wq); +} + +static int applesmc_create_modules(struct applesmc_device *smc) +{ + int ret; + + ret = applesmc_create_nodes(smc, info_group, 1); + if (ret) + goto out; + ret = applesmc_create_nodes(smc, BCLM_group, 1); + if (ret) + goto out_info; + + ret = applesmc_create_nodes(smc, fan_group, smc->reg.fan_count); + if (ret) + goto out_bclm; + + ret = applesmc_create_nodes(smc, temp_group, smc->reg.index_count); + if (ret) + goto out_fans; + + ret = applesmc_create_accelerometer(smc); + if (ret) + goto out_temperature; + + ret = applesmc_create_light_sensor(smc); + if (ret) + goto out_accelerometer; + + ret = applesmc_create_key_backlight(smc); + if (ret) + goto out_light_sysfs; + + smc->hwmon_dev = hwmon_device_register(&smc->dev->dev); + if (IS_ERR(smc->hwmon_dev)) { + ret = PTR_ERR(smc->hwmon_dev); + goto out_light_ledclass; + } + + return 0; + +out_light_ledclass: + applesmc_release_key_backlight(smc); +out_light_sysfs: + applesmc_release_light_sensor(smc); +out_accelerometer: + applesmc_release_accelerometer(smc); +out_temperature: + applesmc_destroy_nodes(smc, temp_group); +out_fans: + applesmc_destroy_nodes(smc, fan_group); +out_bclm: + applesmc_destroy_nodes(smc, BCLM_group); +out_info: + applesmc_destroy_nodes(smc, info_group); +out: + return ret; +} + +static void applesmc_destroy_modules(struct applesmc_device *smc) +{ + hwmon_device_unregister(smc->hwmon_dev); + applesmc_release_key_backlight(smc); + applesmc_release_light_sensor(smc); + applesmc_release_accelerometer(smc); + applesmc_destroy_nodes(smc, temp_group); + applesmc_destroy_nodes(smc, fan_group); + applesmc_destroy_nodes(smc, BCLM_group); + applesmc_destroy_nodes(smc, info_group); +} + +static int __init applesmc_init(void) +{ + int ret; + + if (!dmi_check_system(t2_mac_tbl)) { + pr_warn("supported laptop not found!\n"); + ret = -ENODEV; + goto out; + } + + ret = acpi_bus_register_driver(&applesmc_driver); + if (ret) + goto out; + + return 0; + +out: + pr_warn("driver init failed (ret=%d)!\n", ret); + return ret; +} + +static void __exit applesmc_exit(void) +{ + acpi_bus_unregister_driver(&applesmc_driver); +} + +module_init(applesmc_init); +module_exit(applesmc_exit); + +MODULE_AUTHOR("Nicolas Boichat"); +MODULE_AUTHOR("Paul Pawlowski"); +MODULE_DESCRIPTION("Apple SMC"); +MODULE_LICENSE("GPL v2"); +MODULE_DEVICE_TABLE(dmi, t2_mac_tbl); diff -ruN --no-dereference -x .git a/drivers/hwmon/applesmc.c b/drivers/hwmon/applesmc.c --- a/drivers/hwmon/applesmc.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/hwmon/applesmc.c 2026-10-13 14:33:30.000000000 +0200 @@ -17,6 +17,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -1316,7 +1317,7 @@ { int ret; - if (!dmi_check_system(applesmc_whitelist)) { + if (!dmi_check_system(applesmc_whitelist) || dmi_check_system(t2_mac_tbl)) { pr_warn("supported laptop not found!\n"); ret = -ENODEV; goto out; diff -ruN --no-dereference -x .git a/drivers/i2c/i2c-core-acpi.c b/drivers/i2c/i2c-core-acpi.c --- a/drivers/i2c/i2c-core-acpi.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/i2c/i2c-core-acpi.c 2026-10-13 14:33:30.000000000 +0200 @@ -373,6 +373,7 @@ * the device works without issues on Windows at what is expected to be * a 400KHz frequency. The root cause of the issue is not known. */ + { "ASUE140D", 0 }, { "DLL0945", 0 }, { "ELAN0678", 0 }, { "ELAN06FA", 0 }, diff -ruN --no-dereference -x .git a/drivers/input/evdev.c b/drivers/input/evdev.c --- a/drivers/input/evdev.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/input/evdev.c 2026-10-13 14:33:30.000000000 +0200 @@ -47,6 +47,7 @@ struct fasync_struct *fasync; struct evdev *evdev; struct list_head node; + struct rcu_head rcu; enum input_clock_type clk_type; bool revoked; unsigned long *evmasks[EV_CNT]; @@ -373,13 +374,22 @@ spin_unlock(&evdev->client_lock); } +static void evdev_reclaim_client(struct rcu_head *rp) +{ + struct evdev_client *client = container_of(rp, struct evdev_client, rcu); + unsigned int i; + for (i = 0; i < EV_CNT; ++i) + bitmap_free(client->evmasks[i]); + kvfree(client); +} + static void evdev_detach_client(struct evdev *evdev, struct evdev_client *client) { spin_lock(&evdev->client_lock); list_del_rcu(&client->node); spin_unlock(&evdev->client_lock); - synchronize_rcu(); + call_rcu(&client->rcu, evdev_reclaim_client); } static int evdev_open_device(struct evdev *evdev) @@ -432,7 +442,6 @@ { struct evdev_client *client = file->private_data; struct evdev *evdev = client->evdev; - unsigned int i; mutex_lock(&evdev->mutex); @@ -444,11 +453,6 @@ evdev_detach_client(evdev, client); - for (i = 0; i < EV_CNT; ++i) - bitmap_free(client->evmasks[i]); - - kvfree(client); - evdev_close_device(evdev); return 0; @@ -491,7 +495,6 @@ err_free_client: evdev_detach_client(evdev, client); - kvfree(client); return error; } diff -ruN --no-dereference -x .git a/drivers/media/v4l2-core/Kconfig b/drivers/media/v4l2-core/Kconfig --- a/drivers/media/v4l2-core/Kconfig 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/media/v4l2-core/Kconfig 2026-10-13 14:33:30.000000000 +0200 @@ -40,6 +40,11 @@ config V4L2_JPEG_HELPER tristate +config V4L2_LOOPBACK + tristate "V4L2 loopback device" + help + V4L2 loopback device + # Used by drivers that need v4l2-h264.ko config V4L2_H264 tristate diff -ruN --no-dereference -x .git a/drivers/media/v4l2-core/Makefile b/drivers/media/v4l2-core/Makefile --- a/drivers/media/v4l2-core/Makefile 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/media/v4l2-core/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -34,5 +34,7 @@ obj-$(CONFIG_V4L2_MEM2MEM_DEV) += v4l2-mem2mem.o obj-$(CONFIG_V4L2_VP9) += v4l2-vp9.o +obj-$(CONFIG_V4L2_LOOPBACK) += v4l2loopback.o + obj-$(CONFIG_VIDEO_TUNER) += tuner.o obj-$(CONFIG_VIDEO_DEV) += v4l2-dv-timings.o videodev.o diff -ruN --no-dereference -x .git a/drivers/media/v4l2-core/v4l2loopback.c b/drivers/media/v4l2-core/v4l2loopback.c --- a/drivers/media/v4l2-core/v4l2loopback.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/media/v4l2-core/v4l2loopback.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,3375 @@ +/* -*- c-file-style: "linux" -*- */ +/* + * v4l2loopback.c -- video4linux2 loopback driver + * + * Copyright (C) 2005-2009 Vasily Levin (vasaka@gmail.com) + * Copyright (C) 2010-2023 IOhannes m zmoelnig (zmoelnig@iem.at) + * Copyright (C) 2011 Stefan Diewald (stefan.diewald@mytum.de) + * Copyright (C) 2012 Anton Novikov (random.plant@gmail.com) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "v4l2loopback.h" + +#define V4L2LOOPBACK_CTL_ADD_legacy 0x4C80 +#define V4L2LOOPBACK_CTL_REMOVE_legacy 0x4C81 +#define V4L2LOOPBACK_CTL_QUERY_legacy 0x4C82 + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) +#error This module is not supported on kernels before 4.0.0. +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) +#define strscpy strlcpy +#endif + +#if defined(timer_setup) +#define HAVE_TIMER_SETUP +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 7, 0) +#define VFL_TYPE_VIDEO VFL_TYPE_GRABBER +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0) +#define timer_delete_sync del_timer_sync +#endif + +#define fh_to_opener(ptr) container_of((ptr), struct v4l2_loopback_opener, fh) +#define file_to_opener(ptr) \ + container_of(file_to_v4l2_fh(ptr), struct v4l2_loopback_opener, fh) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 18, 0) +#define v4l2l_f_to_opener(_file, _fh) fh_to_opener(_fh) +#define v4l2_fh_add(fh, filp) v4l2_fh_add(fh) +#define v4l2_fh_del(fh, filp) v4l2_fh_del(fh) +#else +#define v4l2l_f_to_opener(_file, _fh) file_to_opener(_file) +#endif + +#define V4L2LOOPBACK_VERSION_CODE \ + KERNEL_VERSION(V4L2LOOPBACK_VERSION_MAJOR, V4L2LOOPBACK_VERSION_MINOR, \ + V4L2LOOPBACK_VERSION_BUGFIX) + +MODULE_DESCRIPTION("V4L2 loopback video device"); +MODULE_AUTHOR("Vasily Levin, " + "IOhannes m zmoelnig ," + "Stefan Diewald," + "Anton Novikov" + "et al."); +#ifdef SNAPSHOT_VERSION +MODULE_VERSION(__stringify(SNAPSHOT_VERSION)); +#else +MODULE_VERSION("" __stringify(V4L2LOOPBACK_VERSION_MAJOR) "." __stringify( + V4L2LOOPBACK_VERSION_MINOR) "." __stringify(V4L2LOOPBACK_VERSION_BUGFIX)); +#endif +MODULE_LICENSE("GPL"); + +/* + * helpers + */ +#define dprintk(fmt, args...) \ + do { \ + if (debug > 0) { \ + printk(KERN_INFO "v4l2-loopback[" __stringify( \ + __LINE__) "], pid(%d): " fmt, \ + task_pid_nr(current), ##args); \ + } \ + } while (0) + +#define MARK() \ + do { \ + if (debug > 1) { \ + printk(KERN_INFO "%s:%d[%s], pid(%d)\n", __FILE__, \ + __LINE__, __func__, task_pid_nr(current)); \ + } \ + } while (0) + +#define dprintkrw(fmt, args...) \ + do { \ + if (debug > 2) { \ + printk(KERN_INFO "v4l2-loopback[" __stringify( \ + __LINE__) "], pid(%d): " fmt, \ + task_pid_nr(current), ##args); \ + } \ + } while (0) + +static inline void v4l2l_get_timestamp(struct v4l2_buffer *b) +{ + struct timespec64 ts; + ktime_get_ts64(&ts); + + b->timestamp.tv_sec = ts.tv_sec; + b->timestamp.tv_usec = (ts.tv_nsec / NSEC_PER_USEC); + b->flags |= V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; + b->flags &= ~V4L2_BUF_FLAG_TIMESTAMP_COPY; +} + +#if BITS_PER_LONG == 32 +#include /* do_div() for 64bit division */ +static inline int v4l2l_mod64(const s64 A, const u32 B) +{ + u64 a = (u64)A; + u32 b = B; + + if (A > 0) + return do_div(a, b); + a = -A; + return -do_div(a, b); +} +#else +static inline int v4l2l_mod64(const s64 A, const u32 B) +{ + return A % B; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0) +typedef unsigned __poll_t; +#endif + +/* module constants + * can be overridden during he build process using something like + * make KCPPFLAGS="-DMAX_DEVICES=100" + */ + +/* maximum number of v4l2loopback devices that can be created */ +#ifndef MAX_DEVICES +#define MAX_DEVICES 8 +#endif + +/* whether the default is to announce capabilities exclusively or not */ +#ifndef V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS +#define V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS 0 +#endif + +/* when a producer is considered to have gone stale */ +#ifndef MAX_TIMEOUT +#define MAX_TIMEOUT (100 * 1000) /* in msecs */ +#endif + +/* max buffers that can be mapped, actually they + * are all mapped to max_buffers buffers */ +#ifndef MAX_BUFFERS +#define MAX_BUFFERS 32 +#endif + +/* module parameters */ +static int debug = 0; +module_param(debug, int, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(debug, "debugging level (higher values == more verbose)"); + +#define V4L2LOOPBACK_DEFAULT_MAX_BUFFERS 2 +static int max_buffers = V4L2LOOPBACK_DEFAULT_MAX_BUFFERS; +module_param(max_buffers, int, S_IRUGO); +MODULE_PARM_DESC(max_buffers, + "how many buffers should be allocated [DEFAULT: " __stringify( + V4L2LOOPBACK_DEFAULT_MAX_BUFFERS) "]"); + +/* how many times a device can be opened + * the per-module default value can be overridden on a per-device basis using + * the /sys/devices interface + * + * note that max_openers should be at least 2 in order to get a working system: + * one opener for the producer and one opener for the consumer + * however, we leave that to the user + */ +#define V4L2LOOPBACK_DEFAULT_MAX_OPENERS 10 +static int max_openers = V4L2LOOPBACK_DEFAULT_MAX_OPENERS; +module_param(max_openers, int, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC( + max_openers, + "how many users can open the loopback device [DEFAULT: " __stringify( + V4L2LOOPBACK_DEFAULT_MAX_OPENERS) "]"); + +static int devices = -1; +module_param(devices, int, 0); +MODULE_PARM_DESC(devices, "how many devices should be created"); + +static int video_nr[MAX_DEVICES] = { [0 ...(MAX_DEVICES - 1)] = -1 }; +module_param_array(video_nr, int, NULL, 0444); +MODULE_PARM_DESC(video_nr, + "video device numbers (-1=auto, 0=/dev/video0, etc.)"); + +static char *card_label[MAX_DEVICES]; +module_param_array(card_label, charp, NULL, 0000); +MODULE_PARM_DESC(card_label, "card labels for each device"); + +static bool exclusive_caps[MAX_DEVICES] = { + [0 ...(MAX_DEVICES - 1)] = V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS +}; +module_param_array(exclusive_caps, bool, NULL, 0444); +/* FIXXME: wording */ +MODULE_PARM_DESC( + exclusive_caps, + "whether to announce OUTPUT/CAPTURE capabilities exclusively or not [DEFAULT: " __stringify( + V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS) "]"); + +/* format specifications */ +#define V4L2LOOPBACK_SIZE_MIN_WIDTH 2 +#define V4L2LOOPBACK_SIZE_MIN_HEIGHT 1 +#define V4L2LOOPBACK_SIZE_DEFAULT_MAX_WIDTH 8192 +#define V4L2LOOPBACK_SIZE_DEFAULT_MAX_HEIGHT 8192 + +#define V4L2LOOPBACK_SIZE_DEFAULT_WIDTH 640 +#define V4L2LOOPBACK_SIZE_DEFAULT_HEIGHT 480 + +static int max_width = V4L2LOOPBACK_SIZE_DEFAULT_MAX_WIDTH; +module_param(max_width, int, S_IRUGO); +MODULE_PARM_DESC(max_width, + "maximum allowed frame width [DEFAULT: " __stringify( + V4L2LOOPBACK_SIZE_DEFAULT_MAX_WIDTH) "]"); +static int max_height = V4L2LOOPBACK_SIZE_DEFAULT_MAX_HEIGHT; +module_param(max_height, int, S_IRUGO); +MODULE_PARM_DESC(max_height, + "maximum allowed frame height [DEFAULT: " __stringify( + V4L2LOOPBACK_SIZE_DEFAULT_MAX_HEIGHT) "]"); + +static DEFINE_IDR(v4l2loopback_index_idr); +static DEFINE_MUTEX(v4l2loopback_ctl_mutex); + +/* frame intervals */ +#define V4L2LOOPBACK_FRAME_INTERVAL_MAX __UINT32_MAX__ +#define V4L2LOOPBACK_FPS_DEFAULT 30 +#define V4L2LOOPBACK_FPS_MAX 1000 + +/* control IDs */ +#define V4L2LOOPBACK_CID_BASE (V4L2_CID_USER_BASE | 0xf000) +#define CID_KEEP_FORMAT (V4L2LOOPBACK_CID_BASE + 0) +#define CID_SUSTAIN_FRAMERATE (V4L2LOOPBACK_CID_BASE + 1) +#define CID_TIMEOUT (V4L2LOOPBACK_CID_BASE + 2) +#define CID_TIMEOUT_IMAGE_IO (V4L2LOOPBACK_CID_BASE + 3) + +static int v4l2loopback_s_ctrl(struct v4l2_ctrl *ctrl); +static const struct v4l2_ctrl_ops v4l2loopback_ctrl_ops = { + .s_ctrl = v4l2loopback_s_ctrl, +}; +static const struct v4l2_ctrl_config v4l2loopback_ctrl_keepformat = { + // clang-format off + .ops = &v4l2loopback_ctrl_ops, + .id = CID_KEEP_FORMAT, + .name = "keep_format", + .type = V4L2_CTRL_TYPE_BOOLEAN, + .min = 0, + .max = 1, + .step = 1, + .def = 0, + // clang-format on +}; +static const struct v4l2_ctrl_config v4l2loopback_ctrl_sustainframerate = { + // clang-format off + .ops = &v4l2loopback_ctrl_ops, + .id = CID_SUSTAIN_FRAMERATE, + .name = "sustain_framerate", + .type = V4L2_CTRL_TYPE_BOOLEAN, + .min = 0, + .max = 1, + .step = 1, + .def = 0, + // clang-format on +}; +static const struct v4l2_ctrl_config v4l2loopback_ctrl_timeout = { + // clang-format off + .ops = &v4l2loopback_ctrl_ops, + .id = CID_TIMEOUT, + .name = "timeout", + .type = V4L2_CTRL_TYPE_INTEGER, + .min = 0, + .max = MAX_TIMEOUT, + .step = 1, + .def = 0, + // clang-format on +}; +static const struct v4l2_ctrl_config v4l2loopback_ctrl_timeoutimageio = { + // clang-format off + .ops = &v4l2loopback_ctrl_ops, + .id = CID_TIMEOUT_IMAGE_IO, + .name = "timeout_image_io", + .type = V4L2_CTRL_TYPE_BUTTON, + .min = 0, + .max = 0, + .step = 0, + .def = 0, + // clang-format on +}; + +/* module structures */ +struct v4l2loopback_private { + int device_nr; +}; + +/* TODO(vasaka) use typenames which are common to kernel, but first find out if + * it is needed */ +/* struct keeping state and settings of loopback device */ + +struct v4l2l_buffer { + struct v4l2_buffer buffer; + struct list_head list_head; + atomic_t use_count; +}; + +struct v4l2_loopback_device { + struct v4l2_device v4l2_dev; + struct v4l2_ctrl_handler ctrl_handler; + struct video_device *vdev; + + /* loopback device-specific parameters */ + char card_label[32]; + bool announce_all_caps; /* announce both OUTPUT and CAPTURE capabilities + * when true; else announce OUTPUT when no + * writer is streaming, otherwise CAPTURE. */ + int max_openers; /* how many times can this device be opened */ + int min_width, max_width; + int min_height, max_height; + + /* pixel and stream format */ + struct v4l2_pix_format pix_format; + bool pix_format_has_valid_sizeimage; + struct v4l2_captureparm capture_param; + unsigned long frame_jiffies; + + /* ctrls */ + int keep_format; /* CID_KEEP_FORMAT; lock the format, do not free + * on close(), and when `!announce_all_caps` do NOT + * fall back to OUTPUT when no writers attached (clear + * `keep_format` to attach a new writer) */ + int sustain_framerate; /* CID_SUSTAIN_FRAMERATE; duplicate frames to maintain + (close to) nominal framerate */ + unsigned long timeout_jiffies; /* CID_TIMEOUT; 0 means disabled */ + int timeout_image_io; /* CID_TIMEOUT_IMAGE_IO; next opener will + * queue/dequeue the timeout image buffer */ + + /* buffers for OUTPUT and CAPTURE */ + u8 *image; /* pointer to actual buffers data */ + unsigned long image_size; /* number of bytes alloc'd for all buffers */ + struct v4l2l_buffer buffers[MAX_BUFFERS]; /* inner driver buffers */ + u32 buffer_count; /* should not be big, 4 is a good choice */ + u32 buffer_size; /* number of bytes alloc'd per buffer */ + u32 used_buffer_count; /* number of buffers allocated to openers */ + struct list_head outbufs_list; /* FIFO queue for OUTPUT buffers */ + u32 bufpos2index[MAX_BUFFERS]; /* mapping of `(position % used_buffers)` + * to `buffers[index]` */ + s64 write_position; /* sequence number of last 'displayed' buffer plus + * one */ + + /* synchronization between openers */ + atomic_t open_count; + struct mutex image_mutex; /* mutex for allocating image(s) and + * exchanging format tokens */ + spinlock_t lock; /* lock for the timeout and framerate timers */ + spinlock_t list_lock; /* lock for the OUTPUT buffer queue */ + wait_queue_head_t read_event; + u32 format_tokens; /* tokens to 'set format' for OUTPUT, CAPTURE, or + * timeout buffers */ + u32 stream_tokens; /* tokens to 'start' OUTPUT, CAPTURE, or timeout + * stream */ + + /* sustain framerate */ + struct timer_list sustain_timer; + unsigned int reread_count; + + /* timeout */ + u8 *timeout_image; /* copied to outgoing buffers when timeout passes */ + struct v4l2l_buffer timeout_buffer; + u32 timeout_buffer_size; /* number bytes alloc'd for timeout buffer */ + struct timer_list timeout_timer; + int timeout_happened; +}; + +enum v4l2l_io_method { + V4L2L_IO_NONE = 0, + V4L2L_IO_MMAP = 1, + V4L2L_IO_FILE = 2, + V4L2L_IO_TIMEOUT = 3, +}; + +/* struct keeping state and type of opener */ +struct v4l2_loopback_opener { + u32 format_token; /* token (if any) for type used in call to S_FMT or + * REQBUFS */ + u32 stream_token; /* token (if any) for type used in call to STREAMON */ + u32 buffer_count; /* number of buffers (if any) that opener acquired via + * REQBUFS */ + s64 read_position; /* sequence number of the next 'captured' frame */ + unsigned int reread_count; + enum v4l2l_io_method io_method; + + struct v4l2_fh fh; +}; + +/* this is heavily inspired by the bttv driver found in the linux kernel */ +struct v4l2l_format { + char *name; + int fourcc; /* video4linux 2 */ + int depth; /* bit/pixel */ + int flags; +}; +/* set the v4l2l_format.flags to PLANAR for non-packed formats */ +#define FORMAT_FLAGS_PLANAR 0x01 +#define FORMAT_FLAGS_COMPRESSED 0x02 + +#include "v4l2loopback_formats.h" + +#ifndef V4L2_TYPE_IS_CAPTURE +#define V4L2_TYPE_IS_CAPTURE(type) \ + ((type) == V4L2_BUF_TYPE_VIDEO_CAPTURE || \ + (type) == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) +#endif /* V4L2_TYPE_IS_CAPTURE */ +#ifndef V4L2_TYPE_IS_OUTPUT +#define V4L2_TYPE_IS_OUTPUT(type) \ + ((type) == V4L2_BUF_TYPE_VIDEO_OUTPUT || \ + (type) == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) +#endif /* V4L2_TYPE_IS_OUTPUT */ + +/* token values for privilege to set format or start/stop stream */ +#define V4L2L_TOKEN_CAPTURE 0x01 +#define V4L2L_TOKEN_OUTPUT 0x02 +#define V4L2L_TOKEN_TIMEOUT 0x04 +#define V4L2L_TOKEN_MASK \ + (V4L2L_TOKEN_CAPTURE | V4L2L_TOKEN_OUTPUT | V4L2L_TOKEN_TIMEOUT) + +/* helpers for token exchange and token status */ +#define token_from_type(type) \ + (V4L2_TYPE_IS_CAPTURE(type) ? V4L2L_TOKEN_CAPTURE : V4L2L_TOKEN_OUTPUT) +#define acquire_token(dev, opener, label, token) \ + do { \ + (opener)->label##_token = token; \ + (dev)->label##_tokens &= ~token; \ + } while (0) +#define release_token(dev, opener, label) \ + do { \ + (dev)->label##_tokens |= (opener)->label##_token; \ + (opener)->label##_token = 0; \ + } while (0) +#define has_output_token(token) (token & V4L2L_TOKEN_OUTPUT) +#define has_capture_token(token) (token & V4L2L_TOKEN_CAPTURE) +#define has_no_owners(dev) ((~((dev)->format_tokens) & V4L2L_TOKEN_MASK) == 0) +#define has_other_owners(opener, dev) \ + (~((dev)->format_tokens ^ (opener)->format_token) & V4L2L_TOKEN_MASK) +#define need_timeout_buffer(dev, token) \ + ((dev)->timeout_jiffies > 0 || (token) & V4L2L_TOKEN_TIMEOUT) + +static const unsigned int FORMATS = ARRAY_SIZE(formats); + +static char *fourcc2str(unsigned int fourcc, char buf[5]) +{ + buf[0] = (fourcc >> 0) & 0xFF; + buf[1] = (fourcc >> 8) & 0xFF; + buf[2] = (fourcc >> 16) & 0xFF; + buf[3] = (fourcc >> 24) & 0xFF; + buf[4] = 0; + + return buf; +} + +static const struct v4l2l_format *format_by_fourcc(int fourcc) +{ + unsigned int i; + char buf[5]; + + for (i = 0; i < FORMATS; i++) { + if (formats[i].fourcc == fourcc) + return formats + i; + } + + dprintk("unsupported format '%4s'\n", fourcc2str(fourcc, buf)); + return NULL; +} + +static void pix_format_set_size(struct v4l2_pix_format *f, + const struct v4l2l_format *fmt, + unsigned int width, unsigned int height) +{ + f->width = width; + f->height = height; + + if (fmt->flags & FORMAT_FLAGS_PLANAR) { + f->bytesperline = width; /* Y plane */ + f->sizeimage = (width * height * fmt->depth) >> 3; + } else if (fmt->flags & FORMAT_FLAGS_COMPRESSED) { + /* doesn't make sense for compressed formats */ + f->bytesperline = 0; + f->sizeimage = (width * height * fmt->depth) >> 3; + } else { + f->bytesperline = (width * fmt->depth) >> 3; + f->sizeimage = height * f->bytesperline; + } +} + +static int v4l2l_fill_format(struct v4l2_format *fmt, const u32 minwidth, + const u32 maxwidth, const u32 minheight, + const u32 maxheight) +{ + u32 width = fmt->fmt.pix.width, height = fmt->fmt.pix.height; + u32 pixelformat = fmt->fmt.pix.pixelformat; + struct v4l2_format fmt0 = *fmt; + u32 bytesperline = 0, sizeimage = 0; + + if (!width) + width = V4L2LOOPBACK_SIZE_DEFAULT_WIDTH; + if (!height) + height = V4L2LOOPBACK_SIZE_DEFAULT_HEIGHT; + width = clamp_val(width, minwidth, maxwidth); + height = clamp_val(height, minheight, maxheight); + + /* sets: width,height,pixelformat,bytesperline,sizeimage */ + if (!(V4L2_TYPE_IS_MULTIPLANAR(fmt0.type))) { + fmt0.fmt.pix.bytesperline = 0; + fmt0.fmt.pix.sizeimage = 0; + } + + if (0) { + ; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 2, 0) + } else if (!v4l2_fill_pixfmt(&fmt0.fmt.pix, pixelformat, width, + height)) { + ; + } else if (!v4l2_fill_pixfmt_mp(&fmt0.fmt.pix_mp, pixelformat, width, + height)) { + ; +#endif + } else { + const struct v4l2l_format *format = + format_by_fourcc(pixelformat); + if (!format) + return -EINVAL; + pix_format_set_size(&fmt0.fmt.pix, format, width, height); + fmt0.fmt.pix.pixelformat = format->fourcc; + } + + if (V4L2_TYPE_IS_MULTIPLANAR(fmt0.type)) { + *fmt = fmt0; + + if ((fmt->fmt.pix_mp.colorspace == V4L2_COLORSPACE_DEFAULT) || + (fmt->fmt.pix_mp.colorspace > V4L2_COLORSPACE_DCI_P3)) + fmt->fmt.pix_mp.colorspace = V4L2_COLORSPACE_SRGB; + if (V4L2_FIELD_ANY == fmt->fmt.pix_mp.field) + fmt->fmt.pix_mp.field = V4L2_FIELD_NONE; + } else { + bytesperline = fmt->fmt.pix.bytesperline; + sizeimage = fmt->fmt.pix.sizeimage; + + *fmt = fmt0; + + if (!fmt->fmt.pix.bytesperline) + fmt->fmt.pix.bytesperline = bytesperline; + if (!fmt->fmt.pix.sizeimage) + fmt->fmt.pix.sizeimage = sizeimage; + + if ((fmt->fmt.pix.colorspace == V4L2_COLORSPACE_DEFAULT) || + (fmt->fmt.pix.colorspace > V4L2_COLORSPACE_DCI_P3)) + fmt->fmt.pix.colorspace = V4L2_COLORSPACE_SRGB; + if (V4L2_FIELD_ANY == fmt->fmt.pix.field) + fmt->fmt.pix.field = V4L2_FIELD_NONE; + } + + return 0; +} + +/* Checks if v4l2l_fill_format() has set a valid, fixed sizeimage val. */ +static bool v4l2l_pix_format_has_valid_sizeimage(struct v4l2_format *fmt) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 2, 0) + const struct v4l2_format_info *info; + + info = v4l2_format_info(fmt->fmt.pix.pixelformat); + if (info && info->mem_planes == 1) + return true; +#endif + + return false; +} + +static int pix_format_eq(const struct v4l2_pix_format *ref, + const struct v4l2_pix_format *tgt, int strict) +{ + /* check if the two formats are equivalent. + * ANY fields are handled gracefully + */ +#define _pix_format_eq0(x) \ + if (ref->x != tgt->x) \ + result = 0 +#define _pix_format_eq1(x, def) \ + do { \ + if ((def != tgt->x) && (ref->x != tgt->x)) { \ + printk(KERN_INFO #x " failed"); \ + result = 0; \ + } \ + } while (0) + int result = 1; + _pix_format_eq0(width); + _pix_format_eq0(height); + _pix_format_eq0(pixelformat); + if (!strict) + return result; + _pix_format_eq1(field, V4L2_FIELD_ANY); + _pix_format_eq0(bytesperline); + _pix_format_eq0(sizeimage); + _pix_format_eq1(colorspace, V4L2_COLORSPACE_DEFAULT); + return result; +} + +static void set_timeperframe(struct v4l2_loopback_device *dev, + struct v4l2_fract *tpf) +{ + if (!tpf->denominator && !tpf->numerator) { + tpf->numerator = 1; + tpf->denominator = V4L2LOOPBACK_FPS_DEFAULT; + } else if (tpf->numerator > + V4L2LOOPBACK_FRAME_INTERVAL_MAX * tpf->denominator) { + /* divide-by-zero or greater than maximum interval => min FPS */ + tpf->numerator = V4L2LOOPBACK_FRAME_INTERVAL_MAX; + tpf->denominator = 1; + } else if (tpf->numerator * V4L2LOOPBACK_FPS_MAX < tpf->denominator) { + /* zero or lower than minimum interval => max FPS */ + tpf->numerator = 1; + tpf->denominator = V4L2LOOPBACK_FPS_MAX; + } + + dev->capture_param.timeperframe = *tpf; + dev->frame_jiffies = + max(1UL, (msecs_to_jiffies(1000) * tpf->numerator) / + tpf->denominator); +} + +static struct v4l2_loopback_device *v4l2loopback_cd2dev(struct device *cd); + +/* device attributes */ +/* available via sysfs: /sys/devices/virtual/video4linux/video* */ + +static ssize_t attr_show_format(struct device *cd, + struct device_attribute *attr, char *buf) +{ + /* gets the current format as "FOURCC:WxH@f/s", e.g. "YUYV:320x240@1000/30" */ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + const struct v4l2_fract *tpf; + char buf4cc[5], buf_fps[32]; + + if (!dev || (has_no_owners(dev) && !dev->keep_format)) + return 0; + tpf = &dev->capture_param.timeperframe; + + fourcc2str(dev->pix_format.pixelformat, buf4cc); + if (tpf->numerator == 1) + snprintf(buf_fps, sizeof(buf_fps), "%u", tpf->denominator); + else + snprintf(buf_fps, sizeof(buf_fps), "%u/%u", tpf->denominator, + tpf->numerator); + return sprintf(buf, "%4s:%ux%u@%s\n", buf4cc, dev->pix_format.width, + dev->pix_format.height, buf_fps); +} + +static ssize_t attr_store_format(struct device *cd, + struct device_attribute *attr, const char *buf, + size_t len) +{ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + int fps_num = 0, fps_den = 1; + + if (!dev) + return -ENODEV; + + /* only fps changing is supported */ + if (sscanf(buf, "@%u/%u", &fps_num, &fps_den) > 0) { + struct v4l2_fract f = { .numerator = fps_den, + .denominator = fps_num }; + set_timeperframe(dev, &f); + return len; + } + return -EINVAL; +} + +static DEVICE_ATTR(format, S_IRUGO | S_IWUSR, attr_show_format, + attr_store_format); + +static ssize_t attr_show_buffers(struct device *cd, + struct device_attribute *attr, char *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + + if (!dev) + return -ENODEV; + + return sprintf(buf, "%u\n", dev->used_buffer_count); +} + +static DEVICE_ATTR(buffers, S_IRUGO, attr_show_buffers, NULL); + +static ssize_t attr_show_maxopeners(struct device *cd, + struct device_attribute *attr, char *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + + if (!dev) + return -ENODEV; + + return sprintf(buf, "%d\n", dev->max_openers); +} + +static ssize_t attr_store_maxopeners(struct device *cd, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct v4l2_loopback_device *dev = NULL; + unsigned long curr = 0; + + if (kstrtoul(buf, 0, &curr)) + return -EINVAL; + + dev = v4l2loopback_cd2dev(cd); + if (!dev) + return -ENODEV; + + if (dev->max_openers == curr) + return len; + + if (curr > __INT_MAX__ || dev->open_count.counter > curr) { + /* request to limit to less openers as are currently attached to us */ + return -EINVAL; + } + + dev->max_openers = (int)curr; + + return len; +} + +static DEVICE_ATTR(max_openers, S_IRUGO | S_IWUSR, attr_show_maxopeners, + attr_store_maxopeners); + +static ssize_t attr_show_state(struct device *cd, struct device_attribute *attr, + char *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + + if (!dev) + return -ENODEV; + + if (!has_output_token(dev->stream_tokens) || dev->keep_format) { + return sprintf(buf, "capture\n"); + } else + return sprintf(buf, "output\n"); + + return -EAGAIN; +} + +static DEVICE_ATTR(state, S_IRUGO, attr_show_state, NULL); + +static void v4l2loopback_remove_sysfs(struct video_device *vdev) +{ +#define V4L2_SYSFS_DESTROY(x) device_remove_file(&vdev->dev, &dev_attr_##x) + + if (vdev) { + V4L2_SYSFS_DESTROY(format); + V4L2_SYSFS_DESTROY(buffers); + V4L2_SYSFS_DESTROY(max_openers); + V4L2_SYSFS_DESTROY(state); + /* ... */ + } +} + +static void v4l2loopback_create_sysfs(struct video_device *vdev) +{ + int res = 0; + +#define V4L2_SYSFS_CREATE(x) \ + res = device_create_file(&vdev->dev, &dev_attr_##x); \ + if (res < 0) \ + break + if (!vdev) + return; + do { + V4L2_SYSFS_CREATE(format); + V4L2_SYSFS_CREATE(buffers); + V4L2_SYSFS_CREATE(max_openers); + V4L2_SYSFS_CREATE(state); + /* ... */ + } while (0); + + if (res >= 0) + return; + dev_err(&vdev->dev, "%s error: %d\n", __func__, res); +} + +/* Event APIs */ + +#define V4L2LOOPBACK_EVENT_BASE (V4L2_EVENT_PRIVATE_START) +#define V4L2LOOPBACK_EVENT_OFFSET 0x08E00000 +#define V4L2_EVENT_PRI_CLIENT_USAGE \ + (V4L2LOOPBACK_EVENT_BASE + V4L2LOOPBACK_EVENT_OFFSET + 1) + +struct v4l2_event_client_usage { + __u32 count; +}; + +/* global module data */ +/* find a device based on it's device-number (e.g. '3' for /dev/video3) */ +struct v4l2loopback_lookup_cb_data { + int device_nr; + struct v4l2_loopback_device *device; +}; +static int v4l2loopback_lookup_cb(int id, void *ptr, void *data) +{ + struct v4l2_loopback_device *device = ptr; + struct v4l2loopback_lookup_cb_data *cbdata = data; + if (cbdata && device && device->vdev) { + if (device->vdev->num == cbdata->device_nr) { + cbdata->device = device; + cbdata->device_nr = id; + return 1; + } + } + return 0; +} +static int v4l2loopback_lookup(int device_nr, + struct v4l2_loopback_device **device) +{ + struct v4l2loopback_lookup_cb_data data = { + .device_nr = device_nr, + .device = NULL, + }; + int err = idr_for_each(&v4l2loopback_index_idr, &v4l2loopback_lookup_cb, + &data); + if (1 == err) { + if (device) + *device = data.device; + return data.device_nr; + } + return -ENODEV; +} +#define v4l2loopback_get_vdev_nr(vdev) \ + ((struct v4l2loopback_private *)video_get_drvdata(vdev))->device_nr +static struct v4l2_loopback_device *v4l2loopback_cd2dev(struct device *cd) +{ + struct video_device *loopdev = to_video_device(cd); + int device_nr = v4l2loopback_get_vdev_nr(loopdev); + + return idr_find(&v4l2loopback_index_idr, device_nr); +} + +static struct v4l2_loopback_device *v4l2loopback_getdevice(struct file *f) +{ + struct v4l2loopback_private *ptr = video_drvdata(f); + int nr = ptr->device_nr; + + return idr_find(&v4l2loopback_index_idr, nr); +} + +/* forward declarations */ +static void client_usage_queue_event(struct video_device *vdev); +static bool any_mapped_buffer(struct v4l2_loopback_device *dev); +static void get_buffer(struct v4l2l_buffer *buf); +static void put_buffer(struct v4l2l_buffer *buf); +static int allocate_buffers(struct v4l2_loopback_device *dev, + struct v4l2_pix_format *pix_format); +static void init_buffers(struct v4l2_loopback_device *dev, u32 bytes_used, + u32 buffer_size); +static void free_buffers(struct v4l2_loopback_device *dev); +static int allocate_timeout_buffer(struct v4l2_loopback_device *dev); +static void free_timeout_buffer(struct v4l2_loopback_device *dev); +static void check_timers(struct v4l2_loopback_device *dev); +static const struct v4l2_file_operations v4l2_loopback_fops; +static const struct v4l2_ioctl_ops v4l2_loopback_ioctl_ops; + +/* V4L2 ioctl caps and params calls */ +/* returns device capabilities + * called on VIDIOC_QUERYCAP + */ +static int vidioc_querycap(struct file *file, void *fh, + struct v4l2_capability *cap) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + int device_nr = v4l2loopback_get_vdev_nr(dev->vdev); + __u32 capabilities = V4L2_CAP_STREAMING | V4L2_CAP_READWRITE; + + strscpy(cap->driver, "v4l2 loopback", sizeof(cap->driver)); + snprintf(cap->card, sizeof(cap->card), "%s", dev->card_label); + snprintf(cap->bus_info, sizeof(cap->bus_info), + "platform:v4l2loopback-%03d", device_nr); + + if (dev->announce_all_caps) { + capabilities |= V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_VIDEO_OUTPUT; + } else { + if (opener->io_method == V4L2L_IO_TIMEOUT || + (has_output_token(dev->stream_tokens) && + !dev->keep_format)) { + capabilities |= V4L2_CAP_VIDEO_OUTPUT; + } else + capabilities |= V4L2_CAP_VIDEO_CAPTURE; + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) + dev->vdev->device_caps = +#endif /* >=linux-4.7.0 */ + cap->device_caps = cap->capabilities = capabilities; + + cap->capabilities |= V4L2_CAP_DEVICE_CAPS; + + memset(cap->reserved, 0, sizeof(cap->reserved)); + return 0; +} + +static int vidioc_enum_framesizes(struct file *file, void *fh, + struct v4l2_frmsizeenum *argp) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + + /* there can be only one... */ + if (argp->index) + return -EINVAL; + + if (dev->keep_format || has_other_owners(opener, dev)) { + /* only current frame size supported */ + if (argp->pixel_format != dev->pix_format.pixelformat) + return -EINVAL; + + argp->type = V4L2_FRMSIZE_TYPE_DISCRETE; + + argp->discrete.width = dev->pix_format.width; + argp->discrete.height = dev->pix_format.height; + } else { + /* return continuous sizes if pixel format is supported */ + if (NULL == format_by_fourcc(argp->pixel_format)) + return -EINVAL; + + if (dev->min_width == dev->max_width && + dev->min_height == dev->max_height) { + argp->type = V4L2_FRMSIZE_TYPE_DISCRETE; + + argp->discrete.width = dev->min_width; + argp->discrete.height = dev->min_height; + } else { + argp->type = V4L2_FRMSIZE_TYPE_CONTINUOUS; + + argp->stepwise.min_width = dev->min_width; + argp->stepwise.min_height = dev->min_height; + + argp->stepwise.max_width = dev->max_width; + argp->stepwise.max_height = dev->max_height; + + argp->stepwise.step_width = 1; + argp->stepwise.step_height = 1; + } + } + return 0; +} + +/* Test if the device is currently 'capable' of the buffer (stream) type when + * the `exclusive_caps` parameter is set. `keep_format` should lock the format + * and prevent free of buffers */ +static int check_buffer_capability(struct v4l2_loopback_device *dev, + struct v4l2_loopback_opener *opener, + enum v4l2_buf_type type) +{ + /* short-circuit for (non-compliant) timeout image mode */ + if (opener->io_method == V4L2L_IO_TIMEOUT) + return 0; + if (dev->announce_all_caps) + return (type == V4L2_BUF_TYPE_VIDEO_CAPTURE || + type == V4L2_BUF_TYPE_VIDEO_OUTPUT) ? + 0 : + -EINVAL; + /* CAPTURE if opener has a capture format or a writer is streaming; + * else OUTPUT. */ + switch (type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + if (!(has_capture_token(opener->format_token) || + !has_output_token(dev->stream_tokens))) + return -EINVAL; + break; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + if (!(has_output_token(opener->format_token) || + has_output_token(dev->stream_tokens))) + return -EINVAL; + break; + default: + return -EINVAL; + } + return 0; +} +/* returns frameinterval (fps) for the set resolution + * called on VIDIOC_ENUM_FRAMEINTERVALS + */ +static int vidioc_enum_frameintervals(struct file *file, void *fh, + struct v4l2_frmivalenum *argp) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + + /* there can be only one... */ + if (argp->index) + return -EINVAL; + + if (dev->keep_format || has_other_owners(opener, dev)) { + /* keep_format also locks the frame rate */ + if (argp->width != dev->pix_format.width || + argp->height != dev->pix_format.height || + argp->pixel_format != dev->pix_format.pixelformat) + return -EINVAL; + + argp->type = V4L2_FRMIVAL_TYPE_DISCRETE; + argp->discrete = dev->capture_param.timeperframe; + } else { + if (argp->width < dev->min_width || + argp->width > dev->max_width || + argp->height < dev->min_height || + argp->height > dev->max_height || + !format_by_fourcc(argp->pixel_format)) + return -EINVAL; + + argp->type = V4L2_FRMIVAL_TYPE_CONTINUOUS; + argp->stepwise.min.numerator = 1; + argp->stepwise.min.denominator = V4L2LOOPBACK_FPS_MAX; + argp->stepwise.max.numerator = V4L2LOOPBACK_FRAME_INTERVAL_MAX; + argp->stepwise.max.denominator = 1; + argp->stepwise.step.numerator = 1; + argp->stepwise.step.denominator = 1; + } + + return 0; +} + +/* Enumerate device formats + * Returns: + * - EINVAL the index is out of bounds; or if non-zero when format is fixed + * - EFAULT unexpected null pointer */ +static int vidioc_enum_fmt_vid(struct file *file, void *fh, + struct v4l2_fmtdesc *f) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + int fixed = dev->keep_format || has_other_owners(opener, dev); + const struct v4l2l_format *fmt; + + if (check_buffer_capability(dev, opener, f->type) < 0) + return -EINVAL; + + if (!(f->index < FORMATS)) + return -EINVAL; + /* TODO: Support 6.14 V4L2_FMTDESC_FLAG_ENUM_ALL */ + if (fixed && f->index) + return -EINVAL; + + fmt = fixed ? format_by_fourcc(dev->pix_format.pixelformat) : + &formats[f->index]; + if (!fmt) + return -EFAULT; + + f->flags = 0; + if (fmt->flags & FORMAT_FLAGS_COMPRESSED) + f->flags |= V4L2_FMT_FLAG_COMPRESSED; + snprintf(f->description, sizeof(f->description), fmt->name); + f->pixelformat = fmt->fourcc; + return 0; +} + +/* Tests (or tries) the format. + * Returns: + * - EINVAL if the buffer type or format is not supported + */ +static int vidioc_try_fmt_vid(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + + if (check_buffer_capability(dev, opener, f->type) < 0) + return -EINVAL; + if (v4l2l_fill_format(f, dev->min_width, dev->max_width, + dev->min_height, dev->max_height) != 0) + return -EINVAL; + if (dev->keep_format || has_other_owners(opener, dev)) + /* use existing format - including colorspace info */ + f->fmt.pix = dev->pix_format; + + return 0; +} + +/* Sets new format. Fills 'f' argument with the requested or existing format. + * Side-effect: buffers are allocated for the (returned) format. + * Returns: + * - EINVAL if the type is not supported + * - EBUSY if buffers are already allocated + * TODO: (vasaka) set subregions of input + */ +static int vidioc_s_fmt_vid(struct file *file, void *fh, struct v4l2_format *f) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + u32 token = opener->io_method == V4L2L_IO_TIMEOUT ? + V4L2L_TOKEN_TIMEOUT : + token_from_type(f->type); + int changed, result; + char buf[5]; + + result = vidioc_try_fmt_vid(file, fh, f); + if (result < 0) + return result; + + if (opener->buffer_count > 0) + /* must free buffers before format can be set */ + return -EBUSY; + + result = mutex_lock_killable(&dev->image_mutex); + if (result < 0) + return result; + + if (opener->format_token) + release_token(dev, opener, format); + if (!(dev->format_tokens & token)) { + result = -EBUSY; + goto exit_s_fmt_unlock; + } + + dprintk("S_FMT[%s] %4s:%ux%u size=%u\n", + V4L2_TYPE_IS_CAPTURE(f->type) ? "CAPTURE" : "OUTPUT", + fourcc2str(f->fmt.pix.pixelformat, buf), f->fmt.pix.width, + f->fmt.pix.height, f->fmt.pix.sizeimage); + changed = !pix_format_eq(&dev->pix_format, &f->fmt.pix, 0); + if (changed || has_no_owners(dev)) { + result = allocate_buffers(dev, &f->fmt.pix); + if (result < 0) + goto exit_s_fmt_unlock; + } + if ((dev->timeout_image && changed) || + (!dev->timeout_image && need_timeout_buffer(dev, token))) { + result = allocate_timeout_buffer(dev); + if (result < 0) + goto exit_s_fmt_free; + } + if (changed) { + dev->pix_format = f->fmt.pix; + dev->pix_format_has_valid_sizeimage = + v4l2l_pix_format_has_valid_sizeimage(f); + } + acquire_token(dev, opener, format, token); + if (opener->io_method == V4L2L_IO_TIMEOUT) + dev->timeout_image_io = 0; + goto exit_s_fmt_unlock; +exit_s_fmt_free: + free_buffers(dev); +exit_s_fmt_unlock: + mutex_unlock(&dev->image_mutex); + return result; +} + +/* ------------------ CAPTURE ----------------------- */ +/* ioctl for VIDIOC_ENUM_FMT, _G_FMT, _S_FMT, and _TRY_FMT when buffer type + * is V4L2_BUF_TYPE_VIDEO_CAPTURE */ + +static int vidioc_enum_fmt_cap(struct file *file, void *fh, + struct v4l2_fmtdesc *f) +{ + return vidioc_enum_fmt_vid(file, fh, f); +} + +static int vidioc_g_fmt_cap(struct file *file, void *fh, struct v4l2_format *f) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + if (check_buffer_capability(dev, opener, f->type) < 0) + return -EINVAL; + f->fmt.pix = dev->pix_format; + return 0; +} + +static int vidioc_try_fmt_cap(struct file *file, void *fh, + struct v4l2_format *f) +{ + return vidioc_try_fmt_vid(file, fh, f); +} + +static int vidioc_s_fmt_cap(struct file *file, void *fh, struct v4l2_format *f) +{ + return vidioc_s_fmt_vid(file, fh, f); +} + +/* ------------------ OUTPUT ----------------------- */ +/* ioctl for VIDIOC_ENUM_FMT, _G_FMT, _S_FMT, and _TRY_FMT when buffer type + * is V4L2_BUF_TYPE_VIDEO_OUTPUT */ + +static int vidioc_enum_fmt_out(struct file *file, void *fh, + struct v4l2_fmtdesc *f) +{ + return vidioc_enum_fmt_vid(file, fh, f); +} + +static int vidioc_g_fmt_out(struct file *file, void *fh, struct v4l2_format *f) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + if (check_buffer_capability(dev, opener, f->type) < 0) + return -EINVAL; + /* + * LATER: this should return the currently valid format + * gstreamer doesn't like it, if this returns -EINVAL, as it + * then concludes that there is _no_ valid format + * CHECK whether this assumption is wrong, + * or whether we have to always provide a valid format + */ + f->fmt.pix = dev->pix_format; + return 0; +} + +static int vidioc_try_fmt_out(struct file *file, void *fh, + struct v4l2_format *f) +{ + return vidioc_try_fmt_vid(file, fh, f); +} + +static int vidioc_s_fmt_out(struct file *file, void *fh, struct v4l2_format *f) +{ + return vidioc_s_fmt_vid(file, fh, f); +} + +// #define V4L2L_OVERLAY +#ifdef V4L2L_OVERLAY +/* ------------------ OVERLAY ----------------------- */ +/* currently unsupported */ +/* GSTreamer's v4l2sink is buggy, as it requires the overlay to work + * while it should only require it, if overlay is requested + * once the gstreamer element is fixed, remove the overlay dummies + */ +#warning OVERLAY dummies +static int vidioc_g_fmt_overlay(struct file *file, void *priv, + struct v4l2_format *fmt) +{ + return 0; +} + +static int vidioc_s_fmt_overlay(struct file *file, void *priv, + struct v4l2_format *fmt) +{ + return 0; +} +#endif /* V4L2L_OVERLAY */ + +/* ------------------ PARAMs ----------------------- */ + +/* get some data flow parameters, only capability, fps and readbuffers has + * effect on this driver + * called on VIDIOC_G_PARM + */ +static int vidioc_g_parm(struct file *file, void *fh, + struct v4l2_streamparm *parm) +{ + /* do not care about type of opener, hope these enums would always be + * compatible */ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + if (check_buffer_capability(dev, opener, parm->type) < 0) + return -EINVAL; + parm->parm.capture = dev->capture_param; + return 0; +} + +/* get some data flow parameters, only capability, fps and readbuffers has + * effect on this driver + * called on VIDIOC_S_PARM + */ +static int vidioc_s_parm(struct file *file, void *fh, + struct v4l2_streamparm *parm) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + + dprintk("S_PARM(frame-time=%u/%u)\n", + parm->parm.capture.timeperframe.numerator, + parm->parm.capture.timeperframe.denominator); + if (check_buffer_capability(dev, opener, parm->type) < 0) + return -EINVAL; + + switch (parm->type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + set_timeperframe(dev, &parm->parm.capture.timeperframe); + break; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + set_timeperframe(dev, &parm->parm.output.timeperframe); + break; + default: + return -EINVAL; + } + + parm->parm.capture = dev->capture_param; + return 0; +} + +#ifdef V4L2LOOPBACK_WITH_STD +/* sets a tv standard, actually we do not need to handle this any special way + * added to support effecttv + * called on VIDIOC_S_STD + */ +static int vidioc_s_std(struct file *file, void *fh, v4l2_std_id *_std) +{ + v4l2_std_id req_std = 0, supported_std = 0; + const v4l2_std_id all_std = V4L2_STD_ALL, no_std = 0; + + if (_std) { + req_std = *_std; + *_std = all_std; + } + + /* we support everything in V4L2_STD_ALL, but not more... */ + supported_std = (all_std & req_std); + if (no_std == supported_std) + return -EINVAL; + + return 0; +} + +/* gets a fake video standard + * called on VIDIOC_G_STD + */ +static int vidioc_g_std(struct file *file, void *fh, v4l2_std_id *norm) +{ + if (norm) + *norm = V4L2_STD_ALL; + return 0; +} +/* gets a fake video standard + * called on VIDIOC_QUERYSTD + */ +static int vidioc_querystd(struct file *file, void *fh, v4l2_std_id *norm) +{ + if (norm) + *norm = V4L2_STD_ALL; + return 0; +} +#endif /* V4L2LOOPBACK_WITH_STD */ + +static int v4l2loopback_set_ctrl(struct v4l2_loopback_device *dev, u32 id, + s64 val) +{ + int result = 0; + switch (id) { + case CID_KEEP_FORMAT: + if (val < 0 || val > 1) + return -EINVAL; + dev->keep_format = val; + result = mutex_lock_killable(&dev->image_mutex); + if (result < 0) + return result; + if (!dev->keep_format) { + if (has_no_owners(dev) && !any_mapped_buffer(dev)) + free_buffers(dev); + } + mutex_unlock(&dev->image_mutex); + break; + case CID_SUSTAIN_FRAMERATE: + if (val < 0 || val > 1) + return -EINVAL; + spin_lock_bh(&dev->lock); + dev->sustain_framerate = val; + check_timers(dev); + spin_unlock_bh(&dev->lock); + break; + case CID_TIMEOUT: + if (val < 0 || val > MAX_TIMEOUT) + return -EINVAL; + if (val > 0) { + result = mutex_lock_killable(&dev->image_mutex); + if (result < 0) + return result; + /* on-the-fly allocate if device is owned; else + * allocate occurs on next S_FMT or REQBUFS */ + if (!has_no_owners(dev)) + result = allocate_timeout_buffer(dev); + mutex_unlock(&dev->image_mutex); + if (result < 0) { + /* disable timeout as buffer not alloc'd */ + spin_lock_bh(&dev->lock); + dev->timeout_jiffies = 0; + spin_unlock_bh(&dev->lock); + return result; + } + } + spin_lock_bh(&dev->lock); + dev->timeout_jiffies = msecs_to_jiffies(val); + check_timers(dev); + spin_unlock_bh(&dev->lock); + break; + case CID_TIMEOUT_IMAGE_IO: + dev->timeout_image_io = 1; + break; + default: + return -EINVAL; + } + return 0; +} + +static int v4l2loopback_s_ctrl(struct v4l2_ctrl *ctrl) +{ + struct v4l2_loopback_device *dev = container_of( + ctrl->handler, struct v4l2_loopback_device, ctrl_handler); + return v4l2loopback_set_ctrl(dev, ctrl->id, ctrl->val); +} + +/* returns set of device outputs, in our case there is only one + * called on VIDIOC_ENUMOUTPUT + */ +static int vidioc_enum_output(struct file *file, void *fh, + struct v4l2_output *outp) +{ + __u32 index = outp->index; + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_OUTPUT)) + return -ENOTTY; + if (index) + return -EINVAL; + + /* clear all data (including the reserved fields) */ + memset(outp, 0, sizeof(*outp)); + + outp->index = index; + strscpy(outp->name, "loopback in", sizeof(outp->name)); + outp->type = V4L2_OUTPUT_TYPE_ANALOG; + outp->audioset = 0; + outp->modulator = 0; +#ifdef V4L2LOOPBACK_WITH_STD + outp->std = V4L2_STD_ALL; +#ifdef V4L2_OUT_CAP_STD + outp->capabilities |= V4L2_OUT_CAP_STD; +#endif /* V4L2_OUT_CAP_STD */ +#endif /* V4L2LOOPBACK_WITH_STD */ + + return 0; +} + +/* which output is currently active, + * called on VIDIOC_G_OUTPUT + */ +static int vidioc_g_output(struct file *file, void *fh, unsigned int *index) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_OUTPUT)) + return -ENOTTY; + if (index) + *index = 0; + return 0; +} + +/* set output, can make sense if we have more than one video src, + * called on VIDIOC_S_OUTPUT + */ +static int vidioc_s_output(struct file *file, void *fh, unsigned int index) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_OUTPUT)) + return -ENOTTY; + return index == 0 ? index : -EINVAL; +} + +/* returns set of device inputs, in our case there is only one, + * but later I may add more + * called on VIDIOC_ENUMINPUT + */ +static int vidioc_enum_input(struct file *file, void *fh, + struct v4l2_input *inp) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + __u32 index = inp->index; + + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_CAPTURE)) + return -ENOTTY; + if (index) + return -EINVAL; + + /* clear all data (including the reserved fields) */ + memset(inp, 0, sizeof(*inp)); + + inp->index = index; + strscpy(inp->name, "loopback", sizeof(inp->name)); + inp->type = V4L2_INPUT_TYPE_CAMERA; + inp->audioset = 0; + inp->tuner = 0; + inp->status = 0; + +#ifdef V4L2LOOPBACK_WITH_STD + inp->std = V4L2_STD_ALL; +#ifdef V4L2_IN_CAP_STD + inp->capabilities |= V4L2_IN_CAP_STD; +#endif +#endif /* V4L2LOOPBACK_WITH_STD */ + + if (has_output_token(dev->stream_tokens) && !dev->keep_format) + /* if no outputs attached; pretend device is powered off */ + inp->status |= V4L2_IN_ST_NO_SIGNAL; + + return 0; +} + +/* which input is currently active, + * called on VIDIOC_G_INPUT + */ +static int vidioc_g_input(struct file *file, void *fh, unsigned int *index) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_CAPTURE)) + return -ENOTTY; /* NOTE: -EAGAIN might be more informative */ + if (index) + *index = 0; + return 0; +} + +/* set input, can make sense if we have more than one video src, + * called on VIDIOC_S_INPUT + */ +static int vidioc_s_input(struct file *file, void *fh, unsigned int index) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + if (index != 0) + return -EINVAL; + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_CAPTURE)) + return -ENOTTY; /* NOTE: -EAGAIN might be more informative */ + return 0; +} + +/* --------------- V4L2 ioctl buffer related calls ----------------- */ + +#define is_allocated(opener, type, index) \ + (opener->format_token & (opener->io_method == V4L2L_IO_TIMEOUT ? \ + V4L2L_TOKEN_TIMEOUT : \ + token_from_type(type)) && \ + (index) < (opener)->buffer_count) +#define BUFFER_DEBUG_FMT_STR \ + "buffer#%u @ %p type=%u bytesused=%u length=%u flags=%x " \ + "field=%u timestamp= %lld.%06lldsequence=%u\n" +#define BUFFER_DEBUG_FMT_ARGS(buf) \ + (buf)->index, (buf), (buf)->type, (buf)->bytesused, (buf)->length, \ + (buf)->flags, (buf)->field, \ + (long long)(buf)->timestamp.tv_sec, \ + (long long)(buf)->timestamp.tv_usec, (buf)->sequence +/* Buffer flag helpers */ +#define unset_flags(flags) \ + do { \ + flags &= ~V4L2_BUF_FLAG_QUEUED; \ + flags &= ~V4L2_BUF_FLAG_DONE; \ + } while (0) +#define set_queued(flags) \ + do { \ + flags |= V4L2_BUF_FLAG_QUEUED; \ + flags &= ~V4L2_BUF_FLAG_DONE; \ + } while (0) +#define set_done(flags) \ + do { \ + flags &= ~V4L2_BUF_FLAG_QUEUED; \ + flags |= V4L2_BUF_FLAG_DONE; \ + } while (0) + +static bool any_mapped_buffer(struct v4l2_loopback_device *dev) +{ + u32 index; + for (index = 0; index < dev->buffer_count; ++index) + if (dev->buffers[index].buffer.flags & V4L2_BUF_FLAG_MAPPED) + return true; + return false; +} + +static void get_buffer(struct v4l2l_buffer *buf) +{ + if (atomic_inc_return(&buf->use_count)) + buf->buffer.flags |= V4L2_BUF_FLAG_MAPPED; +} + +static void put_buffer(struct v4l2l_buffer *buf) +{ + if (atomic_dec_and_test(&buf->use_count)) + buf->buffer.flags &= ~V4L2_BUF_FLAG_MAPPED; +} + +static void prepare_buffer_queue(struct v4l2_loopback_device *dev, int count) +{ + struct v4l2l_buffer *bufd, *n; + u32 pos; + + spin_lock_bh(&dev->list_lock); + + /* ensure sufficient number of buffers in queue */ + for (pos = 0; pos < count; ++pos) { + bufd = &dev->buffers[pos]; + if (list_empty(&bufd->list_head)) + list_add_tail(&bufd->list_head, &dev->outbufs_list); + } + if (list_empty(&dev->outbufs_list)) + goto exit_prepare_queue_unlock; + + /* remove any excess buffers */ + list_for_each_entry_safe(bufd, n, &dev->outbufs_list, list_head) { + if (bufd->buffer.index >= count) + list_del_init(&bufd->list_head); + } + + /* buffers are no longer queued; and `write_position` will correspond + * to the first item of `outbufs_list`. */ + pos = v4l2l_mod64(dev->write_position, count); + list_for_each_entry(bufd, &dev->outbufs_list, list_head) { + unset_flags(bufd->buffer.flags); + dev->bufpos2index[pos % count] = bufd->buffer.index; + ++pos; + } +exit_prepare_queue_unlock: + spin_unlock_bh(&dev->list_lock); +} + +/* forward declaration */ +static int vidioc_streamoff(struct file *file, void *fh, + enum v4l2_buf_type type); +/* negotiate buffer type + * only mmap streaming supported + * called on VIDIOC_REQBUFS + */ +static int vidioc_reqbufs(struct file *file, void *fh, + struct v4l2_requestbuffers *reqbuf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + u32 token = opener->io_method == V4L2L_IO_TIMEOUT ? + V4L2L_TOKEN_TIMEOUT : + token_from_type(reqbuf->type); + u32 req_count = reqbuf->count; + int result = 0; + + dprintk("REQBUFS(memory=%u, req_count=%u) and device-bufs=%u/%u " + "[used/max]\n", + reqbuf->memory, req_count, dev->used_buffer_count, + dev->buffer_count); + + if (req_count > dev->buffer_count) + req_count = dev->buffer_count; + + switch (reqbuf->memory) { + case V4L2_MEMORY_MMAP: +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0) + reqbuf->capabilities = 0; /* only guarantee MMAP support */ +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) + reqbuf->flags = 0; /* no memory consistency support */ +#endif + break; + default: + return -EINVAL; + } + + if (opener->format_token & ~token) + /* different (buffer) type already assigned to descriptor by + * S_FMT or REQBUFS */ + return -EINVAL; + + MARK(); + result = mutex_lock_killable(&dev->image_mutex); + if (result < 0) + return result; /* -EINTR */ + + /* CASE queue/dequeue timeout-buffer only: */ + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) { + opener->buffer_count = req_count; + if (req_count == 0) + release_token(dev, opener, format); + goto exit_reqbufs_unlock; + } + + MARK(); + /* CASE count is zero: streamoff, free buffers, release their token */ + if (req_count == 0) { + if (dev->format_tokens & token) { + acquire_token(dev, opener, format, token); + opener->io_method = V4L2L_IO_MMAP; + } + result = vidioc_streamoff(file, fh, reqbuf->type); + opener->buffer_count = 0; + /* undocumented requirement - REQBUFS with count zero should + * ALSO release lock on logical stream */ + if (opener->format_token) + release_token(dev, opener, format); + if (has_no_owners(dev)) + dev->used_buffer_count = 0; + goto exit_reqbufs_unlock; + } + + /* CASE count non-zero: allocate buffers and acquire token for them */ + MARK(); + switch (reqbuf->type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + if (!(dev->format_tokens & token || + opener->format_token & token)) + /* only exclusive ownership for each stream */ + result = -EBUSY; + break; + default: + result = -EINVAL; + } + if (result < 0) + goto exit_reqbufs_unlock; + + if (has_other_owners(opener, dev) && dev->used_buffer_count > 0) { + /* allow 'allocation' of existing number of buffers */ + req_count = dev->used_buffer_count; + } else if (any_mapped_buffer(dev)) { + /* do not allow re-allocation if buffers are mapped */ + result = -EBUSY; + goto exit_reqbufs_unlock; + } + + MARK(); + opener->buffer_count = 0; + if (has_no_owners(dev)) { + result = allocate_buffers(dev, &dev->pix_format); + if (result < 0) + goto exit_reqbufs_unlock; + } + if (!dev->timeout_image && need_timeout_buffer(dev, token)) { + result = allocate_timeout_buffer(dev); + if (result < 0) + goto exit_reqbufs_unlock; + } + acquire_token(dev, opener, format, token); + + MARK(); + switch (opener->io_method) { + case V4L2L_IO_TIMEOUT: + dev->timeout_image_io = 0; + opener->buffer_count = req_count; + break; + default: + opener->io_method = V4L2L_IO_MMAP; + prepare_buffer_queue(dev, req_count); + dev->used_buffer_count = opener->buffer_count = req_count; + } +exit_reqbufs_unlock: + mutex_unlock(&dev->image_mutex); + reqbuf->count = opener->buffer_count; + return result; +} + +/* returns buffer asked for; + * give app as many buffers as it wants, if it less than MAX, + * but map them in our inner buffers + * called on VIDIOC_QUERYBUF + */ +static int vidioc_querybuf(struct file *file, void *fh, struct v4l2_buffer *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + u32 type = buf->type; + u32 index = buf->index; + + if ((type != V4L2_BUF_TYPE_VIDEO_CAPTURE) && + (type != V4L2_BUF_TYPE_VIDEO_OUTPUT)) + return -EINVAL; + if (!is_allocated(opener, type, index)) + return -EINVAL; + + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) { + *buf = dev->timeout_buffer.buffer; + buf->index = index; + } else + *buf = dev->buffers[index].buffer; + + buf->type = type; + + if (!(buf->flags & (V4L2_BUF_FLAG_DONE | V4L2_BUF_FLAG_QUEUED))) { + /* v4l2-compliance requires these to be zero */ + buf->sequence = 0; + buf->timestamp.tv_sec = buf->timestamp.tv_usec = 0; + } else if (V4L2_TYPE_IS_CAPTURE(type)) { + /* guess flags based on sequence values */ + if (buf->sequence >= opener->read_position) { + set_done(buf->flags); + } else if (buf->flags & V4L2_BUF_FLAG_DONE) { + set_queued(buf->flags); + } + } + dprintkrw("QUERYBUF(%s, index=%u) -> " BUFFER_DEBUG_FMT_STR, + V4L2_TYPE_IS_CAPTURE(type) ? "CAPTURE" : "OUTPUT", index, + BUFFER_DEBUG_FMT_ARGS(buf)); + return 0; +} + +static void buffer_written(struct v4l2_loopback_device *dev, + struct v4l2l_buffer *buf) +{ + timer_delete_sync(&dev->sustain_timer); + timer_delete_sync(&dev->timeout_timer); + + mutex_lock(&dev->image_mutex); + if (dev->used_buffer_count != 0) { + spin_lock_bh(&dev->list_lock); + list_move_tail(&buf->list_head, &dev->outbufs_list); + spin_unlock_bh(&dev->list_lock); + + spin_lock_bh(&dev->lock); + dev->bufpos2index[v4l2l_mod64(dev->write_position, + dev->used_buffer_count)] = + buf->buffer.index; + + ++dev->write_position; + dev->reread_count = 0; + spin_unlock_bh(&dev->lock); + } + mutex_unlock(&dev->image_mutex); + + spin_lock_bh(&dev->lock); + check_timers(dev); + spin_unlock_bh(&dev->lock); +} + +/* put buffer to queue + * called on VIDIOC_QBUF + */ +static int vidioc_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + struct v4l2l_buffer *bufd; + u32 index = buf->index; + u32 type = buf->type; + + if (!is_allocated(opener, type, index)) + return -EINVAL; + bufd = &dev->buffers[index]; + + switch (buf->memory) { + case V4L2_MEMORY_MMAP: + if (!(bufd->buffer.flags & V4L2_BUF_FLAG_MAPPED)) + dprintkrw("QBUF() unmapped buffer [index=%u]\n", index); + break; + default: + return -EINVAL; + } + + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) { + set_queued(buf->flags); + return 0; + } + + switch (type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + dprintkrw("QBUF(CAPTURE, index=%u) -> " BUFFER_DEBUG_FMT_STR, + index, BUFFER_DEBUG_FMT_ARGS(buf)); + set_queued(buf->flags); + break; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + dprintkrw("QBUF(OUTPUT, index=%u) -> " BUFFER_DEBUG_FMT_STR, + index, BUFFER_DEBUG_FMT_ARGS(buf)); + if (!(bufd->buffer.flags & V4L2_BUF_FLAG_TIMESTAMP_COPY) && + (buf->timestamp.tv_sec == 0 && + buf->timestamp.tv_usec == 0)) { + v4l2l_get_timestamp(&bufd->buffer); + } else { + bufd->buffer.timestamp = buf->timestamp; + bufd->buffer.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; + bufd->buffer.flags &= + ~V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; + } + if (dev->pix_format_has_valid_sizeimage) { + if (buf->bytesused >= dev->pix_format.sizeimage) { + bufd->buffer.bytesused = + dev->pix_format.sizeimage; + } else { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0) + dev_warn_ratelimited( + &dev->vdev->dev, +#else + dprintkrw( +#endif + "warning queued output buffer bytesused too small %u < %u\n", + buf->bytesused, + dev->pix_format.sizeimage); + bufd->buffer.bytesused = buf->bytesused; + } + } else { + bufd->buffer.bytesused = buf->bytesused; + } + bufd->buffer.sequence = dev->write_position; + set_queued(bufd->buffer.flags); + *buf = bufd->buffer; + buffer_written(dev, bufd); + set_done(bufd->buffer.flags); + wake_up_all(&dev->read_event); + break; + default: + return -EINVAL; + } + buf->type = type; + return 0; +} + +static int can_read(struct v4l2_loopback_device *dev, + struct v4l2_loopback_opener *opener) +{ + int ret; + + spin_lock_bh(&dev->lock); + check_timers(dev); + ret = dev->write_position > opener->read_position || + dev->reread_count > opener->reread_count || dev->timeout_happened; + spin_unlock_bh(&dev->lock); + return ret; +} + +static int get_capture_buffer(struct file *file) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = + v4l2l_f_to_opener(file, file->private_data); + int pos, timeout_happened; + u32 index; + + if ((file->f_flags & O_NONBLOCK) && + (dev->write_position <= opener->read_position && + dev->reread_count <= opener->reread_count && + !dev->timeout_happened)) + return -EAGAIN; + wait_event_interruptible(dev->read_event, can_read(dev, opener)); + + mutex_lock(&dev->image_mutex); + if (!dev->image || dev->used_buffer_count == 0) { + mutex_unlock(&dev->image_mutex); + return -EINVAL; + } + spin_lock_bh(&dev->lock); + if (dev->write_position == opener->read_position) { + if (dev->reread_count > opener->reread_count + 2) + opener->reread_count = dev->reread_count - 1; + ++opener->reread_count; + pos = v4l2l_mod64(opener->read_position + + dev->used_buffer_count - 1, + dev->used_buffer_count); + } else { + opener->reread_count = 0; + if (dev->write_position > + opener->read_position + dev->used_buffer_count) + opener->read_position = dev->write_position - 1; + pos = v4l2l_mod64(opener->read_position, + dev->used_buffer_count); + ++opener->read_position; + } + timeout_happened = dev->timeout_happened && (dev->timeout_jiffies > 0); + dev->timeout_happened = 0; + index = dev->bufpos2index[pos]; + if (timeout_happened) + get_buffer(&dev->buffers[index]); + + spin_unlock_bh(&dev->lock); + mutex_unlock(&dev->image_mutex); + + if (timeout_happened) { + /* although allocated on-demand, timeout_image is freed only + * in free_buffers(), so we don't need to worry about it being + * deallocated suddenly */ + memcpy(dev->image + dev->buffers[index].buffer.m.offset, + dev->timeout_image, dev->buffer_size); + put_buffer(&dev->buffers[index]); + } + return (int)index; +} + +/* put buffer to dequeue + * called on VIDIOC_DQBUF + */ +static int vidioc_dqbuf(struct file *file, void *fh, struct v4l2_buffer *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + u32 type = buf->type; + int index; + struct v4l2l_buffer *bufd; + + if (buf->memory != V4L2_MEMORY_MMAP) + return -EINVAL; + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) { + *buf = dev->timeout_buffer.buffer; + buf->type = type; + unset_flags(buf->flags); + return 0; + } + if ((opener->buffer_count == 0) || + !(opener->format_token & token_from_type(type))) + return -EINVAL; + + switch (type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + index = get_capture_buffer(file); + if (index < 0) + return index; + *buf = dev->buffers[index].buffer; + unset_flags(buf->flags); + break; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + spin_lock_bh(&dev->list_lock); + + bufd = list_first_entry_or_null(&dev->outbufs_list, + struct v4l2l_buffer, list_head); + if (bufd) + list_move_tail(&bufd->list_head, &dev->outbufs_list); + + spin_unlock_bh(&dev->list_lock); + if (!bufd) + return -EFAULT; + unset_flags(bufd->buffer.flags); + *buf = bufd->buffer; + break; + default: + return -EINVAL; + } + + buf->type = type; + dprintkrw("DQBUF(%s, index=%u) -> " BUFFER_DEBUG_FMT_STR, + V4L2_TYPE_IS_CAPTURE(type) ? "CAPTURE" : "OUTPUT", index, + BUFFER_DEBUG_FMT_ARGS(buf)); + return 0; +} + +/* ------------- STREAMING ------------------- */ + +/* start streaming + * called on VIDIOC_STREAMON + */ +static int vidioc_streamon(struct file *file, void *fh, enum v4l2_buf_type type) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + u32 token = token_from_type(type); + + /* short-circuit when using timeout buffer set */ + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) + return 0; + /* opener must have claimed (same) buffer set via REQBUFS */ + if (!opener->buffer_count || !(opener->format_token & token)) + return -EINVAL; + + switch (type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + if (has_output_token(dev->stream_tokens) && !dev->keep_format) + return -EIO; + if (dev->stream_tokens & token) { + acquire_token(dev, opener, stream, token); + client_usage_queue_event(dev->vdev); + } + return 0; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + if (dev->stream_tokens & token) + acquire_token(dev, opener, stream, token); + return 0; + default: + return -EINVAL; + } +} + +/* stop streaming + * called on VIDIOC_STREAMOFF + */ +static int vidioc_streamoff(struct file *file, void *fh, + enum v4l2_buf_type type) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + u32 token = token_from_type(type); + + /* short-circuit when using timeout buffer set */ + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) + return 0; + /* short-circuit when buffer set has no owner */ + if (dev->format_tokens & token) + return 0; + /* opener needs a claim to buffer set */ + if (!opener->format_token) + return -EBUSY; + if (opener->format_token & ~token) + return -EINVAL; + + switch (type) { + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + if (opener->stream_token & token) + release_token(dev, opener, stream); + /* reset output queue */ + if (dev->used_buffer_count > 0) + prepare_buffer_queue(dev, dev->used_buffer_count); + return 0; + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + if (opener->stream_token & token) { + release_token(dev, opener, stream); + client_usage_queue_event(dev->vdev); + } + return 0; + default: + return -EINVAL; + } +} + +#ifdef CONFIG_VIDEO_V4L1_COMPAT +static int vidiocgmbuf(struct file *file, void *fh, struct video_mbuf *p) +{ + struct v4l2_loopback_device *dev; + MARK(); + + dev = v4l2loopback_getdevice(file); + p->frames = dev->buffer_count; + p->offsets[0] = 0; + p->offsets[1] = 0; + p->size = dev->buffer_size; + return 0; +} +#endif + +static void client_usage_queue_event(struct video_device *vdev) +{ + struct v4l2_event ev; + struct v4l2_loopback_device *dev; + + dev = container_of(vdev->v4l2_dev, struct v4l2_loopback_device, + v4l2_dev); + + memset(&ev, 0, sizeof(ev)); + ev.type = V4L2_EVENT_PRI_CLIENT_USAGE; + ((struct v4l2_event_client_usage *)&ev.u)->count = + !has_capture_token(dev->stream_tokens); + + v4l2_event_queue(vdev, &ev); +} + +static int client_usage_ops_add(struct v4l2_subscribed_event *sev, + unsigned elems) +{ + if (!(sev->flags & V4L2_EVENT_SUB_FL_SEND_INITIAL)) + return 0; + + client_usage_queue_event(sev->fh->vdev); + return 0; +} + +static void client_usage_ops_replace(struct v4l2_event *old, + const struct v4l2_event *new) +{ + *((struct v4l2_event_client_usage *)&old->u) = + *((struct v4l2_event_client_usage *)&new->u); +} + +static void client_usage_ops_merge(const struct v4l2_event *old, + struct v4l2_event *new) +{ + *((struct v4l2_event_client_usage *)&new->u) = + *((struct v4l2_event_client_usage *)&old->u); +} + +const struct v4l2_subscribed_event_ops client_usage_ops = { + .add = client_usage_ops_add, + .replace = client_usage_ops_replace, + .merge = client_usage_ops_merge, +}; + +static int vidioc_subscribe_event(struct v4l2_fh *fh, + const struct v4l2_event_subscription *sub) +{ + switch (sub->type) { + case V4L2_EVENT_CTRL: + return v4l2_ctrl_subscribe_event(fh, sub); + case V4L2_EVENT_PRI_CLIENT_USAGE: + return v4l2_event_subscribe(fh, sub, 0, &client_usage_ops); + } + + return -EINVAL; +} + +/* file operations */ +static void vm_open(struct vm_area_struct *vma) +{ + struct v4l2l_buffer *buf = vma->vm_private_data; + MARK(); + get_buffer(buf); +} + +static void vm_close(struct vm_area_struct *vma) +{ + struct v4l2l_buffer *buf = vma->vm_private_data; + MARK(); + put_buffer(buf); +} + +static struct vm_operations_struct vm_ops = { + .open = vm_open, + .close = vm_close, +}; + +static int v4l2_loopback_mmap(struct file *file, struct vm_area_struct *vma) +{ + u8 *addr; + unsigned long start, size, offset; + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = + v4l2l_f_to_opener(file, file->private_data); + struct v4l2l_buffer *buffer = NULL; + int result = 0; + MARK(); + + offset = (unsigned long)vma->vm_pgoff << PAGE_SHIFT; + start = (unsigned long)vma->vm_start; + size = (unsigned long)(vma->vm_end - vma->vm_start); /* always != 0 */ + + /* ensure buffer size, count, and allocated image(s) are not altered by + * other file descriptors */ + result = mutex_lock_killable(&dev->image_mutex); + if (result < 0) + return result; + + if (size > dev->buffer_size) { + dprintk("mmap() attempt to map %lubytes when %ubytes are " + "allocated to buffers\n", + size, dev->buffer_size); + result = -EINVAL; + goto exit_mmap_unlock; + } + if (offset % dev->buffer_size != 0) { + dprintk("mmap() offset does not match start of any buffer\n"); + result = -EINVAL; + goto exit_mmap_unlock; + } + switch (opener->format_token) { + case V4L2L_TOKEN_TIMEOUT: + if (offset != (unsigned long)dev->buffer_size * MAX_BUFFERS) { + dprintk("mmap() incorrect offset for timeout image\n"); + result = -EINVAL; + goto exit_mmap_unlock; + } + buffer = &dev->timeout_buffer; + addr = dev->timeout_image; + break; + default: + if (offset >= dev->image_size) { + dprintk("mmap() attempt to map beyond all buffers\n"); + result = -EINVAL; + goto exit_mmap_unlock; + } + u32 index = offset / dev->buffer_size; + buffer = &dev->buffers[index]; + addr = dev->image + offset; + break; + } + + while (size > 0) { + struct page *page = vmalloc_to_page(addr); + + result = vm_insert_page(vma, start, page); + if (result < 0) + goto exit_mmap_unlock; + + start += PAGE_SIZE; + addr += PAGE_SIZE; + size -= PAGE_SIZE; + } + + vma->vm_ops = &vm_ops; + vma->vm_private_data = buffer; + get_buffer(buffer); + +exit_mmap_unlock: + mutex_unlock(&dev->image_mutex); + return result; +} + +static unsigned int v4l2_loopback_poll(struct file *file, + struct poll_table_struct *pts) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = + v4l2l_f_to_opener(file, file->private_data); + __poll_t req_events = poll_requested_events(pts); + int ret_mask = 0; + + /* call poll_wait in first call, regardless, to ensure that the + * wait-queue is not null */ + poll_wait(file, &dev->read_event, pts); + poll_wait(file, &opener->fh.wait, pts); + + if (req_events & POLLPRI) { + if (v4l2_event_pending(&opener->fh)) { + ret_mask |= POLLPRI; + if (!(req_events & DEFAULT_POLLMASK)) + return ret_mask; + } + } + + switch (opener->format_token) { + case V4L2L_TOKEN_OUTPUT: + if (opener->stream_token != 0 || + opener->io_method == V4L2L_IO_NONE) + ret_mask |= POLLOUT | POLLWRNORM; + break; + case V4L2L_TOKEN_CAPTURE: + if ((opener->io_method == V4L2L_IO_NONE || + opener->stream_token != 0) && + can_read(dev, opener)) + ret_mask |= POLLIN | POLLWRNORM; + break; + case V4L2L_TOKEN_TIMEOUT: + ret_mask |= POLLOUT | POLLWRNORM; + break; + default: + break; + } + + return ret_mask; +} + +/* do not want to limit device opens, it can be as many readers as user want, + * writers are limited by means of setting writer field */ +static int v4l2_loopback_open(struct file *file) +{ + struct v4l2_loopback_device *dev; + struct v4l2_loopback_opener *opener; + + dev = v4l2loopback_getdevice(file); + if (dev->open_count.counter >= dev->max_openers) + return -EBUSY; + /* kfree on close */ + opener = kzalloc(sizeof(*opener), GFP_KERNEL); + if (opener == NULL) + return -ENOMEM; + + atomic_inc(&dev->open_count); + if (dev->timeout_image_io && dev->format_tokens & V4L2L_TOKEN_TIMEOUT) + /* will clear timeout_image_io once buffer set acquired */ + opener->io_method = V4L2L_IO_TIMEOUT; + + v4l2_fh_init(&opener->fh, video_devdata(file)); + file->private_data = &opener->fh; + + v4l2_fh_add(&opener->fh, file); + dprintk("open() -> dev@%p with image@%p\n", dev, + dev ? dev->image : NULL); + return 0; +} + +static int v4l2_loopback_close(struct file *file) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = + v4l2l_f_to_opener(file, file->private_data); + int result = 0; + dprintk("close() -> dev@%p with image@%p\n", dev, + dev ? dev->image : NULL); + + if (opener->format_token) { + struct v4l2_requestbuffers reqbuf = { + .count = 0, .memory = V4L2_MEMORY_MMAP, .type = 0 + }; + switch (opener->format_token) { + case V4L2L_TOKEN_CAPTURE: + reqbuf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + break; + case V4L2L_TOKEN_OUTPUT: + case V4L2L_TOKEN_TIMEOUT: + reqbuf.type = V4L2_BUF_TYPE_VIDEO_OUTPUT; + break; + } + if (reqbuf.type) + result = vidioc_reqbufs(file, file->private_data, + &reqbuf); + if (result < 0) + dprintk("failed to free buffers REQBUFS(count=0) " + " returned %d\n", + result); + mutex_lock(&dev->image_mutex); + release_token(dev, opener, format); + mutex_unlock(&dev->image_mutex); + } + + if (atomic_dec_and_test(&dev->open_count)) { + timer_delete_sync(&dev->sustain_timer); + timer_delete_sync(&dev->timeout_timer); + if (!dev->keep_format) { + mutex_lock(&dev->image_mutex); + free_buffers(dev); + mutex_unlock(&dev->image_mutex); + } + } + + v4l2_fh_del(&opener->fh, file); + v4l2_fh_exit(&opener->fh); + + kfree(opener); + return 0; +} + +static int start_fileio(struct file *file, void *fh, enum v4l2_buf_type type) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = v4l2l_f_to_opener(file, fh); + struct v4l2_requestbuffers reqbuf = { .count = dev->buffer_count, + .memory = V4L2_MEMORY_MMAP, + .type = type }; + int token = token_from_type(type); + int result; + + if (opener->format_token & V4L2L_TOKEN_TIMEOUT || + opener->format_token & ~token) + return -EBUSY; /* NOTE: -EBADF might be more informative */ + + /* short-circuit if already have stream token */ + if (opener->stream_token && opener->io_method == V4L2L_IO_FILE) + return 0; + + /* otherwise attempt to acquire stream token and assign IO method */ + if (!(dev->stream_tokens & token) || opener->io_method != V4L2L_IO_NONE) + return -EBUSY; + + result = vidioc_reqbufs(file, fh, &reqbuf); + if (result < 0) + return result; + result = vidioc_streamon(file, fh, type); + if (result < 0) + return result; + + opener->io_method = V4L2L_IO_FILE; + return 0; +} + +static ssize_t v4l2_loopback_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_buffer *b; + int index, result; + + dprintkrw("read() %zu bytes\n", count); + result = start_fileio(file, file->private_data, + V4L2_BUF_TYPE_VIDEO_CAPTURE); + if (result < 0) + return result; + + index = get_capture_buffer(file); + if (index < 0) + return index; + + mutex_lock(&dev->image_mutex); + if (!dev->image) { + mutex_unlock(&dev->image_mutex); + return -EINVAL; + } + get_buffer(&dev->buffers[index]); + mutex_unlock(&dev->image_mutex); + + b = &dev->buffers[index].buffer; + if (count > b->bytesused) + count = b->bytesused; + + if (copy_to_user((void *)buf, (void *)(dev->image + b->m.offset), + count)) { + printk(KERN_ERR "v4l2-loopback read() failed copy_to_user()\n"); + put_buffer(&dev->buffers[index]); + return -EFAULT; + } + put_buffer(&dev->buffers[index]); + return count; +} + +static ssize_t v4l2_loopback_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_buffer *b; + int index, result; + + dprintkrw("write() %zu bytes\n", count); + result = start_fileio(file, file->private_data, + V4L2_BUF_TYPE_VIDEO_OUTPUT); + if (result < 0) + return result; + + if (count > dev->buffer_size) + count = dev->buffer_size; + + mutex_lock(&dev->image_mutex); + if (!dev->image || dev->used_buffer_count == 0) { + mutex_unlock(&dev->image_mutex); + return -EINVAL; + } + index = v4l2l_mod64(dev->write_position, dev->used_buffer_count); + get_buffer(&dev->buffers[index]); + mutex_unlock(&dev->image_mutex); + + b = &dev->buffers[index].buffer; + if (copy_from_user((void *)(dev->image + b->m.offset), (void *)buf, + count)) { + printk(KERN_ERR + "v4l2-loopback write() failed copy_from_user()\n"); + put_buffer(&dev->buffers[index]); + return -EFAULT; + } + put_buffer(&dev->buffers[index]); + b->bytesused = count; + + v4l2l_get_timestamp(b); + b->sequence = dev->write_position; + set_queued(b->flags); + buffer_written(dev, &dev->buffers[index]); + set_done(b->flags); + wake_up_all(&dev->read_event); + + return count; +} + +/* init functions */ +/* frees buffers, if allocated */ +static void free_buffers(struct v4l2_loopback_device *dev) +{ + dprintk("free_buffers() with image@%p\n", dev->image); + if (!dev->image) + return; + if (!has_no_owners(dev) || any_mapped_buffer(dev)) + /* maybe an opener snuck in before image_mutex was acquired */ + printk(KERN_WARNING + "v4l2-loopback free_buffers() buffers of video device " + "#%u freed while still mapped to userspace\n", + dev->vdev->num); + vfree(dev->image); + dev->image = NULL; + dev->image_size = 0; + dev->buffer_size = 0; +} + +static void free_timeout_buffer(struct v4l2_loopback_device *dev) +{ + dprintk("free_timeout_buffer() with timeout_image@%p\n", + dev->timeout_image); + if (!dev->timeout_image) + return; + + if ((dev->timeout_jiffies > 0 && !has_no_owners(dev)) || + dev->timeout_buffer.buffer.flags & V4L2_BUF_FLAG_MAPPED) + printk(KERN_WARNING + "v4l2-loopback free_timeout_buffer() timeout image " + "of device #%u freed while still mapped to userspace\n", + dev->vdev->num); + + vfree(dev->timeout_image); + dev->timeout_image = NULL; + dev->timeout_buffer_size = 0; +} +/* allocates buffers if no (other) openers are already using them */ +static int allocate_buffers(struct v4l2_loopback_device *dev, + struct v4l2_pix_format *pix_format) +{ + u32 buffer_size = PAGE_ALIGN(pix_format->sizeimage); + unsigned long image_size = + (unsigned long)buffer_size * (unsigned long)dev->buffer_count; + /* vfree on close file operation in case no open handles left */ + + if (buffer_size == 0 || dev->buffer_count == 0 || + buffer_size < pix_format->sizeimage) + return -EINVAL; + + if ((__LONG_MAX__ / buffer_size) < dev->buffer_count) + return -ENOSPC; + + dprintk("allocate_buffers() size %lubytes = %ubytes x %ubuffers\n", + image_size, buffer_size, dev->buffer_count); + if (dev->image) { + /* check that no buffers are expected in user-space */ + if (!has_no_owners(dev) || any_mapped_buffer(dev)) + return -EBUSY; + dprintk("allocate_buffers() existing size=%lubytes\n", + dev->image_size); + /* FIXME: prevent double allocation more intelligently! */ + if (image_size == dev->image_size) { + dprintk("allocate_buffers() keep existing\n"); + return 0; + } + free_buffers(dev); + } + + /* FIXME: set buffers to 0 */ + dev->image = vmalloc(image_size); + if (dev->image == NULL) { + dev->buffer_size = dev->image_size = 0; + return -ENOMEM; + } + init_buffers(dev, pix_format->sizeimage, buffer_size); + dev->buffer_size = buffer_size; + dev->image_size = image_size; + dprintk("allocate_buffers() -> vmalloc'd %lubytes\n", dev->image_size); + return 0; +} +static int allocate_timeout_buffer(struct v4l2_loopback_device *dev) +{ + /* device's `buffer_size` and `buffers` must be initialised in + * allocate_buffers() */ + + dprintk("allocate_timeout_buffer() size %ubytes\n", dev->buffer_size); + if (dev->buffer_size == 0) + return -EINVAL; + + if (dev->timeout_image) { + if (dev->timeout_buffer.buffer.flags & V4L2_BUF_FLAG_MAPPED) + return -EBUSY; + if (dev->buffer_size == dev->timeout_buffer_size) + return 0; + free_timeout_buffer(dev); + } + + dev->timeout_image = vzalloc(dev->buffer_size); + if (!dev->timeout_image) { + dev->timeout_buffer_size = 0; + return -ENOMEM; + } + dev->timeout_buffer_size = dev->buffer_size; + return 0; +} +/* init inner buffers, they are capture mode and flags are set as for capture + * mode buffers */ +static void init_buffers(struct v4l2_loopback_device *dev, u32 bytes_used, + u32 buffer_size) +{ + u32 i; + + for (i = 0; i < dev->buffer_count; ++i) { + struct v4l2_buffer *b = &dev->buffers[i].buffer; + b->index = i; + b->bytesused = bytes_used; + b->length = buffer_size; + b->field = V4L2_FIELD_NONE; + b->flags = 0; + b->m.offset = i * buffer_size; + b->memory = V4L2_MEMORY_MMAP; + b->sequence = 0; + b->timestamp.tv_sec = 0; + b->timestamp.tv_usec = 0; + b->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + + v4l2l_get_timestamp(b); + } + dev->timeout_buffer = dev->buffers[0]; + dev->timeout_buffer.buffer.m.offset = MAX_BUFFERS * buffer_size; +} + +/* fills and register video device */ +static void init_vdev(struct video_device *vdev, int nr) +{ +#ifdef V4L2LOOPBACK_WITH_STD + vdev->tvnorms = V4L2_STD_ALL; +#endif /* V4L2LOOPBACK_WITH_STD */ + + vdev->vfl_type = VFL_TYPE_VIDEO; + vdev->fops = &v4l2_loopback_fops; + vdev->ioctl_ops = &v4l2_loopback_ioctl_ops; + vdev->release = &video_device_release; + vdev->minor = -1; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) + vdev->device_caps = V4L2_CAP_DEVICE_CAPS | V4L2_CAP_VIDEO_CAPTURE | + V4L2_CAP_VIDEO_OUTPUT | V4L2_CAP_READWRITE | + V4L2_CAP_STREAMING; +#endif + + if (debug > 1) + vdev->dev_debug = V4L2_DEV_DEBUG_IOCTL | + V4L2_DEV_DEBUG_IOCTL_ARG; + + vdev->vfl_dir = VFL_DIR_M2M; +} + +/* init default capture parameters, only fps may be changed in future */ +static void init_capture_param(struct v4l2_captureparm *capture_param) +{ + capture_param->capability = V4L2_CAP_TIMEPERFRAME; /* since 2.16 */ + capture_param->capturemode = 0; + capture_param->extendedmode = 0; + capture_param->readbuffers = max_buffers; + capture_param->timeperframe.numerator = 1; + capture_param->timeperframe.denominator = V4L2LOOPBACK_FPS_DEFAULT; +} + +static void check_timers(struct v4l2_loopback_device *dev) +{ + if (has_output_token(dev->stream_tokens)) + return; + + if (dev->timeout_jiffies > 0 && !timer_pending(&dev->timeout_timer)) + mod_timer(&dev->timeout_timer, jiffies + dev->timeout_jiffies); + if (dev->sustain_framerate && !timer_pending(&dev->sustain_timer)) + mod_timer(&dev->sustain_timer, + jiffies + dev->frame_jiffies * 3 / 2); +} +#ifdef HAVE_TIMER_SETUP +static void sustain_timer_clb(struct timer_list *t) +{ + struct v4l2_loopback_device *dev = + container_of(t, struct v4l2_loopback_device, sustain_timer); +#else +static void sustain_timer_clb(unsigned long nr) +{ + struct v4l2_loopback_device *dev = + idr_find(&v4l2loopback_index_idr, nr); +#endif + spin_lock(&dev->lock); + if (dev->sustain_framerate) { + dev->reread_count++; + dprintkrw("sustain_timer_clb() write_pos=%lld reread=%u\n", + (long long)dev->write_position, dev->reread_count); + if (dev->reread_count == 1) + mod_timer(&dev->sustain_timer, + jiffies + max(1UL, dev->frame_jiffies / 2)); + else + mod_timer(&dev->sustain_timer, + jiffies + dev->frame_jiffies); + wake_up_all(&dev->read_event); + } + spin_unlock(&dev->lock); +} +#ifdef HAVE_TIMER_SETUP +static void timeout_timer_clb(struct timer_list *t) +{ + struct v4l2_loopback_device *dev = + container_of(t, struct v4l2_loopback_device, timeout_timer); +#else +static void timeout_timer_clb(unsigned long nr) +{ + struct v4l2_loopback_device *dev = + idr_find(&v4l2loopback_index_idr, nr); +#endif + spin_lock(&dev->lock); + if (dev->timeout_jiffies > 0) { + dev->timeout_happened = 1; + mod_timer(&dev->timeout_timer, jiffies + dev->timeout_jiffies); + wake_up_all(&dev->read_event); + } + spin_unlock(&dev->lock); +} + +/* init loopback main structure */ +#define DEFAULT_FROM_CONF(confmember, default_condition, default_value) \ + ((conf) ? \ + ((conf->confmember default_condition) ? (default_value) : \ + (conf->confmember)) : \ + default_value) + +static int v4l2_loopback_add(struct v4l2_loopback_config *conf, int *ret_nr) +{ + struct v4l2_loopback_device *dev; + struct v4l2_ctrl_handler *hdl; + struct v4l2loopback_private *vdev_priv = NULL; + int err; + + u32 _width = V4L2LOOPBACK_SIZE_DEFAULT_WIDTH; + u32 _height = V4L2LOOPBACK_SIZE_DEFAULT_HEIGHT; + + u32 _min_width = DEFAULT_FROM_CONF(min_width, + < V4L2LOOPBACK_SIZE_MIN_WIDTH, + V4L2LOOPBACK_SIZE_MIN_WIDTH); + u32 _min_height = DEFAULT_FROM_CONF(min_height, + < V4L2LOOPBACK_SIZE_MIN_HEIGHT, + V4L2LOOPBACK_SIZE_MIN_HEIGHT); + u32 _max_width = DEFAULT_FROM_CONF(max_width, < _min_width, max_width); + u32 _max_height = + DEFAULT_FROM_CONF(max_height, < _min_height, max_height); + bool _announce_all_caps = (conf && conf->announce_all_caps >= 0) ? + (bool)(conf->announce_all_caps) : + !(V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS); + int _max_buffers = DEFAULT_FROM_CONF(max_buffers, <= 0, max_buffers); + int _max_openers = DEFAULT_FROM_CONF(max_openers, <= 0, max_openers); + struct v4l2_format _fmt; + + int nr = -1; + + /* Fix: Ensure buffer count does not exceed the statically allocated array size */ + if (_max_buffers > MAX_BUFFERS) + _max_buffers = MAX_BUFFERS; + + if (conf) { + const int output_nr = conf->output_nr; +#ifdef SPLIT_DEVICES + const int capture_nr = conf->capture_nr; +#else + const int capture_nr = output_nr; +#endif + if (capture_nr >= 0 && output_nr == capture_nr) { + nr = output_nr; + } else if (capture_nr < 0 && output_nr < 0) { + nr = -1; + } else if (capture_nr < 0) { + nr = output_nr; + } else if (output_nr < 0) { + nr = capture_nr; + } else { + printk(KERN_ERR + "v4l2-loopback add() split OUTPUT and CAPTURE " + "devices not yet supported.\n"); + printk(KERN_INFO + "v4l2-loopback add() both devices must have the " + "same number (%d != %d).\n", + output_nr, capture_nr); + return -EINVAL; + } + } + + if (idr_find(&v4l2loopback_index_idr, nr)) + return -EEXIST; + + /* initialisation of a new device */ + dprintk("add() creating device #%d\n", nr); + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) + return -ENOMEM; + + /* allocate id, if @id >= 0, we're requesting that specific id */ + if (nr >= 0) { + err = idr_alloc(&v4l2loopback_index_idr, dev, nr, nr + 1, + GFP_KERNEL); + if (err == -ENOSPC) + err = -EEXIST; + } else { + err = idr_alloc(&v4l2loopback_index_idr, dev, 0, 0, GFP_KERNEL); + } + if (err < 0) + goto out_free_dev; + + /* register new device */ + MARK(); + nr = err; + + if (conf && conf->card_label[0]) { + snprintf(dev->card_label, sizeof(dev->card_label), "%s", + conf->card_label); + } else { + snprintf(dev->card_label, sizeof(dev->card_label), + "Dummy video device (0x%04X)", nr); + } + snprintf(dev->v4l2_dev.name, sizeof(dev->v4l2_dev.name), + "v4l2loopback-%03d", nr); + + err = v4l2_device_register(NULL, &dev->v4l2_dev); + if (err) + goto out_free_idr; + + /* initialise the _video_ device */ + MARK(); + err = -ENOMEM; + dev->vdev = video_device_alloc(); + if (dev->vdev == NULL) + goto out_unregister; + + vdev_priv = kzalloc(sizeof(struct v4l2loopback_private), GFP_KERNEL); + if (vdev_priv == NULL) + goto out_unregister; + + video_set_drvdata(dev->vdev, vdev_priv); + if (video_get_drvdata(dev->vdev) == NULL) + goto out_unregister; + + snprintf(dev->vdev->name, sizeof(dev->vdev->name), "%s", + dev->card_label); + vdev_priv->device_nr = nr; + init_vdev(dev->vdev, nr); + dev->vdev->v4l2_dev = &dev->v4l2_dev; + + /* initialise v4l2-loopback specific parameters */ + MARK(); + dev->announce_all_caps = _announce_all_caps; + dev->min_width = _min_width; + dev->min_height = _min_height; + dev->max_width = _max_width; + dev->max_height = _max_height; + dev->max_openers = _max_openers; + + /* set (initial) pixel and stream format */ + _width = clamp_val(_width, _min_width, _max_width); + _height = clamp_val(_height, _min_height, _max_height); + _fmt = (struct v4l2_format){ + .type = V4L2_BUF_TYPE_VIDEO_CAPTURE, + .fmt.pix = { .width = _width, + .height = _height, + .pixelformat = formats[0].fourcc, + .colorspace = V4L2_COLORSPACE_DEFAULT, + .field = V4L2_FIELD_NONE } + }; + + err = v4l2l_fill_format(&_fmt, _min_width, _max_width, _min_height, + _max_height); + if (err) + /* highly unexpected failure to assign default format */ + goto out_unregister; + dev->pix_format = _fmt.fmt.pix; + init_capture_param(&dev->capture_param); + set_timeperframe(dev, &dev->capture_param.timeperframe); + + /* ctrls parameters */ + dev->keep_format = 0; + dev->sustain_framerate = 0; + dev->timeout_jiffies = 0; + dev->timeout_image_io = 0; + + /* initialise OUTPUT and CAPTURE buffer values */ + dev->image = NULL; + dev->image_size = 0; + dev->buffer_count = _max_buffers; + dev->buffer_size = 0; + dev->used_buffer_count = 0; + INIT_LIST_HEAD(&dev->outbufs_list); + do { + u32 index; + for (index = 0; index < dev->buffer_count; ++index) + INIT_LIST_HEAD(&dev->buffers[index].list_head); + + } while (0); + memset(dev->bufpos2index, 0, sizeof(dev->bufpos2index)); + dev->write_position = 0; + + /* initialise synchronisation data */ + atomic_set(&dev->open_count, 0); + mutex_init(&dev->image_mutex); + spin_lock_init(&dev->lock); + spin_lock_init(&dev->list_lock); + init_waitqueue_head(&dev->read_event); + dev->format_tokens = V4L2L_TOKEN_MASK; + dev->stream_tokens = V4L2L_TOKEN_MASK; + + /* initialise sustain frame rate and timeout parameters, and timers */ + dev->reread_count = 0; + dev->timeout_image = NULL; + dev->timeout_happened = 0; +#ifdef HAVE_TIMER_SETUP + timer_setup(&dev->sustain_timer, sustain_timer_clb, 0); + timer_setup(&dev->timeout_timer, timeout_timer_clb, 0); +#else + setup_timer(&dev->sustain_timer, sustain_timer_clb, nr); + setup_timer(&dev->timeout_timer, timeout_timer_clb, nr); +#endif + + /* initialise the control handler and add controls */ + MARK(); + hdl = &dev->ctrl_handler; + err = v4l2_ctrl_handler_init(hdl, 4); + if (err) + goto out_unregister; + v4l2_ctrl_new_custom(hdl, &v4l2loopback_ctrl_keepformat, NULL); + v4l2_ctrl_new_custom(hdl, &v4l2loopback_ctrl_sustainframerate, NULL); + v4l2_ctrl_new_custom(hdl, &v4l2loopback_ctrl_timeout, NULL); + v4l2_ctrl_new_custom(hdl, &v4l2loopback_ctrl_timeoutimageio, NULL); + if (hdl->error) { + err = hdl->error; + goto out_free_handler; + } + dev->v4l2_dev.ctrl_handler = hdl; + + err = v4l2_ctrl_handler_setup(hdl); + if (err) + goto out_free_handler; + + /* register the device (creates /dev/video*) */ + MARK(); + if (video_register_device(dev->vdev, VFL_TYPE_VIDEO, nr) < 0) { + printk(KERN_ERR + "v4l2-loopback add() failed video_register_device()\n"); + err = -EFAULT; + goto out_free_device; + } + v4l2loopback_create_sysfs(dev->vdev); + /* NOTE: ambivalent if sysfs entries fail */ + + if (ret_nr) + *ret_nr = dev->vdev->num; + return 0; + +out_free_device: +out_free_handler: + v4l2_ctrl_handler_free(&dev->ctrl_handler); +out_unregister: + if (dev->vdev) + video_set_drvdata(dev->vdev, NULL); + video_device_release(dev->vdev); + if (vdev_priv != NULL) + kfree(vdev_priv); + v4l2_device_unregister(&dev->v4l2_dev); +out_free_idr: + idr_remove(&v4l2loopback_index_idr, nr); +out_free_dev: + kfree(dev); + return err; +} + +static void v4l2_loopback_remove(struct v4l2_loopback_device *dev) +{ + int device_nr = v4l2loopback_get_vdev_nr(dev->vdev); + mutex_lock(&dev->image_mutex); + free_buffers(dev); + free_timeout_buffer(dev); + mutex_unlock(&dev->image_mutex); + v4l2loopback_remove_sysfs(dev->vdev); + v4l2_ctrl_handler_free(&dev->ctrl_handler); + kfree(video_get_drvdata(dev->vdev)); + video_unregister_device(dev->vdev); + v4l2_device_unregister(&dev->v4l2_dev); + idr_remove(&v4l2loopback_index_idr, device_nr); + kfree(dev); +} + +static long v4l2loopback_control_ioctl(struct file *file, unsigned int cmd, + unsigned long parm) +{ + struct v4l2_loopback_device *dev; + struct v4l2_loopback_config conf; + struct v4l2_loopback_config *confptr = &conf; + int device_nr, capture_nr, output_nr; + int ret; + const __u32 version = V4L2LOOPBACK_VERSION_CODE; + + ret = mutex_lock_killable(&v4l2loopback_ctl_mutex); + if (ret) + return ret; + + ret = -EINVAL; + switch (cmd) { + default: + ret = -ENOSYS; + break; + /* add a v4l2loopback device (pair), based on the user-provided specs */ + case V4L2LOOPBACK_CTL_ADD: + case V4L2LOOPBACK_CTL_ADD_legacy: + if (parm) { + if ((ret = copy_from_user(&conf, (void *)parm, + sizeof(conf))) < 0) + break; + } else + confptr = NULL; + ret = v4l2_loopback_add(confptr, &device_nr); + if (ret >= 0) + ret = device_nr; + break; + /* remove a v4l2loopback device (both capture and output) */ + case V4L2LOOPBACK_CTL_REMOVE: + case V4L2LOOPBACK_CTL_REMOVE_legacy: + ret = v4l2loopback_lookup((__u32)parm, &dev); + if (ret >= 0 && dev) { + ret = -EBUSY; + if (dev->open_count.counter > 0) + break; + v4l2_loopback_remove(dev); + ret = 0; + }; + break; + /* get information for a loopback device. + * this is mostly about limits (which cannot be queried directly with VIDIOC_G_FMT and friends + */ + case V4L2LOOPBACK_CTL_QUERY: + case V4L2LOOPBACK_CTL_QUERY_legacy: + if (!parm) + break; + if ((ret = copy_from_user(&conf, (void *)parm, sizeof(conf))) < + 0) + break; + capture_nr = output_nr = conf.output_nr; +#ifdef SPLIT_DEVICES + capture_nr = conf.capture_nr; +#endif + device_nr = (output_nr < 0) ? capture_nr : output_nr; + MARK(); + /* get the device from either capture_nr or output_nr (whatever is valid) */ + if ((ret = v4l2loopback_lookup(device_nr, &dev)) < 0) + break; + MARK(); + /* if we got the device from output_nr and there is a valid capture_nr, + * make sure that both refer to the same device (or bail out) + */ + if ((device_nr != capture_nr) && (capture_nr >= 0) && + ((ret = v4l2loopback_lookup(capture_nr, 0)) < 0)) + break; + MARK(); + /* if otoh, we got the device from capture_nr and there is a valid output_nr, + * make sure that both refer to the same device (or bail out) + */ + if ((device_nr != output_nr) && (output_nr >= 0) && + ((ret = v4l2loopback_lookup(output_nr, 0)) < 0)) + break; + + /* v4l2_loopback_config identified a single device, so fetch the data */ + snprintf(conf.card_label, sizeof(conf.card_label), "%s", + dev->card_label); + + conf.output_nr = dev->vdev->num; +#ifdef SPLIT_DEVICES + conf.capture_nr = dev->vdev->num; +#endif + conf.min_width = dev->min_width; + conf.min_height = dev->min_height; + conf.max_width = dev->max_width; + conf.max_height = dev->max_height; + conf.announce_all_caps = dev->announce_all_caps; + conf.max_buffers = dev->buffer_count; + conf.max_openers = dev->max_openers; + conf.debug = debug; + MARK(); + if (copy_to_user((void *)parm, &conf, sizeof(conf))) { + ret = -EFAULT; + break; + } + ret = 0; + break; + case V4L2LOOPBACK_CTL_VERSION: + if (!parm) + break; + if (copy_to_user((void *)parm, &version, sizeof(version))) { + ret = -EFAULT; + break; + } + ret = 0; + break; + } + + mutex_unlock(&v4l2loopback_ctl_mutex); + MARK(); + return ret; +} + +/* LINUX KERNEL */ + +static const struct file_operations v4l2loopback_ctl_fops = { + // clang-format off + .owner = THIS_MODULE, + .open = nonseekable_open, + .unlocked_ioctl = v4l2loopback_control_ioctl, + .compat_ioctl = v4l2loopback_control_ioctl, + .llseek = noop_llseek, + // clang-format on +}; + +static struct miscdevice v4l2loopback_misc = { + // clang-format off + .minor = MISC_DYNAMIC_MINOR, + .name = "v4l2loopback", + .fops = &v4l2loopback_ctl_fops, + // clang-format on +}; + +static const struct v4l2_file_operations v4l2_loopback_fops = { + // clang-format off + .owner = THIS_MODULE, + .open = v4l2_loopback_open, + .release = v4l2_loopback_close, + .read = v4l2_loopback_read, + .write = v4l2_loopback_write, + .poll = v4l2_loopback_poll, + .mmap = v4l2_loopback_mmap, + .unlocked_ioctl = video_ioctl2, + // clang-format on +}; + +static const struct v4l2_ioctl_ops v4l2_loopback_ioctl_ops = { + // clang-format off + .vidioc_querycap = &vidioc_querycap, + .vidioc_enum_framesizes = &vidioc_enum_framesizes, + .vidioc_enum_frameintervals = &vidioc_enum_frameintervals, + + .vidioc_enum_output = &vidioc_enum_output, + .vidioc_g_output = &vidioc_g_output, + .vidioc_s_output = &vidioc_s_output, + + .vidioc_enum_input = &vidioc_enum_input, + .vidioc_g_input = &vidioc_g_input, + .vidioc_s_input = &vidioc_s_input, + + .vidioc_enum_fmt_vid_cap = &vidioc_enum_fmt_cap, + .vidioc_g_fmt_vid_cap = &vidioc_g_fmt_cap, + .vidioc_s_fmt_vid_cap = &vidioc_s_fmt_cap, + .vidioc_try_fmt_vid_cap = &vidioc_try_fmt_cap, + + .vidioc_enum_fmt_vid_out = &vidioc_enum_fmt_out, + .vidioc_s_fmt_vid_out = &vidioc_s_fmt_out, + .vidioc_g_fmt_vid_out = &vidioc_g_fmt_out, + .vidioc_try_fmt_vid_out = &vidioc_try_fmt_out, + +#ifdef V4L2L_OVERLAY + .vidioc_s_fmt_vid_overlay = &vidioc_s_fmt_overlay, + .vidioc_g_fmt_vid_overlay = &vidioc_g_fmt_overlay, +#endif + +#ifdef V4L2LOOPBACK_WITH_STD + .vidioc_s_std = &vidioc_s_std, + .vidioc_g_std = &vidioc_g_std, + .vidioc_querystd = &vidioc_querystd, +#endif /* V4L2LOOPBACK_WITH_STD */ + + .vidioc_g_parm = &vidioc_g_parm, + .vidioc_s_parm = &vidioc_s_parm, + + .vidioc_reqbufs = &vidioc_reqbufs, + .vidioc_querybuf = &vidioc_querybuf, + .vidioc_qbuf = &vidioc_qbuf, + .vidioc_dqbuf = &vidioc_dqbuf, + + .vidioc_streamon = &vidioc_streamon, + .vidioc_streamoff = &vidioc_streamoff, + +#ifdef CONFIG_VIDEO_V4L1_COMPAT + .vidiocgmbuf = &vidiocgmbuf, +#endif + + .vidioc_subscribe_event = &vidioc_subscribe_event, + .vidioc_unsubscribe_event = &v4l2_event_unsubscribe, + // clang-format on +}; + +static int free_device_cb(int id, void *ptr, void *data) +{ + struct v4l2_loopback_device *dev = ptr; + v4l2_loopback_remove(dev); + return 0; +} +static void free_devices(void) +{ + idr_for_each(&v4l2loopback_index_idr, &free_device_cb, NULL); + idr_destroy(&v4l2loopback_index_idr); +} + +static int __init v4l2loopback_init_module(void) +{ + const u32 min_width = V4L2LOOPBACK_SIZE_MIN_WIDTH; + const u32 min_height = V4L2LOOPBACK_SIZE_MIN_HEIGHT; + int err; + int i; + MARK(); + + err = misc_register(&v4l2loopback_misc); + if (err < 0) + return err; + + if (devices < 0) { + devices = 1; + + /* try guessing the devices from the "video_nr" parameter */ + for (i = MAX_DEVICES - 1; i >= 0; i--) { + if (video_nr[i] >= 0) { + devices = i + 1; + break; + } + } + } + + if (devices > MAX_DEVICES) { + devices = MAX_DEVICES; + printk(KERN_INFO + "v4l2-loopback init() number of initial devices is " + "limited to: %d\n", + MAX_DEVICES); + } + + if (max_buffers > MAX_BUFFERS) { + max_buffers = MAX_BUFFERS; + printk(KERN_INFO + "v4l2-loopback init() number of buffers is limited " + "to: %d\n", + MAX_BUFFERS); + } + + if (max_openers < 0) { + printk(KERN_INFO + "v4l2-loopback init() allowing %d openers rather " + "than %d\n", + 2, max_openers); + max_openers = 2; + } + + if (max_width < min_width) { + max_width = V4L2LOOPBACK_SIZE_DEFAULT_MAX_WIDTH; + printk(KERN_INFO "v4l2-loopback init() using max_width %d\n", + max_width); + } + if (max_height < min_height) { + max_height = V4L2LOOPBACK_SIZE_DEFAULT_MAX_HEIGHT; + printk(KERN_INFO "v4l2-loopback init() using max_height %d\n", + max_height); + } + + for (i = 0; i < devices; i++) { + struct v4l2_loopback_config cfg = { + // clang-format off + .output_nr = video_nr[i], +#ifdef SPLIT_DEVICES + .capture_nr = video_nr[i], +#endif + .min_width = min_width, + .min_height = min_height, + .max_width = max_width, + .max_height = max_height, + .announce_all_caps = (!exclusive_caps[i]), + .max_buffers = max_buffers, + .max_openers = max_openers, + .debug = debug, + // clang-format on + }; + cfg.card_label[0] = 0; + if (card_label[i]) + snprintf(cfg.card_label, sizeof(cfg.card_label), "%s", + card_label[i]); + err = v4l2_loopback_add(&cfg, 0); + if (err) { + free_devices(); + goto error; + } + } + + dprintk("module installed\n"); + + printk(KERN_INFO "v4l2-loopback driver version %d.%d.%d%s loaded\n", + // clang-format off + (V4L2LOOPBACK_VERSION_CODE >> 16) & 0xff, + (V4L2LOOPBACK_VERSION_CODE >> 8) & 0xff, + (V4L2LOOPBACK_VERSION_CODE ) & 0xff, +#ifdef SNAPSHOT_VERSION + " (" __stringify(SNAPSHOT_VERSION) ")" +#else + "" +#endif + ); + // clang-format on + + return 0; +error: + misc_deregister(&v4l2loopback_misc); + return err; +} + +static void v4l2loopback_cleanup_module(void) +{ + MARK(); + /* unregister the device -> it deletes /dev/video* */ + free_devices(); + /* and get rid of /dev/v4l2loopback */ + misc_deregister(&v4l2loopback_misc); + dprintk("module removed\n"); +} + +MODULE_ALIAS_MISCDEV(MISC_DYNAMIC_MINOR); + +module_init(v4l2loopback_init_module); +module_exit(v4l2loopback_cleanup_module); diff -ruN --no-dereference -x .git a/drivers/media/v4l2-core/v4l2loopback.h b/drivers/media/v4l2-core/v4l2loopback.h --- a/drivers/media/v4l2-core/v4l2loopback.h 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/media/v4l2-core/v4l2loopback.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +/* + * v4l2loopback.h + * + * Written by IOhannes m zmölnig, 7/1/20. + * + * Copyright 2020 by IOhannes m zmölnig. Redistribution of this file is + * permitted under the GNU General Public License. + */ +#ifndef _V4L2LOOPBACK_H +#define _V4L2LOOPBACK_H + +#define V4L2LOOPBACK_VERSION_MAJOR 0 +#define V4L2LOOPBACK_VERSION_MINOR 15 +#define V4L2LOOPBACK_VERSION_BUGFIX 4 + +/* /dev/v4l2loopback interface */ + +struct v4l2_loopback_config { + /** + * the device-number (/dev/video) + * V4L2LOOPBACK_CTL_ADD: + * setting this to a value<0, will allocate an available one + * if nr>=0 and the device already exists, the ioctl will EEXIST + * if output_nr and capture_nr are the same, only a single device will be created + * NOTE: currently split-devices (where output_nr and capture_nr differ) + * are not implemented yet. + * until then, requesting different device-IDs will result in EINVAL. + * + * V4L2LOOPBACK_CTL_QUERY: + * either both output_nr and capture_nr must refer to the same loopback, + * or one (and only one) of them must be -1 + * + */ + __s32 output_nr; + __s32 unused; /*capture_nr;*/ + + /** + * a nice name for your device + * if (*card_label)==0, an automatic name is assigned + */ + char card_label[32]; + + /** + * allowed frame size + * if too low, default values are used + */ + __u32 min_width; + __u32 max_width; + __u32 min_height; + __u32 max_height; + + /** + * number of buffers to allocate for the queue + * if set to <=0, default values are used + */ + __s32 max_buffers; + + /** + * how many consumers are allowed to open this device concurrently + * if set to <=0, default values are used + */ + __s32 max_openers; + + /** + * set the debugging level for this device + */ + __s32 debug; + + /** + * whether to announce OUTPUT/CAPTURE capabilities exclusively + * for this device or not + * (!exclusive_caps) + * NOTE: this is going to be removed once separate output/capture + * devices are implemented + */ + __s32 announce_all_caps; +}; + +#define V4L2LOOPBACK_CTL_IOCTLMAGIC '~' + +/* a pointer to an (unsigned int) that - on success - will hold + * the version code of the v4l2loopback module + * as returned by KERNEL_VERSION(MAJOR, MINOR, BUGFIX) + */ +#define V4L2LOOPBACK_CTL_VERSION _IOR(V4L2LOOPBACK_CTL_IOCTLMAGIC, 0, __u32) + +/* a pointer to a (struct v4l2_loopback_config) that has all values you wish to impose on the + * to-be-created device set. + * if the ptr is NULL, a new device is created with default values at the driver's discretion. + * + * returns the device_nr of the OUTPUT device (which can be used with V4L2LOOPBACK_CTL_QUERY, + * to get more information on the device) + */ +#define V4L2LOOPBACK_CTL_ADD \ + _IOW(V4L2LOOPBACK_CTL_IOCTLMAGIC, 1, struct v4l2_loopback_config) + +/* the device-number (either CAPTURE or OUTPUT) associated with the loopback-device */ +#define V4L2LOOPBACK_CTL_REMOVE _IOW(V4L2LOOPBACK_CTL_IOCTLMAGIC, 2, __u32) + +/* a pointer to a (struct v4l2_loopback_config) that has output_nr and/or capture_nr set + * (the two values must either refer to video-devices associated with the same loopback device + * or exactly one of them must be <0 + */ +#define V4L2LOOPBACK_CTL_QUERY \ + _IOWR(V4L2LOOPBACK_CTL_IOCTLMAGIC, 3, struct v4l2_loopback_config) + +#endif /* _V4L2LOOPBACK_H */ diff -ruN --no-dereference -x .git a/drivers/media/v4l2-core/v4l2loopback_formats.h b/drivers/media/v4l2-core/v4l2loopback_formats.h --- a/drivers/media/v4l2-core/v4l2loopback_formats.h 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/media/v4l2-core/v4l2loopback_formats.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,445 @@ +static const struct v4l2l_format formats[] = { +#ifndef V4L2_PIX_FMT_VP9 +#define V4L2_PIX_FMT_VP9 v4l2_fourcc('V', 'P', '9', '0') +#endif +#ifndef V4L2_PIX_FMT_HEVC +#define V4L2_PIX_FMT_HEVC v4l2_fourcc('H', 'E', 'V', 'C') +#endif + + /* here come the packed formats */ + { + .name = "32 bpp RGB, le", + .fourcc = V4L2_PIX_FMT_BGR32, + .depth = 32, + .flags = 0, + }, + { + .name = "32 bpp RGB, be", + .fourcc = V4L2_PIX_FMT_RGB32, + .depth = 32, + .flags = 0, + }, + { + .name = "24 bpp RGB, le", + .fourcc = V4L2_PIX_FMT_BGR24, + .depth = 24, + .flags = 0, + }, + { + .name = "24 bpp RGB, be", + .fourcc = V4L2_PIX_FMT_RGB24, + .depth = 24, + .flags = 0, + }, +#ifdef V4L2_PIX_FMT_ABGR32 + { + .name = "32 bpp RGBA, le", + .fourcc = V4L2_PIX_FMT_ABGR32, + .depth = 32, + .flags = 0, + }, +#endif +#ifdef V4L2_PIX_FMT_RGBA32 + { + .name = "32 bpp RGBA", + .fourcc = V4L2_PIX_FMT_RGBA32, + .depth = 32, + .flags = 0, + }, +#endif +#ifdef V4L2_PIX_FMT_RGB332 + { + .name = "8 bpp RGB-3-3-2", + .fourcc = V4L2_PIX_FMT_RGB332, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB332 */ +#ifdef V4L2_PIX_FMT_RGB444 + { + .name = "16 bpp RGB (xxxxrrrr ggggbbbb)", + .fourcc = V4L2_PIX_FMT_RGB444, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB444 */ +#ifdef V4L2_PIX_FMT_RGB555 + { + .name = "16 bpp RGB-5-5-5", + .fourcc = V4L2_PIX_FMT_RGB555, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB555 */ +#ifdef V4L2_PIX_FMT_RGB565 + { + .name = "16 bpp RGB-5-6-5", + .fourcc = V4L2_PIX_FMT_RGB565, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB565 */ +#ifdef V4L2_PIX_FMT_RGB555X + { + .name = "16 bpp RGB-5-5-5 BE", + .fourcc = V4L2_PIX_FMT_RGB555X, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB555X */ +#ifdef V4L2_PIX_FMT_RGB565X + { + .name = "16 bpp RGB-5-6-5 BE", + .fourcc = V4L2_PIX_FMT_RGB565X, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB565X */ +#ifdef V4L2_PIX_FMT_BGR666 + { + .name = "18 bpp BGR-6-6-6", + .fourcc = V4L2_PIX_FMT_BGR666, + .depth = 18, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_BGR666 */ + { + .name = "4:2:2, packed, YUYV", + .fourcc = V4L2_PIX_FMT_YUYV, + .depth = 16, + .flags = 0, + }, + { + .name = "4:2:2, packed, UYVY", + .fourcc = V4L2_PIX_FMT_UYVY, + .depth = 16, + .flags = 0, + }, +#ifdef V4L2_PIX_FMT_YVYU + { + .name = "4:2:2, packed YVYU", + .fourcc = V4L2_PIX_FMT_YVYU, + .depth = 16, + .flags = 0, + }, +#endif +#ifdef V4L2_PIX_FMT_VYUY + { + .name = "4:2:2, packed VYUY", + .fourcc = V4L2_PIX_FMT_VYUY, + .depth = 16, + .flags = 0, + }, +#endif + { + .name = "4:2:2, packed YYUV", + .fourcc = V4L2_PIX_FMT_YYUV, + .depth = 16, + .flags = 0, + }, + { + .name = "YUV-8-8-8-8", + .fourcc = V4L2_PIX_FMT_YUV32, + .depth = 32, + .flags = 0, + }, + { + .name = "8 bpp, Greyscale", + .fourcc = V4L2_PIX_FMT_GREY, + .depth = 8, + .flags = 0, + }, +#ifdef V4L2_PIX_FMT_Y4 + { + .name = "4 bpp Greyscale", + .fourcc = V4L2_PIX_FMT_Y4, + .depth = 4, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_Y4 */ +#ifdef V4L2_PIX_FMT_Y6 + { + .name = "6 bpp Greyscale", + .fourcc = V4L2_PIX_FMT_Y6, + .depth = 6, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_Y6 */ +#ifdef V4L2_PIX_FMT_Y10 + { + .name = "10 bpp Greyscale", + .fourcc = V4L2_PIX_FMT_Y10, + .depth = 10, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_Y10 */ +#ifdef V4L2_PIX_FMT_Y12 + { + .name = "12 bpp Greyscale", + .fourcc = V4L2_PIX_FMT_Y12, + .depth = 12, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_Y12 */ + { + .name = "16 bpp, Greyscale", + .fourcc = V4L2_PIX_FMT_Y16, + .depth = 16, + .flags = 0, + }, +#ifdef V4L2_PIX_FMT_YUV444 + { + .name = "16 bpp xxxxyyyy uuuuvvvv", + .fourcc = V4L2_PIX_FMT_YUV444, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_YUV444 */ +#ifdef V4L2_PIX_FMT_YUV555 + { + .name = "16 bpp YUV-5-5-5", + .fourcc = V4L2_PIX_FMT_YUV555, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_YUV555 */ +#ifdef V4L2_PIX_FMT_YUV565 + { + .name = "16 bpp YUV-5-6-5", + .fourcc = V4L2_PIX_FMT_YUV565, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_YUV565 */ + +/* bayer formats */ +#ifdef V4L2_PIX_FMT_SRGGB8 + { + .name = "Bayer RGGB 8bit", + .fourcc = V4L2_PIX_FMT_SRGGB8, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_SRGGB8 */ +#ifdef V4L2_PIX_FMT_SGRBG8 + { + .name = "Bayer GRBG 8bit", + .fourcc = V4L2_PIX_FMT_SGRBG8, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_SGRBG8 */ +#ifdef V4L2_PIX_FMT_SGBRG8 + { + .name = "Bayer GBRG 8bit", + .fourcc = V4L2_PIX_FMT_SGBRG8, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_SGBRG8 */ +#ifdef V4L2_PIX_FMT_SBGGR8 + { + .name = "Bayer BA81 8bit", + .fourcc = V4L2_PIX_FMT_SBGGR8, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_SBGGR8 */ + + /* here come the planar formats */ + { + .name = "4:1:0, planar, Y-Cr-Cb", + .fourcc = V4L2_PIX_FMT_YVU410, + .depth = 9, + .flags = FORMAT_FLAGS_PLANAR, + }, + { + .name = "4:2:0, planar, Y-Cr-Cb", + .fourcc = V4L2_PIX_FMT_YVU420, + .depth = 12, + .flags = FORMAT_FLAGS_PLANAR, + }, + { + .name = "4:1:0, planar, Y-Cb-Cr", + .fourcc = V4L2_PIX_FMT_YUV410, + .depth = 9, + .flags = FORMAT_FLAGS_PLANAR, + }, + { + .name = "4:2:0, planar, Y-Cb-Cr", + .fourcc = V4L2_PIX_FMT_YUV420, + .depth = 12, + .flags = FORMAT_FLAGS_PLANAR, + }, +#ifdef V4L2_PIX_FMT_YUV422P + { + .name = "16 bpp YVU422 planar", + .fourcc = V4L2_PIX_FMT_YUV422P, + .depth = 16, + .flags = FORMAT_FLAGS_PLANAR, + }, +#endif /* V4L2_PIX_FMT_YUV422P */ +#ifdef V4L2_PIX_FMT_YUV411P + { + .name = "16 bpp YVU411 planar", + .fourcc = V4L2_PIX_FMT_YUV411P, + .depth = 16, + .flags = FORMAT_FLAGS_PLANAR, + }, +#endif /* V4L2_PIX_FMT_YUV411P */ +#ifdef V4L2_PIX_FMT_Y41P + { + .name = "12 bpp YUV 4:1:1", + .fourcc = V4L2_PIX_FMT_Y41P, + .depth = 12, + .flags = FORMAT_FLAGS_PLANAR, + }, +#endif /* V4L2_PIX_FMT_Y41P */ +#ifdef V4L2_PIX_FMT_NV12 + { + .name = "12 bpp Y/CbCr 4:2:0 ", + .fourcc = V4L2_PIX_FMT_NV12, + .depth = 12, + .flags = FORMAT_FLAGS_PLANAR, + }, +#endif /* V4L2_PIX_FMT_NV12 */ + +/* here come the compressed formats */ + +#ifdef V4L2_PIX_FMT_MJPEG + { + .name = "Motion-JPEG", + .fourcc = V4L2_PIX_FMT_MJPEG, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MJPEG */ +#ifdef V4L2_PIX_FMT_JPEG + { + .name = "JFIF JPEG", + .fourcc = V4L2_PIX_FMT_JPEG, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_JPEG */ +#ifdef V4L2_PIX_FMT_DV + { + .name = "DV1394", + .fourcc = V4L2_PIX_FMT_DV, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_DV */ +#ifdef V4L2_PIX_FMT_MPEG + { + .name = "MPEG-1/2/4 Multiplexed", + .fourcc = V4L2_PIX_FMT_MPEG, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MPEG */ +#ifdef V4L2_PIX_FMT_H264 + { + .name = "H264 with start codes", + .fourcc = V4L2_PIX_FMT_H264, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_H264 */ +#ifdef V4L2_PIX_FMT_H264_NO_SC + { + .name = "H264 without start codes", + .fourcc = V4L2_PIX_FMT_H264_NO_SC, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_H264_NO_SC */ +#ifdef V4L2_PIX_FMT_H264_MVC + { + .name = "H264 MVC", + .fourcc = V4L2_PIX_FMT_H264_MVC, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_H264_MVC */ +#ifdef V4L2_PIX_FMT_H263 + { + .name = "H263", + .fourcc = V4L2_PIX_FMT_H263, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_H263 */ +#ifdef V4L2_PIX_FMT_MPEG1 + { + .name = "MPEG-1 ES", + .fourcc = V4L2_PIX_FMT_MPEG1, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MPEG1 */ +#ifdef V4L2_PIX_FMT_MPEG2 + { + .name = "MPEG-2 ES", + .fourcc = V4L2_PIX_FMT_MPEG2, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MPEG2 */ +#ifdef V4L2_PIX_FMT_MPEG4 + { + .name = "MPEG-4 part 2 ES", + .fourcc = V4L2_PIX_FMT_MPEG4, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MPEG4 */ +#ifdef V4L2_PIX_FMT_XVID + { + .name = "Xvid", + .fourcc = V4L2_PIX_FMT_XVID, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_XVID */ +#ifdef V4L2_PIX_FMT_VC1_ANNEX_G + { + .name = "SMPTE 421M Annex G compliant stream", + .fourcc = V4L2_PIX_FMT_VC1_ANNEX_G, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_VC1_ANNEX_G */ +#ifdef V4L2_PIX_FMT_VC1_ANNEX_L + { + .name = "SMPTE 421M Annex L compliant stream", + .fourcc = V4L2_PIX_FMT_VC1_ANNEX_L, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_VC1_ANNEX_L */ +#ifdef V4L2_PIX_FMT_VP8 + { + .name = "VP8", + .fourcc = V4L2_PIX_FMT_VP8, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_VP8 */ +#ifdef V4L2_PIX_FMT_VP9 + { + .name = "VP9", + .fourcc = V4L2_PIX_FMT_VP9, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_VP9 */ +#ifdef V4L2_PIX_FMT_HEVC + { + .name = "HEVC", + .fourcc = V4L2_PIX_FMT_HEVC, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_HEVC */ +}; diff -ruN --no-dereference -x .git a/drivers/net/wireguard/queueing.h b/drivers/net/wireguard/queueing.h --- a/drivers/net/wireguard/queueing.h 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/net/wireguard/queueing.h 2026-10-13 14:33:30.000000000 +0200 @@ -78,12 +78,14 @@ u8 l4_hash = skb->l4_hash; u8 sw_hash = skb->sw_hash; u32 hash = skb->hash; + u8 tstamp_type = skb->tstamp_type; skb_scrub_packet(skb, true); memset(&skb->headers, 0, sizeof(skb->headers)); if (encapsulating) { skb->l4_hash = l4_hash; skb->sw_hash = sw_hash; skb->hash = hash; + skb->tstamp_type = tstamp_type; } skb->queue_mapping = 0; skb->nohdr = 0; diff -ruN --no-dereference -x .git a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c --- a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c 2026-10-13 14:33:30.000000000 +0200 @@ -516,6 +516,10 @@ u32 link_id = u32_get_bits(info->control.flags, IEEE80211_TX_CTRL_MLO_LINK); + if (unlikely(test_bit(STATUS_FW_ERROR, &mld->trans->status))) { + ieee80211_free_txskb(hw, skb); + return; + } /* In AP mode, mgmt frames are sent on the bcast station, * so the FW can't translate the MLD addr to the link addr. Do it here */ diff -ruN --no-dereference -x .git a/drivers/net/wireless/intel/iwlwifi/mld/tx.c b/drivers/net/wireless/intel/iwlwifi/mld/tx.c --- a/drivers/net/wireless/intel/iwlwifi/mld/tx.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/net/wireless/intel/iwlwifi/mld/tx.c 2026-10-13 14:33:30.000000000 +0200 @@ -987,6 +987,9 @@ struct sk_buff *skb = NULL; u8 zero_addr[ETH_ALEN] = {}; + if (unlikely(test_bit(STATUS_FW_ERROR, &mld->trans->status))) + return; + /* * Don't transmit during firmware restart. The firmware is dead, * so iwl_trans_tx() would return -EIO for each frame. Avoid the diff -ruN --no-dereference -x .git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile --- a/drivers/pci/controller/Makefile 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/pci/controller/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -1,4 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 +ifdef CONFIG_X86_64 +ifdef CONFIG_SATA_AHCI +obj-y += intel-nvme-remap.o +endif +endif + obj-$(CONFIG_PCIE_CADENCE) += cadence/ obj-$(CONFIG_PCI_FTPCI100) += pci-ftpci100.o obj-$(CONFIG_PCI_IXP4XX) += pci-ixp4xx.o diff -ruN --no-dereference -x .git a/drivers/pci/controller/intel-nvme-remap.c b/drivers/pci/controller/intel-nvme-remap.c --- a/drivers/pci/controller/intel-nvme-remap.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/pci/controller/intel-nvme-remap.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,465 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Intel remapped NVMe device support. + * + * Copyright (c) 2019 Endless Mobile, Inc. + * Author: Daniel Drake + * + * Some products ship by default with the SATA controller in "RAID" or + * "Intel RST Premium With Intel Optane System Acceleration" mode. Under this + * mode, which we refer to as "remapped NVMe" mode, any installed NVMe + * devices disappear from the PCI bus, and instead their I/O memory becomes + * available within the AHCI device BARs. + * + * This scheme is understood to be a way of avoiding usage of the standard + * Windows NVMe driver under that OS, instead mandating usage of Intel's + * driver instead, which has better power management, and presumably offers + * some RAID/disk-caching solutions too. + * + * Here in this driver, we support the remapped NVMe mode by claiming the + * AHCI device and creating a fake PCIe root port. On the new bus, the + * original AHCI device is exposed with only minor tweaks. Then, fake PCI + * devices corresponding to the remapped NVMe devices are created. The usual + * ahci and nvme drivers are then expected to bind to these devices and + * operate as normal. + * + * The PCI configuration space for the NVMe devices is completely + * unavailable, so we fake a minimal one and hope for the best. + * + * Interrupts are shared between the AHCI and NVMe devices. For simplicity, + * we only support the legacy interrupt here, although MSI support + * could potentially be added later. + */ + +#define MODULE_NAME "intel-nvme-remap" + +#include +#include +#include +#include +#include + +#define AHCI_PCI_BAR_STANDARD 5 + +struct nvme_remap_dev { + struct pci_dev *dev; /* AHCI device */ + struct pci_bus *bus; /* our fake PCI bus */ + struct pci_sysdata sysdata; + int irq_base; /* our fake interrupts */ + + /* + * When we detect an all-ones write to a BAR register, this flag + * is set, so that we return the BAR size on the next read (a + * standard PCI behaviour). + * This includes the assumption that an all-ones BAR write is + * immediately followed by a read of the same register. + */ + bool bar_sizing; + + /* + * Resources copied from the AHCI device, to be regarded as + * resources on our fake bus. + */ + struct resource ahci_resources[PCI_NUM_RESOURCES]; + + /* Resources corresponding to the NVMe devices. */ + struct resource remapped_dev_mem[AHCI_MAX_REMAP]; + + /* Number of remapped NVMe devices found. */ + int num_remapped_devices; +}; + +static inline struct nvme_remap_dev *nrdev_from_bus(struct pci_bus *bus) +{ + return container_of(bus->sysdata, struct nvme_remap_dev, sysdata); +} + + +/******** PCI configuration space **********/ + +/* + * Helper macros for tweaking returned contents of PCI configuration space. + * + * value contains len bytes of data read from reg. + * If fixup_reg is included in that range, fix up the contents of that + * register to fixed_value. + */ +#define NR_FIX8(fixup_reg, fixed_value) do { \ + if (reg <= fixup_reg && fixup_reg < reg + len) \ + ((u8 *) value)[fixup_reg - reg] = (u8) (fixed_value); \ + } while (0) + +#define NR_FIX16(fixup_reg, fixed_value) do { \ + NR_FIX8(fixup_reg, fixed_value); \ + NR_FIX8(fixup_reg + 1, fixed_value >> 8); \ + } while (0) + +#define NR_FIX24(fixup_reg, fixed_value) do { \ + NR_FIX8(fixup_reg, fixed_value); \ + NR_FIX8(fixup_reg + 1, fixed_value >> 8); \ + NR_FIX8(fixup_reg + 2, fixed_value >> 16); \ + } while (0) + +#define NR_FIX32(fixup_reg, fixed_value) do { \ + NR_FIX16(fixup_reg, (u16) fixed_value); \ + NR_FIX16(fixup_reg + 2, fixed_value >> 16); \ + } while (0) + +/* + * Read PCI config space of the slot 0 (AHCI) device. + * We pass through the read request to the underlying device, but + * tweak the results in some cases. + */ +static int nvme_remap_pci_read_slot0(struct pci_bus *bus, int reg, + int len, u32 *value) +{ + struct nvme_remap_dev *nrdev = nrdev_from_bus(bus); + struct pci_bus *ahci_dev_bus = nrdev->dev->bus; + int ret; + + ret = ahci_dev_bus->ops->read(ahci_dev_bus, nrdev->dev->devfn, + reg, len, value); + if (ret) + return ret; + + /* + * Adjust the device class, to prevent this driver from attempting to + * additionally probe the device we're simulating here. + */ + NR_FIX24(PCI_CLASS_PROG, PCI_CLASS_STORAGE_SATA_AHCI); + + /* + * Unset interrupt pin, otherwise ACPI tries to find routing + * info for our virtual IRQ, fails, and complains. + */ + NR_FIX8(PCI_INTERRUPT_PIN, 0); + + /* + * Truncate the AHCI BAR to not include the region that covers the + * hidden devices. This will cause the ahci driver to successfully + * probe th new device (instead of handing it over to this driver). + */ + if (nrdev->bar_sizing) { + NR_FIX32(PCI_BASE_ADDRESS_5, ~(SZ_16K - 1)); + nrdev->bar_sizing = false; + } + + return PCIBIOS_SUCCESSFUL; +} + +/* + * Read PCI config space of a remapped device. + * Since the original PCI config space is inaccessible, we provide a minimal, + * fake config space instead. + */ +static int nvme_remap_pci_read_remapped(struct pci_bus *bus, unsigned int port, + int reg, int len, u32 *value) +{ + struct nvme_remap_dev *nrdev = nrdev_from_bus(bus); + struct resource *remapped_mem; + + if (port > nrdev->num_remapped_devices) + return PCIBIOS_DEVICE_NOT_FOUND; + + *value = 0; + remapped_mem = &nrdev->remapped_dev_mem[port - 1]; + + /* Set a Vendor ID, otherwise Linux assumes no device is present */ + NR_FIX16(PCI_VENDOR_ID, PCI_VENDOR_ID_INTEL); + + /* Always appear on & bus mastering */ + NR_FIX16(PCI_COMMAND, PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + + /* Set class so that nvme driver probes us */ + NR_FIX24(PCI_CLASS_PROG, PCI_CLASS_STORAGE_EXPRESS); + + if (nrdev->bar_sizing) { + NR_FIX32(PCI_BASE_ADDRESS_0, + ~(resource_size(remapped_mem) - 1)); + nrdev->bar_sizing = false; + } else { + resource_size_t mem_start = remapped_mem->start; + + mem_start |= PCI_BASE_ADDRESS_MEM_TYPE_64; + NR_FIX32(PCI_BASE_ADDRESS_0, mem_start); + mem_start >>= 32; + NR_FIX32(PCI_BASE_ADDRESS_1, mem_start); + } + + return PCIBIOS_SUCCESSFUL; +} + +/* Read PCI configuration space. */ +static int nvme_remap_pci_read(struct pci_bus *bus, unsigned int devfn, + int reg, int len, u32 *value) +{ + if (PCI_SLOT(devfn) == 0) + return nvme_remap_pci_read_slot0(bus, reg, len, value); + else + return nvme_remap_pci_read_remapped(bus, PCI_SLOT(devfn), + reg, len, value); +} + +/* + * Write PCI config space of the slot 0 (AHCI) device. + * Apart from the special case of BAR sizing, we disable all writes. + * Otherwise, the ahci driver could make changes (e.g. unset PCI bus master) + * that would affect the operation of the NVMe devices. + */ +static int nvme_remap_pci_write_slot0(struct pci_bus *bus, int reg, + int len, u32 value) +{ + struct nvme_remap_dev *nrdev = nrdev_from_bus(bus); + struct pci_bus *ahci_dev_bus = nrdev->dev->bus; + + if (reg >= PCI_BASE_ADDRESS_0 && reg <= PCI_BASE_ADDRESS_5) { + /* + * Writing all-ones to a BAR means that the size of the + * memory region is being checked. Flag this so that we can + * reply with an appropriate size on the next read. + */ + if (value == ~0) + nrdev->bar_sizing = true; + + return ahci_dev_bus->ops->write(ahci_dev_bus, + nrdev->dev->devfn, + reg, len, value); + } + + return PCIBIOS_SET_FAILED; +} + +/* + * Write PCI config space of a remapped device. + * Since the original PCI config space is inaccessible, we reject all + * writes, except for the special case of BAR probing. + */ +static int nvme_remap_pci_write_remapped(struct pci_bus *bus, + unsigned int port, + int reg, int len, u32 value) +{ + struct nvme_remap_dev *nrdev = nrdev_from_bus(bus); + + if (port > nrdev->num_remapped_devices) + return PCIBIOS_DEVICE_NOT_FOUND; + + /* + * Writing all-ones to a BAR means that the size of the memory + * region is being checked. Flag this so that we can reply with + * an appropriate size on the next read. + */ + if (value == ~0 && reg >= PCI_BASE_ADDRESS_0 + && reg <= PCI_BASE_ADDRESS_5) { + nrdev->bar_sizing = true; + return PCIBIOS_SUCCESSFUL; + } + + return PCIBIOS_SET_FAILED; +} + +/* Write PCI configuration space. */ +static int nvme_remap_pci_write(struct pci_bus *bus, unsigned int devfn, + int reg, int len, u32 value) +{ + if (PCI_SLOT(devfn) == 0) + return nvme_remap_pci_write_slot0(bus, reg, len, value); + else + return nvme_remap_pci_write_remapped(bus, PCI_SLOT(devfn), + reg, len, value); +} + +static struct pci_ops nvme_remap_pci_ops = { + .read = nvme_remap_pci_read, + .write = nvme_remap_pci_write, +}; + + +/******** Initialization & exit **********/ + +/* + * Find a PCI domain ID to use for our fake bus. + * Start at 0x10000 to not clash with ACPI _SEG domains (16 bits). + */ +static int find_free_domain(void) +{ + int domain = 0xffff; + struct pci_bus *bus = NULL; + + while ((bus = pci_find_next_bus(bus)) != NULL) + domain = max_t(int, domain, pci_domain_nr(bus)); + + return domain + 1; +} + +static int find_remapped_devices(struct nvme_remap_dev *nrdev, + struct list_head *resources) +{ + void __iomem *mmio; + int i, count = 0; + u32 cap; + + mmio = pcim_iomap(nrdev->dev, AHCI_PCI_BAR_STANDARD, + pci_resource_len(nrdev->dev, + AHCI_PCI_BAR_STANDARD)); + if (!mmio) + return -ENODEV; + + /* Check if this device might have remapped nvme devices. */ + if (pci_resource_len(nrdev->dev, AHCI_PCI_BAR_STANDARD) < SZ_512K || + !(readl(mmio + AHCI_VSCAP) & 1)) + return -ENODEV; + + cap = readq(mmio + AHCI_REMAP_CAP); + for (i = AHCI_MAX_REMAP-1; i >= 0; i--) { + struct resource *remapped_mem; + + if ((cap & (1 << i)) == 0) + continue; + if (readl(mmio + ahci_remap_dcc(i)) + != PCI_CLASS_STORAGE_EXPRESS) + continue; + + /* We've found a remapped device */ + remapped_mem = &nrdev->remapped_dev_mem[count++]; + remapped_mem->start = + pci_resource_start(nrdev->dev, AHCI_PCI_BAR_STANDARD) + + ahci_remap_base(i); + remapped_mem->end = remapped_mem->start + + AHCI_REMAP_N_SIZE - 1; + remapped_mem->flags = IORESOURCE_MEM | IORESOURCE_PCI_FIXED; + pci_add_resource(resources, remapped_mem); + } + + pcim_iounmap(nrdev->dev, mmio); + + if (count == 0) + return -ENODEV; + + nrdev->num_remapped_devices = count; + dev_info(&nrdev->dev->dev, "Found %d remapped NVMe devices\n", + nrdev->num_remapped_devices); + return 0; +} + +static void nvme_remap_remove_root_bus(void *data) +{ + struct pci_bus *bus = data; + + pci_stop_root_bus(bus); + pci_remove_root_bus(bus); +} + +static int nvme_remap_probe(struct pci_dev *dev, + const struct pci_device_id *id) +{ + struct nvme_remap_dev *nrdev; + LIST_HEAD(resources); + int i; + int ret; + struct pci_dev *child; + + nrdev = devm_kzalloc(&dev->dev, sizeof(*nrdev), GFP_KERNEL); + if (!nrdev) + return -ENOMEM; + + nrdev->sysdata.domain = find_free_domain(); + nrdev->sysdata.nvme_remap_dev = dev; + nrdev->dev = dev; + pci_set_drvdata(dev, nrdev); + + ret = pcim_enable_device(dev); + if (ret < 0) + return ret; + + pci_set_master(dev); + + ret = find_remapped_devices(nrdev, &resources); + if (ret) + return ret; + + /* Add resources from the original AHCI device */ + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + struct resource *res = &dev->resource[i]; + + if (res->start) { + struct resource *nr_res = &nrdev->ahci_resources[i]; + + nr_res->start = res->start; + nr_res->end = res->end; + nr_res->flags = res->flags; + pci_add_resource(&resources, nr_res); + } + } + + /* Create virtual interrupts */ + nrdev->irq_base = devm_irq_alloc_descs(&dev->dev, -1, 0, + nrdev->num_remapped_devices + 1, + 0); + if (nrdev->irq_base < 0) + return nrdev->irq_base; + + /* Create and populate PCI bus */ + nrdev->bus = pci_create_root_bus(&dev->dev, 0, &nvme_remap_pci_ops, + &nrdev->sysdata, &resources); + if (!nrdev->bus) + return -ENODEV; + + if (devm_add_action_or_reset(&dev->dev, nvme_remap_remove_root_bus, + nrdev->bus)) + return -ENOMEM; + + /* We don't support sharing MSI interrupts between these devices */ + nrdev->bus->bus_flags |= PCI_BUS_FLAGS_NO_MSI; + + pci_scan_child_bus(nrdev->bus); + + list_for_each_entry(child, &nrdev->bus->devices, bus_list) { + /* + * Prevent PCI core from trying to move memory BARs around. + * The hidden NVMe devices are at fixed locations. + */ + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + struct resource *res = &child->resource[i]; + + if (res->flags & IORESOURCE_MEM) + res->flags |= IORESOURCE_PCI_FIXED; + } + + /* Share the legacy IRQ between all devices */ + child->irq = dev->irq; + } + + pci_assign_unassigned_bus_resources(nrdev->bus); + pci_bus_add_devices(nrdev->bus); + + return 0; +} + +static const struct pci_device_id nvme_remap_ids[] = { + /* + * Match all Intel RAID controllers. + * + * There's overlap here with the set of devices detected by the ahci + * driver, but ahci will only successfully probe when there + * *aren't* any remapped NVMe devices, and this driver will only + * successfully probe when there *are* remapped NVMe devices that + * need handling. + */ + { + PCI_VDEVICE(INTEL, PCI_ANY_ID), + .class = PCI_CLASS_STORAGE_RAID << 8, + .class_mask = 0xffffff00, + }, + {0,} +}; +MODULE_DEVICE_TABLE(pci, nvme_remap_ids); + +static struct pci_driver nvme_remap_drv = { + .name = MODULE_NAME, + .id_table = nvme_remap_ids, + .probe = nvme_remap_probe, +}; +module_pci_driver(nvme_remap_drv); + +MODULE_AUTHOR("Daniel Drake "); +MODULE_LICENSE("GPL v2"); diff -ruN --no-dereference -x .git a/drivers/pci/quirks.c b/drivers/pci/quirks.c --- a/drivers/pci/quirks.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/pci/quirks.c 2026-10-13 14:33:30.000000000 +0200 @@ -3747,6 +3747,106 @@ dev->dev_flags |= PCI_DEV_FLAGS_NO_BUS_RESET; } +static bool acs_on_downstream; +static bool acs_on_multifunction; + +#define NUM_ACS_IDS 16 +struct acs_on_id { + unsigned short vendor; + unsigned short device; +}; +static struct acs_on_id acs_on_ids[NUM_ACS_IDS]; +static u8 max_acs_id; + +static __init int pcie_acs_override_setup(char *p) +{ + if (!p) + return -EINVAL; + + while (*p) { + if (!strncmp(p, "downstream", 10)) + acs_on_downstream = true; + if (!strncmp(p, "multifunction", 13)) + acs_on_multifunction = true; + if (!strncmp(p, "id:", 3)) { + char opt[5]; + int ret; + long val; + + if (max_acs_id >= NUM_ACS_IDS - 1) { + pr_warn("Out of PCIe ACS override slots (%d)\n", + NUM_ACS_IDS); + goto next; + } + + p += 3; + snprintf(opt, 5, "%s", p); + ret = kstrtol(opt, 16, &val); + if (ret) { + pr_warn("PCIe ACS ID parse error %d\n", ret); + goto next; + } + acs_on_ids[max_acs_id].vendor = val; + + p += strcspn(p, ":"); + if (*p != ':') { + pr_warn("PCIe ACS invalid ID\n"); + goto next; + } + + p++; + snprintf(opt, 5, "%s", p); + ret = kstrtol(opt, 16, &val); + if (ret) { + pr_warn("PCIe ACS ID parse error %d\n", ret); + goto next; + } + acs_on_ids[max_acs_id].device = val; + max_acs_id++; + } +next: + p += strcspn(p, ","); + if (*p == ',') + p++; + } + + if (acs_on_downstream || acs_on_multifunction || max_acs_id) + pr_warn("Warning: PCIe ACS overrides enabled; This may allow non-IOMMU protected peer-to-peer DMA\n"); + + return 0; +} +early_param("pcie_acs_override", pcie_acs_override_setup); + +static int pcie_acs_overrides(struct pci_dev *dev, u16 acs_flags) +{ + int i; + + /* Never override ACS for legacy devices or devices with ACS caps */ + if (!pci_is_pcie(dev) || + pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS)) + return -ENOTTY; + + for (i = 0; i < max_acs_id; i++) + if (acs_on_ids[i].vendor == dev->vendor && + acs_on_ids[i].device == dev->device) + return 1; + + switch (pci_pcie_type(dev)) { + case PCI_EXP_TYPE_DOWNSTREAM: + case PCI_EXP_TYPE_ROOT_PORT: + if (acs_on_downstream) + return 1; + break; + case PCI_EXP_TYPE_ENDPOINT: + case PCI_EXP_TYPE_UPSTREAM: + case PCI_EXP_TYPE_LEG_END: + case PCI_EXP_TYPE_RC_END: + if (acs_on_multifunction && dev->multifunction) + return 1; + } + + return -ENOTTY; +} /* * After asserting Secondary Bus Reset to downstream devices via a GB10 * Root Port, the link may not retrain correctly. @@ -5245,6 +5345,7 @@ { PCI_VENDOR_ID_ZHAOXIN, PCI_ANY_ID, pci_quirk_zhaoxin_pcie_ports_acs }, /* Wangxun nics */ { PCI_VENDOR_ID_WANGXUN, PCI_ANY_ID, pci_quirk_wangxun_nic_acs }, + { PCI_ANY_ID, PCI_ANY_ID, pcie_acs_overrides }, { 0 } }; diff -ruN --no-dereference -x .git a/drivers/pci/vgaarb.c b/drivers/pci/vgaarb.c --- a/drivers/pci/vgaarb.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/pci/vgaarb.c 2026-10-13 14:33:30.000000000 +0200 @@ -143,6 +143,7 @@ pci_dev_put(vga_default); vga_default = pci_dev_get(pdev); } +EXPORT_SYMBOL_GPL(vga_set_default_device); /** * vga_remove_vgacon - deactivate VGA console diff -ruN --no-dereference -x .git a/drivers/platform/x86/apple-gmux.c b/drivers/platform/x86/apple-gmux.c --- a/drivers/platform/x86/apple-gmux.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/platform/x86/apple-gmux.c 2026-10-13 14:33:30.000000000 +0200 @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -107,6 +108,10 @@ # define MMIO_GMUX_MAX_BRIGHTNESS 0xffff +static bool force_igd; +module_param(force_igd, bool, 0); +MODULE_PARM_DESC(force_idg, "Switch gpu to igd on module load. Make sure that you have apple-set-os set up and the iGPU is in `lspci -s 00:02.0`. (default: false) (bool)"); + static u8 gmux_pio_read8(struct apple_gmux_data *gmux_data, int port) { return inb(gmux_data->iostart + port); @@ -945,6 +950,19 @@ gmux_enable_interrupts(gmux_data); gmux_read_switch_state(gmux_data); + if (force_igd) { + struct pci_dev *pdev; + + pdev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(2, 0)); + if (pdev) { + pr_info("Switching to IGD"); + gmux_switchto(VGA_SWITCHEROO_IGD); + vga_set_default_device(pdev); + } else { + pr_err("force_idg is true, but couldn't find iGPU at 00:02.0! Is apple-set-os working?"); + } + } + /* * Retina MacBook Pros cannot switch the panel's AUX separately * and need eDP pre-calibration. They are distinguishable from diff -ruN --no-dereference -x .git a/drivers/platform/x86/serial-multi-instantiate.c b/drivers/platform/x86/serial-multi-instantiate.c --- a/drivers/platform/x86/serial-multi-instantiate.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/platform/x86/serial-multi-instantiate.c 2026-10-13 14:33:30.000000000 +0200 @@ -307,6 +307,15 @@ smi_devs_unregister(smi); } +static const struct smi_node aw88399_hda = { + .instances = { + { "aw88399-hda", IRQ_RESOURCE_AUTO, 0 }, + { "aw88399-hda", IRQ_RESOURCE_AUTO, 0 }, + {} + }, + .bus_type = SMI_AUTO_DETECT, +}; + static const struct smi_node bsg1160_data = { .instances = { { "bmc150_accel", IRQ_RESOURCE_GPIO, 0 }, @@ -400,11 +409,23 @@ .bus_type = SMI_AUTO_DETECT, }; +static const struct smi_node max98390_hda = { + .instances = { + { "max98390-hda", IRQ_RESOURCE_NONE, 0 }, + { "max98390-hda", IRQ_RESOURCE_NONE, 0 }, + { "max98390-hda", IRQ_RESOURCE_NONE, 0 }, + { "max98390-hda", IRQ_RESOURCE_NONE, 0 }, + {} + }, + .bus_type = SMI_I2C, +}; + /* * Note new device-ids must also be added to ignore_serial_bus_ids in * drivers/acpi/scan.c: acpi_device_enumeration_by_parent(). */ static const struct acpi_device_id smi_acpi_ids[] = { + { "AWDZ8399", (unsigned long)&aw88399_hda }, { "BSG1160", (unsigned long)&bsg1160_data }, { "BSG2150", (unsigned long)&bsg2150_data }, { "CSC3551", (unsigned long)&cs35l41_hda }, @@ -412,6 +433,7 @@ { "CSC3556", (unsigned long)&cs35l56_hda }, { "CSC3557", (unsigned long)&cs35l57_hda }, { "INT3515", (unsigned long)&int3515_data }, + { "MAX98390", (unsigned long)&max98390_hda }, { "TXNW2781", (unsigned long)&tas2781_hda }, /* Non-conforming _HID for Cirrus Logic already released */ { "CLSA0100", (unsigned long)&cs35l41_hda }, diff -ruN --no-dereference -x .git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig --- a/drivers/scsi/Kconfig 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/scsi/Kconfig 2026-10-13 14:33:30.000000000 +0200 @@ -1522,4 +1522,6 @@ source "drivers/scsi/device_handler/Kconfig" +source "drivers/scsi/vhba/Kconfig" + endmenu diff -ruN --no-dereference -x .git a/drivers/scsi/Makefile b/drivers/scsi/Makefile --- a/drivers/scsi/Makefile 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/scsi/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -152,6 +152,7 @@ obj-$(CONFIG_SCSI_ENCLOSURE) += ses.o obj-$(CONFIG_SCSI_HISI_SAS) += hisi_sas/ +obj-$(CONFIG_VHBA) += vhba/ # This goes last, so that "real" scsi devices probe earlier obj-$(CONFIG_SCSI_DEBUG) += scsi_debug.o diff -ruN --no-dereference -x .git a/drivers/scsi/vhba/Kconfig b/drivers/scsi/vhba/Kconfig --- a/drivers/scsi/vhba/Kconfig 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/scsi/vhba/Kconfig 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,9 @@ +config VHBA + tristate "Virtual (SCSI) Host Bus Adapter" + depends on SCSI + help + This is the in-kernel part of CDEmu, a CD/DVD-ROM device + emulator. + + This driver can also be built as a module. If so, the module + will be called vhba. diff -ruN --no-dereference -x .git a/drivers/scsi/vhba/Makefile b/drivers/scsi/vhba/Makefile --- a/drivers/scsi/vhba/Makefile 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/scsi/vhba/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,4 @@ +VHBA_VERSION := 20250329 + +obj-$(CONFIG_VHBA) += vhba.o +ccflags-y := -DVHBA_VERSION=\"$(VHBA_VERSION)\" -Werror diff -ruN --no-dereference -x .git a/drivers/scsi/vhba/vhba.c b/drivers/scsi/vhba/vhba.c --- a/drivers/scsi/vhba/vhba.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/scsi/vhba/vhba.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,1132 @@ +/* + * vhba.c + * + * Copyright (C) 2007-2012 Chia-I Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define pr_fmt(fmt) "vhba: " fmt + +#include + +#include +#include +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#ifdef CONFIG_COMPAT +#include +#endif +#include +#include +#include +#include +#include +#include + + +MODULE_AUTHOR("Chia-I Wu"); +MODULE_VERSION(VHBA_VERSION); +MODULE_DESCRIPTION("Virtual SCSI HBA"); +MODULE_LICENSE("GPL"); + + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) +#define sdev_dbg(sdev, fmt, a...) \ + dev_dbg(&(sdev)->sdev_gendev, fmt, ##a) +#define scmd_dbg(scmd, fmt, a...) \ + dev_dbg(&(scmd)->device->sdev_gendev, fmt, ##a) +#endif + +#define VHBA_MAX_SECTORS_PER_IO 256 +#define VHBA_MAX_BUS 16 +#define VHBA_MAX_ID 16 +#define VHBA_MAX_DEVICES (VHBA_MAX_BUS * (VHBA_MAX_ID-1)) +#define VHBA_KBUF_SIZE PAGE_SIZE + +#define DATA_TO_DEVICE(dir) ((dir) == DMA_TO_DEVICE || (dir) == DMA_BIDIRECTIONAL) +#define DATA_FROM_DEVICE(dir) ((dir) == DMA_FROM_DEVICE || (dir) == DMA_BIDIRECTIONAL) + + +static int vhba_can_queue = 32; +module_param_named(can_queue, vhba_can_queue, int, 0); + + +enum vhba_req_state { + VHBA_REQ_FREE, + VHBA_REQ_PENDING, + VHBA_REQ_READING, + VHBA_REQ_SENT, + VHBA_REQ_WRITING, +}; + +struct vhba_command { + struct scsi_cmnd *cmd; + /* metatags are per-host. not to be confused with + queue tags that are usually per-lun */ + unsigned long metatag; + int status; + struct list_head entry; +}; + +struct vhba_device { + unsigned int num; + spinlock_t cmd_lock; + struct list_head cmd_list; + wait_queue_head_t cmd_wq; + atomic_t refcnt; + + unsigned char *kbuf; + size_t kbuf_size; +}; + +struct vhba_host { + struct Scsi_Host *shost; + spinlock_t cmd_lock; + int cmd_next; + struct vhba_command *commands; + spinlock_t dev_lock; + struct vhba_device *devices[VHBA_MAX_DEVICES]; + int num_devices; + DECLARE_BITMAP(chgmap, VHBA_MAX_DEVICES); + int chgtype[VHBA_MAX_DEVICES]; + struct work_struct scan_devices; +}; + +#define MAX_COMMAND_SIZE 16 + +struct vhba_request { + __u32 metatag; + __u32 lun; + __u8 cdb[MAX_COMMAND_SIZE]; + __u8 cdb_len; + __u32 data_len; +}; + +struct vhba_response { + __u32 metatag; + __u32 status; + __u32 data_len; +}; + + + +static struct vhba_command *vhba_alloc_command (void); +static void vhba_free_command (struct vhba_command *vcmd); + +static struct platform_device vhba_platform_device; + + + +/* These functions define a symmetric 1:1 mapping between device numbers and + the bus and id. We have reserved the last id per bus for the host itself. */ +static void devnum_to_bus_and_id(unsigned int devnum, unsigned int *bus, unsigned int *id) +{ + *bus = devnum / (VHBA_MAX_ID-1); + *id = devnum % (VHBA_MAX_ID-1); +} + +static unsigned int bus_and_id_to_devnum(unsigned int bus, unsigned int id) +{ + return (bus * (VHBA_MAX_ID-1)) + id; +} + +static struct vhba_device *vhba_device_alloc (void) +{ + struct vhba_device *vdev; + + vdev = kzalloc(sizeof(struct vhba_device), GFP_KERNEL); + if (!vdev) { + return NULL; + } + + spin_lock_init(&vdev->cmd_lock); + INIT_LIST_HEAD(&vdev->cmd_list); + init_waitqueue_head(&vdev->cmd_wq); + atomic_set(&vdev->refcnt, 1); + + vdev->kbuf = NULL; + vdev->kbuf_size = 0; + + return vdev; +} + +static void vhba_device_put (struct vhba_device *vdev) +{ + if (atomic_dec_and_test(&vdev->refcnt)) { + kfree(vdev); + } +} + +static struct vhba_device *vhba_device_get (struct vhba_device *vdev) +{ + atomic_inc(&vdev->refcnt); + + return vdev; +} + +static int vhba_device_queue (struct vhba_device *vdev, struct scsi_cmnd *cmd) +{ + struct vhba_host *vhost; + struct vhba_command *vcmd; + unsigned long flags; + + vhost = platform_get_drvdata(&vhba_platform_device); + + vcmd = vhba_alloc_command(); + if (!vcmd) { + return SCSI_MLQUEUE_HOST_BUSY; + } + + vcmd->cmd = cmd; + + spin_lock_irqsave(&vdev->cmd_lock, flags); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) + vcmd->metatag = scsi_cmd_to_rq(vcmd->cmd)->tag; +#else + vcmd->metatag = vcmd->cmd->request->tag; +#endif + list_add_tail(&vcmd->entry, &vdev->cmd_list); + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + wake_up_interruptible(&vdev->cmd_wq); + + return 0; +} + +static int vhba_device_dequeue (struct vhba_device *vdev, struct scsi_cmnd *cmd) +{ + struct vhba_command *vcmd; + int retval; + unsigned long flags; + + spin_lock_irqsave(&vdev->cmd_lock, flags); + list_for_each_entry(vcmd, &vdev->cmd_list, entry) { + if (vcmd->cmd == cmd) { + list_del_init(&vcmd->entry); + break; + } + } + + /* command not found */ + if (&vcmd->entry == &vdev->cmd_list) { + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + return SUCCESS; + } + + while (vcmd->status == VHBA_REQ_READING || vcmd->status == VHBA_REQ_WRITING) { + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + scmd_dbg(cmd, "wait for I/O before aborting\n"); + schedule_timeout(1); + spin_lock_irqsave(&vdev->cmd_lock, flags); + } + + retval = (vcmd->status == VHBA_REQ_SENT) ? FAILED : SUCCESS; + + vhba_free_command(vcmd); + + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + return retval; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) +static int vhba_slave_alloc(struct scsi_device *sdev) +{ + struct Scsi_Host *shost = sdev->host; + + sdev_dbg(sdev, "enabling tagging (queue depth: %i).\n", sdev->queue_depth); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) + if (!shost_use_blk_mq(shost) && shost->bqt) { +#else + if (shost->bqt) { +#endif + blk_queue_init_tags(sdev->request_queue, sdev->queue_depth, shost->bqt); + } + scsi_adjust_queue_depth(sdev, 0, sdev->queue_depth); + + return 0; +} +#endif + +static void vhba_scan_devices_add (struct vhba_host *vhost, int bus, int id) +{ + struct scsi_device *sdev; + + sdev = scsi_device_lookup(vhost->shost, bus, id, 0); + if (!sdev) { + scsi_add_device(vhost->shost, bus, id, 0); + } else { + dev_warn(&vhost->shost->shost_gendev, "tried to add an already-existing device %d:%d:0!\n", bus, id); + scsi_device_put(sdev); + } +} + +static void vhba_scan_devices_remove (struct vhba_host *vhost, int bus, int id) +{ + struct scsi_device *sdev; + + sdev = scsi_device_lookup(vhost->shost, bus, id, 0); + if (sdev) { + scsi_remove_device(sdev); + scsi_device_put(sdev); + } else { + dev_warn(&vhost->shost->shost_gendev, "tried to remove non-existing device %d:%d:0!\n", bus, id); + } +} + +static void vhba_scan_devices (struct work_struct *work) +{ + struct vhba_host *vhost = container_of(work, struct vhba_host, scan_devices); + unsigned long flags; + int change, exists; + unsigned int devnum; + unsigned int bus, id; + + for (;;) { + spin_lock_irqsave(&vhost->dev_lock, flags); + + devnum = find_first_bit(vhost->chgmap, VHBA_MAX_DEVICES); + if (devnum >= VHBA_MAX_DEVICES) { + spin_unlock_irqrestore(&vhost->dev_lock, flags); + break; + } + change = vhost->chgtype[devnum]; + exists = vhost->devices[devnum] != NULL; + + vhost->chgtype[devnum] = 0; + clear_bit(devnum, vhost->chgmap); + + spin_unlock_irqrestore(&vhost->dev_lock, flags); + + devnum_to_bus_and_id(devnum, &bus, &id); + + if (change < 0) { + dev_dbg(&vhost->shost->shost_gendev, "trying to remove target %d:%d:0\n", bus, id); + vhba_scan_devices_remove(vhost, bus, id); + } else if (change > 0) { + dev_dbg(&vhost->shost->shost_gendev, "trying to add target %d:%d:0\n", bus, id); + vhba_scan_devices_add(vhost, bus, id); + } else { + /* quick sequence of add/remove or remove/add; we determine + which one it was by checking if device structure exists */ + if (exists) { + /* remove followed by add: remove and (re)add */ + dev_dbg(&vhost->shost->shost_gendev, "trying to (re)add target %d:%d:0\n", bus, id); + vhba_scan_devices_remove(vhost, bus, id); + vhba_scan_devices_add(vhost, bus, id); + } else { + /* add followed by remove: no-op */ + dev_dbg(&vhost->shost->shost_gendev, "no-op for target %d:%d:0\n", bus, id); + } + } + } +} + +static int vhba_add_device (struct vhba_device *vdev) +{ + struct vhba_host *vhost; + unsigned int devnum; + unsigned long flags; + + vhost = platform_get_drvdata(&vhba_platform_device); + + vhba_device_get(vdev); + + spin_lock_irqsave(&vhost->dev_lock, flags); + if (vhost->num_devices >= VHBA_MAX_DEVICES) { + spin_unlock_irqrestore(&vhost->dev_lock, flags); + vhba_device_put(vdev); + return -EBUSY; + } + + for (devnum = 0; devnum < VHBA_MAX_DEVICES; devnum++) { + if (vhost->devices[devnum] == NULL) { + vdev->num = devnum; + vhost->devices[devnum] = vdev; + vhost->num_devices++; + set_bit(devnum, vhost->chgmap); + vhost->chgtype[devnum]++; + break; + } + } + spin_unlock_irqrestore(&vhost->dev_lock, flags); + + schedule_work(&vhost->scan_devices); + + return 0; +} + +static int vhba_remove_device (struct vhba_device *vdev) +{ + struct vhba_host *vhost; + unsigned long flags; + + vhost = platform_get_drvdata(&vhba_platform_device); + + spin_lock_irqsave(&vhost->dev_lock, flags); + set_bit(vdev->num, vhost->chgmap); + vhost->chgtype[vdev->num]--; + vhost->devices[vdev->num] = NULL; + vhost->num_devices--; + spin_unlock_irqrestore(&vhost->dev_lock, flags); + + vhba_device_put(vdev); + + schedule_work(&vhost->scan_devices); + + return 0; +} + +static struct vhba_device *vhba_lookup_device (int devnum) +{ + struct vhba_host *vhost; + struct vhba_device *vdev = NULL; + unsigned long flags; + + vhost = platform_get_drvdata(&vhba_platform_device); + + if (likely(devnum < VHBA_MAX_DEVICES)) { + spin_lock_irqsave(&vhost->dev_lock, flags); + vdev = vhost->devices[devnum]; + if (vdev) { + vdev = vhba_device_get(vdev); + } + + spin_unlock_irqrestore(&vhost->dev_lock, flags); + } + + return vdev; +} + +static struct vhba_command *vhba_alloc_command (void) +{ + struct vhba_host *vhost; + struct vhba_command *vcmd; + unsigned long flags; + int i; + + vhost = platform_get_drvdata(&vhba_platform_device); + + spin_lock_irqsave(&vhost->cmd_lock, flags); + + vcmd = vhost->commands + vhost->cmd_next++; + if (vcmd->status != VHBA_REQ_FREE) { + for (i = 0; i < vhba_can_queue; i++) { + vcmd = vhost->commands + i; + + if (vcmd->status == VHBA_REQ_FREE) { + vhost->cmd_next = i + 1; + break; + } + } + + if (i == vhba_can_queue) { + vcmd = NULL; + } + } + + if (vcmd) { + vcmd->status = VHBA_REQ_PENDING; + } + + vhost->cmd_next %= vhba_can_queue; + + spin_unlock_irqrestore(&vhost->cmd_lock, flags); + + return vcmd; +} + +static void vhba_free_command (struct vhba_command *vcmd) +{ + struct vhba_host *vhost; + unsigned long flags; + + vhost = platform_get_drvdata(&vhba_platform_device); + + spin_lock_irqsave(&vhost->cmd_lock, flags); + vcmd->status = VHBA_REQ_FREE; + spin_unlock_irqrestore(&vhost->cmd_lock, flags); +} + +static enum scsi_qc_status vhba_queuecommand (struct Scsi_Host *shost, struct scsi_cmnd *cmd) +{ + struct vhba_device *vdev; + enum scsi_qc_status retval; + unsigned int devnum; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) + scmd_dbg(cmd, "queue %p tag %i\n", cmd, scsi_cmd_to_rq(cmd)->tag); +#else + scmd_dbg(cmd, "queue %p tag %i\n", cmd, cmd->request->tag); +#endif + + devnum = bus_and_id_to_devnum(cmd->device->channel, cmd->device->id); + vdev = vhba_lookup_device(devnum); + if (!vdev) { + scmd_dbg(cmd, "no such device\n"); + + cmd->result = DID_NO_CONNECT << 16; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) + scsi_done(cmd); +#else + cmd->scsi_done(cmd); +#endif + + return 0; + } + + retval = vhba_device_queue(vdev, cmd); + + vhba_device_put(vdev); + + return retval; +} + +static int vhba_abort (struct scsi_cmnd *cmd) +{ + struct vhba_device *vdev; + int retval = SUCCESS; + unsigned int devnum; + + scmd_dbg(cmd, "abort %p\n", cmd); + + devnum = bus_and_id_to_devnum(cmd->device->channel, cmd->device->id); + vdev = vhba_lookup_device(devnum); + if (vdev) { + retval = vhba_device_dequeue(vdev, cmd); + vhba_device_put(vdev); + } else { + cmd->result = DID_NO_CONNECT << 16; + } + + return retval; +} + +static struct scsi_host_template vhba_template = { + .module = THIS_MODULE, + .name = "vhba", + .proc_name = "vhba", + .queuecommand = vhba_queuecommand, + .eh_abort_handler = vhba_abort, + .this_id = -1, + .max_sectors = VHBA_MAX_SECTORS_PER_IO, + .sg_tablesize = 256, +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) + .slave_alloc = vhba_slave_alloc, +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 14, 0) + .tag_alloc_policy_rr = true, +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0) + .tag_alloc_policy = BLK_TAG_ALLOC_RR, +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) + .use_blk_tags = 1, +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0) + .max_segment_size = VHBA_KBUF_SIZE, +#endif +}; + +static ssize_t do_request (struct vhba_device *vdev, unsigned long metatag, struct scsi_cmnd *cmd, char __user *buf, size_t buf_len) +{ + struct vhba_request vreq; + ssize_t ret; + + scmd_dbg(cmd, "request %lu (%p), cdb 0x%x, bufflen %d, sg count %d\n", + metatag, cmd, cmd->cmnd[0], scsi_bufflen(cmd), scsi_sg_count(cmd)); + + ret = sizeof(vreq); + if (DATA_TO_DEVICE(cmd->sc_data_direction)) { + ret += scsi_bufflen(cmd); + } + + if (ret > buf_len) { + scmd_dbg(cmd, "buffer too small (%zd < %zd) for a request\n", buf_len, ret); + return -EIO; + } + + vreq.metatag = metatag; + vreq.lun = cmd->device->lun; + memcpy(vreq.cdb, cmd->cmnd, MAX_COMMAND_SIZE); + vreq.cdb_len = cmd->cmd_len; + vreq.data_len = scsi_bufflen(cmd); + + if (copy_to_user(buf, &vreq, sizeof(vreq))) { + return -EFAULT; + } + + if (DATA_TO_DEVICE(cmd->sc_data_direction) && vreq.data_len) { + buf += sizeof(vreq); + + if (scsi_sg_count(cmd)) { + unsigned char *kaddr, *uaddr; + struct scatterlist *sglist = scsi_sglist(cmd); + struct scatterlist *sg; + int i; + + uaddr = (unsigned char *) buf; + + for_each_sg(sglist, sg, scsi_sg_count(cmd), i) { + size_t len = sg->length; + + if (len > vdev->kbuf_size) { + scmd_dbg(cmd, "segment size (%zu) exceeds kbuf size (%zu)!", len, vdev->kbuf_size); + len = vdev->kbuf_size; + } + + kaddr = kmap_atomic(sg_page(sg)); + memcpy(vdev->kbuf, kaddr + sg->offset, len); + kunmap_atomic(kaddr); + + if (copy_to_user(uaddr, vdev->kbuf, len)) { + return -EFAULT; + } + uaddr += len; + } + } else { + if (copy_to_user(buf, scsi_sglist(cmd), vreq.data_len)) { + return -EFAULT; + } + } + } + + return ret; +} + +static ssize_t do_response (struct vhba_device *vdev, unsigned long metatag, struct scsi_cmnd *cmd, const char __user *buf, size_t buf_len, struct vhba_response *res) +{ + ssize_t ret = 0; + + scmd_dbg(cmd, "response %lu (%p), status %x, data len %d, sg count %d\n", + metatag, cmd, res->status, res->data_len, scsi_sg_count(cmd)); + + if (res->status) { + if (res->data_len > SCSI_SENSE_BUFFERSIZE) { + scmd_dbg(cmd, "truncate sense (%d < %d)", SCSI_SENSE_BUFFERSIZE, res->data_len); + res->data_len = SCSI_SENSE_BUFFERSIZE; + } + + if (copy_from_user(cmd->sense_buffer, buf, res->data_len)) { + return -EFAULT; + } + + cmd->result = res->status; + + ret += res->data_len; + } else if (DATA_FROM_DEVICE(cmd->sc_data_direction) && scsi_bufflen(cmd)) { + size_t to_read; + + if (res->data_len > scsi_bufflen(cmd)) { + scmd_dbg(cmd, "truncate data (%d < %d)\n", scsi_bufflen(cmd), res->data_len); + res->data_len = scsi_bufflen(cmd); + } + + to_read = res->data_len; + + if (scsi_sg_count(cmd)) { + unsigned char *kaddr, *uaddr; + struct scatterlist *sglist = scsi_sglist(cmd); + struct scatterlist *sg; + int i; + + uaddr = (unsigned char *)buf; + + for_each_sg(sglist, sg, scsi_sg_count(cmd), i) { + size_t len = (sg->length < to_read) ? sg->length : to_read; + + if (len > vdev->kbuf_size) { + scmd_dbg(cmd, "segment size (%zu) exceeds kbuf size (%zu)!", len, vdev->kbuf_size); + len = vdev->kbuf_size; + } + + if (copy_from_user(vdev->kbuf, uaddr, len)) { + return -EFAULT; + } + uaddr += len; + + kaddr = kmap_atomic(sg_page(sg)); + memcpy(kaddr + sg->offset, vdev->kbuf, len); + kunmap_atomic(kaddr); + + to_read -= len; + if (to_read == 0) { + break; + } + } + } else { + if (copy_from_user(scsi_sglist(cmd), buf, res->data_len)) { + return -EFAULT; + } + + to_read -= res->data_len; + } + + scsi_set_resid(cmd, to_read); + + ret += res->data_len - to_read; + } + + return ret; +} + +static struct vhba_command *next_command (struct vhba_device *vdev) +{ + struct vhba_command *vcmd; + + list_for_each_entry(vcmd, &vdev->cmd_list, entry) { + if (vcmd->status == VHBA_REQ_PENDING) { + break; + } + } + + if (&vcmd->entry == &vdev->cmd_list) { + vcmd = NULL; + } + + return vcmd; +} + +static struct vhba_command *match_command (struct vhba_device *vdev, __u32 metatag) +{ + struct vhba_command *vcmd; + + list_for_each_entry(vcmd, &vdev->cmd_list, entry) { + if (vcmd->metatag == metatag) { + break; + } + } + + if (&vcmd->entry == &vdev->cmd_list) { + vcmd = NULL; + } + + return vcmd; +} + +static struct vhba_command *wait_command (struct vhba_device *vdev, unsigned long flags) +{ + struct vhba_command *vcmd; + DEFINE_WAIT(wait); + + while (!(vcmd = next_command(vdev))) { + if (signal_pending(current)) { + break; + } + + prepare_to_wait(&vdev->cmd_wq, &wait, TASK_INTERRUPTIBLE); + + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + schedule(); + + spin_lock_irqsave(&vdev->cmd_lock, flags); + } + + finish_wait(&vdev->cmd_wq, &wait); + if (vcmd) { + vcmd->status = VHBA_REQ_READING; + } + + return vcmd; +} + +static ssize_t vhba_ctl_read (struct file *file, char __user *buf, size_t buf_len, loff_t *offset) +{ + struct vhba_device *vdev; + struct vhba_command *vcmd; + ssize_t ret; + unsigned long flags; + + vdev = file->private_data; + + /* Get next command */ + if (file->f_flags & O_NONBLOCK) { + /* Non-blocking variant */ + spin_lock_irqsave(&vdev->cmd_lock, flags); + vcmd = next_command(vdev); + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + if (!vcmd) { + return -EWOULDBLOCK; + } + } else { + /* Blocking variant */ + spin_lock_irqsave(&vdev->cmd_lock, flags); + vcmd = wait_command(vdev, flags); + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + if (!vcmd) { + return -ERESTARTSYS; + } + } + + ret = do_request(vdev, vcmd->metatag, vcmd->cmd, buf, buf_len); + + spin_lock_irqsave(&vdev->cmd_lock, flags); + if (ret >= 0) { + vcmd->status = VHBA_REQ_SENT; + *offset += ret; + } else { + vcmd->status = VHBA_REQ_PENDING; + } + + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + return ret; +} + +static ssize_t vhba_ctl_write (struct file *file, const char __user *buf, size_t buf_len, loff_t *offset) +{ + struct vhba_device *vdev; + struct vhba_command *vcmd; + struct vhba_response res; + ssize_t ret; + unsigned long flags; + + if (buf_len < sizeof(res)) { + return -EIO; + } + + if (copy_from_user(&res, buf, sizeof(res))) { + return -EFAULT; + } + + vdev = file->private_data; + + spin_lock_irqsave(&vdev->cmd_lock, flags); + vcmd = match_command(vdev, res.metatag); + if (!vcmd || vcmd->status != VHBA_REQ_SENT) { + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + pr_debug("ctl dev #%u not expecting response\n", vdev->num); + return -EIO; + } + vcmd->status = VHBA_REQ_WRITING; + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + ret = do_response(vdev, vcmd->metatag, vcmd->cmd, buf + sizeof(res), buf_len - sizeof(res), &res); + + spin_lock_irqsave(&vdev->cmd_lock, flags); + if (ret >= 0) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) + scsi_done(vcmd->cmd); +#else + vcmd->cmd->scsi_done(vcmd->cmd); +#endif + ret += sizeof(res); + + /* don't compete with vhba_device_dequeue */ + if (!list_empty(&vcmd->entry)) { + list_del_init(&vcmd->entry); + vhba_free_command(vcmd); + } + } else { + vcmd->status = VHBA_REQ_SENT; + } + + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + return ret; +} + +static long vhba_ctl_ioctl (struct file *file, unsigned int cmd, unsigned long arg) +{ + struct vhba_device *vdev = file->private_data; + struct vhba_host *vhost = platform_get_drvdata(&vhba_platform_device); + + switch (cmd) { + case 0xBEEF001: { + unsigned int ident[4]; /* host, channel, id, lun */ + + ident[0] = vhost->shost->host_no; + devnum_to_bus_and_id(vdev->num, &ident[1], &ident[2]); + ident[3] = 0; /* lun */ + + if (copy_to_user((void *) arg, ident, sizeof(ident))) { + return -EFAULT; + } + + return 0; + } + case 0xBEEF002: { + unsigned int devnum = vdev->num; + + if (copy_to_user((void *) arg, &devnum, sizeof(devnum))) { + return -EFAULT; + } + + return 0; + } + } + + return -ENOTTY; +} + +#ifdef CONFIG_COMPAT +static long vhba_ctl_compat_ioctl (struct file *file, unsigned int cmd, unsigned long arg) +{ + unsigned long compat_arg = (unsigned long)compat_ptr(arg); + return vhba_ctl_ioctl(file, cmd, compat_arg); +} +#endif + +static unsigned int vhba_ctl_poll (struct file *file, poll_table *wait) +{ + struct vhba_device *vdev = file->private_data; + unsigned int mask = 0; + unsigned long flags; + + poll_wait(file, &vdev->cmd_wq, wait); + + spin_lock_irqsave(&vdev->cmd_lock, flags); + if (next_command(vdev)) { + mask |= POLLIN | POLLRDNORM; + } + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + return mask; +} + +static int vhba_ctl_open (struct inode *inode, struct file *file) +{ + struct vhba_device *vdev; + int retval; + + pr_debug("ctl dev open\n"); + + /* check if vhba is probed */ + if (!platform_get_drvdata(&vhba_platform_device)) { + return -ENODEV; + } + + vdev = vhba_device_alloc(); + if (!vdev) { + return -ENOMEM; + } + + vdev->kbuf_size = VHBA_KBUF_SIZE; + vdev->kbuf = kzalloc(vdev->kbuf_size, GFP_KERNEL); + if (!vdev->kbuf) { + return -ENOMEM; + } + + if (!(retval = vhba_add_device(vdev))) { + file->private_data = vdev; + } + + vhba_device_put(vdev); + + return retval; +} + +static int vhba_ctl_release (struct inode *inode, struct file *file) +{ + struct vhba_device *vdev; + struct vhba_command *vcmd; + unsigned long flags; + + vdev = file->private_data; + + pr_debug("ctl dev release\n"); + + vhba_device_get(vdev); + vhba_remove_device(vdev); + + spin_lock_irqsave(&vdev->cmd_lock, flags); + list_for_each_entry(vcmd, &vdev->cmd_list, entry) { + WARN_ON(vcmd->status == VHBA_REQ_READING || vcmd->status == VHBA_REQ_WRITING); + + scmd_dbg(vcmd->cmd, "device released with command %lu (%p)\n", vcmd->metatag, vcmd->cmd); + vcmd->cmd->result = DID_NO_CONNECT << 16; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) + scsi_done(vcmd->cmd); +#else + vcmd->cmd->scsi_done(vcmd->cmd); +#endif + vhba_free_command(vcmd); + } + INIT_LIST_HEAD(&vdev->cmd_list); + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + kfree(vdev->kbuf); + vdev->kbuf = NULL; + + vhba_device_put(vdev); + + return 0; +} + +static struct file_operations vhba_ctl_fops = { + .owner = THIS_MODULE, + .open = vhba_ctl_open, + .release = vhba_ctl_release, + .read = vhba_ctl_read, + .write = vhba_ctl_write, + .poll = vhba_ctl_poll, + .unlocked_ioctl = vhba_ctl_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = vhba_ctl_compat_ioctl, +#endif +}; + +static struct miscdevice vhba_miscdev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "vhba_ctl", + .fops = &vhba_ctl_fops, +}; + +static int vhba_probe (struct platform_device *pdev) +{ + struct Scsi_Host *shost; + struct vhba_host *vhost; + int i; + + vhba_can_queue = clamp(vhba_can_queue, 1, 256); + + shost = scsi_host_alloc(&vhba_template, sizeof(struct vhba_host)); + if (!shost) { + return -ENOMEM; + } + + shost->max_channel = VHBA_MAX_BUS-1; + shost->max_id = VHBA_MAX_ID; + /* we don't support lun > 0 */ + shost->max_lun = 1; + shost->max_cmd_len = MAX_COMMAND_SIZE; + shost->can_queue = vhba_can_queue; + shost->cmd_per_lun = vhba_can_queue; + + vhost = (struct vhba_host *)shost->hostdata; + memset(vhost, 0, sizeof(struct vhba_host)); + + vhost->shost = shost; + vhost->num_devices = 0; + spin_lock_init(&vhost->dev_lock); + spin_lock_init(&vhost->cmd_lock); + INIT_WORK(&vhost->scan_devices, vhba_scan_devices); + vhost->cmd_next = 0; + vhost->commands = kzalloc(vhba_can_queue * sizeof(struct vhba_command), GFP_KERNEL); + if (!vhost->commands) { + return -ENOMEM; + } + + for (i = 0; i < vhba_can_queue; i++) { + vhost->commands[i].status = VHBA_REQ_FREE; + } + + platform_set_drvdata(pdev, vhost); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) + i = scsi_init_shared_tag_map(shost, vhba_can_queue); + if (i) return i; +#endif + + if (scsi_add_host(shost, &pdev->dev)) { + scsi_host_put(shost); + return -ENOMEM; + } + + return 0; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 11, 0) +static int vhba_remove (struct platform_device *pdev) +#else +static void vhba_remove (struct platform_device *pdev) +#endif +{ + struct vhba_host *vhost; + struct Scsi_Host *shost; + + vhost = platform_get_drvdata(pdev); + shost = vhost->shost; + + scsi_remove_host(shost); + scsi_host_put(shost); + + kfree(vhost->commands); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 11, 0) + return 0; +#endif +} + +static void vhba_release (struct device * dev) +{ + return; +} + +static struct platform_device vhba_platform_device = { + .name = "vhba", + .id = -1, + .dev = { + .release = vhba_release, + }, +}; + +static struct platform_driver vhba_platform_driver = { + .driver = { + .owner = THIS_MODULE, + .name = "vhba", + }, + .probe = vhba_probe, + .remove = vhba_remove, +}; + +static int __init vhba_init (void) +{ + int ret; + + ret = platform_device_register(&vhba_platform_device); + if (ret < 0) { + return ret; + } + + ret = platform_driver_register(&vhba_platform_driver); + if (ret < 0) { + platform_device_unregister(&vhba_platform_device); + return ret; + } + + ret = misc_register(&vhba_miscdev); + if (ret < 0) { + platform_driver_unregister(&vhba_platform_driver); + platform_device_unregister(&vhba_platform_device); + return ret; + } + + return 0; +} + +static void __exit vhba_exit(void) +{ + misc_deregister(&vhba_miscdev); + platform_driver_unregister(&vhba_platform_driver); + platform_device_unregister(&vhba_platform_device); +} + +module_init(vhba_init); +module_exit(vhba_exit); + diff -ruN --no-dereference -x .git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig --- a/drivers/soc/apple/Kconfig 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/soc/apple/Kconfig 2026-10-13 14:33:30.000000000 +0200 @@ -4,6 +4,15 @@ menu "Apple SoC drivers" +config APPLE_DOCKCHANNEL + tristate "Apple DockChannel FIFO" + depends on ARCH_APPLE || COMPILE_TEST + help + DockChannel is a simple FIFO used on Apple SoCs for debug and inter-processor + communications. + + Say 'y' here if you have an Apple SoC. + config APPLE_MAILBOX tristate "Apple SoC mailboxes" depends on PM @@ -28,6 +37,19 @@ Say 'y' here if you have an Apple SoC. +config APPLE_RTKIT_HELPER + tristate "Apple Generic RTKit helper co-processor" + depends on APPLE_RTKIT + depends on ARCH_APPLE || COMPILE_TEST + help + Apple SoCs such as the M1 come with various co-processors running + their proprietary RTKit operating system. This option enables support + for a generic co-processor that does not implement any additional + in-band communications. It can be used for testing purposes, or for + coprocessors such as MTP that communicate over a different interface. + + Say 'y' here if you have an Apple SoC. + config APPLE_SART tristate "Apple SART DMA address filter" depends on ARCH_APPLE || COMPILE_TEST diff -ruN --no-dereference -x .git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile --- a/drivers/soc/apple/Makefile 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/soc/apple/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -1,11 +1,17 @@ # SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_APPLE_DOCKCHANNEL) += apple-dockchannel.o +apple-dockchannel-y = dockchannel.o + obj-$(CONFIG_APPLE_MAILBOX) += apple-mailbox.o apple-mailbox-y = mailbox.o obj-$(CONFIG_APPLE_RTKIT) += apple-rtkit.o apple-rtkit-y = rtkit.o rtkit-crashlog.o +obj-$(CONFIG_APPLE_RTKIT_HELPER) += apple-rtkit-helper.o +apple-rtkit-helper-y = rtkit-helper.o + obj-$(CONFIG_APPLE_SART) += apple-sart.o apple-sart-y = sart.o diff -ruN --no-dereference -x .git a/drivers/soc/apple/dockchannel.c b/drivers/soc/apple/dockchannel.c --- a/drivers/soc/apple/dockchannel.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/soc/apple/dockchannel.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,404 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple DockChannel FIFO driver + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DOCKCHANNEL_MAX_IRQ 32 + +#define DOCKCHANNEL_TX_TIMEOUT_MS 1000 +#define DOCKCHANNEL_RX_TIMEOUT_MS 1000 + +#define IRQ_MASK 0x0 +#define IRQ_FLAG 0x4 + +#define IRQ_TX BIT(0) +#define IRQ_RX BIT(1) + +#define CONFIG_TX_THRESH 0x0 +#define CONFIG_RX_THRESH 0x4 + +#define DATA_TX8 0x4 +#define DATA_TX16 0x8 +#define DATA_TX24 0xc +#define DATA_TX32 0x10 +#define DATA_TX_FREE 0x14 +#define DATA_RX8 0x1c +#define DATA_RX16 0x20 +#define DATA_RX24 0x24 +#define DATA_RX32 0x28 +#define DATA_RX_COUNT 0x2c + +struct dockchannel { + struct device *dev; + int tx_irq; + int rx_irq; + + void __iomem *config_base; + void __iomem *data_base; + + u32 fifo_size; + bool awaiting; + struct completion tx_comp; + struct completion rx_comp; + + void *cookie; + void (*data_available)(void *cookie, size_t avail); +}; + +struct dockchannel_common { + struct device *dev; + struct irq_domain *domain; + int irq; + + void __iomem *irq_base; +}; + +/* Dockchannel FIFO functions */ + +static irqreturn_t dockchannel_tx_irq(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + + disable_irq_nosync(irq); + complete(&dockchannel->tx_comp); + + return IRQ_HANDLED; +} + +static irqreturn_t dockchannel_rx_irq(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + + disable_irq_nosync(irq); + + if (dockchannel->awaiting) { + return IRQ_WAKE_THREAD; + } else { + complete(&dockchannel->rx_comp); + return IRQ_HANDLED; + } +} + +static irqreturn_t dockchannel_rx_irq_thread(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + size_t avail = readl_relaxed(dockchannel->data_base + DATA_RX_COUNT); + + dockchannel->awaiting = false; + dockchannel->data_available(dockchannel->cookie, avail); + + return IRQ_HANDLED; +} + +int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t count) +{ + size_t left = count; + const u8 *p = buf; + + while (left > 0) { + size_t avail = readl_relaxed(dockchannel->data_base + DATA_TX_FREE); + size_t block = min(left, avail); + + if (avail == 0) { + size_t threshold = min((size_t)(dockchannel->fifo_size / 2), left); + + writel_relaxed(threshold, dockchannel->config_base + CONFIG_TX_THRESH); + reinit_completion(&dockchannel->tx_comp); + enable_irq(dockchannel->tx_irq); + + if (!wait_for_completion_timeout(&dockchannel->tx_comp, + msecs_to_jiffies(DOCKCHANNEL_TX_TIMEOUT_MS))) { + disable_irq(dockchannel->tx_irq); + return -ETIMEDOUT; + } + + continue; + } + + while (block >= 4) { + writel_relaxed(get_unaligned_le32(p), dockchannel->data_base + DATA_TX32); + p += 4; + left -= 4; + block -= 4; + } + while (block > 0) { + writeb_relaxed(*p++, dockchannel->data_base + DATA_TX8); + left--; + block--; + } + } + + return count; +} +EXPORT_SYMBOL(dockchannel_send); + +int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count) +{ + size_t left = count; + u8 *p = buf; + + while (left > 0) { + size_t avail = readl_relaxed(dockchannel->data_base + DATA_RX_COUNT); + size_t block = min(left, avail); + + if (avail == 0) { + size_t threshold = min((size_t)(dockchannel->fifo_size / 2), left); + + writel_relaxed(threshold, dockchannel->config_base + CONFIG_RX_THRESH); + reinit_completion(&dockchannel->rx_comp); + enable_irq(dockchannel->rx_irq); + + if (!wait_for_completion_timeout(&dockchannel->rx_comp, + msecs_to_jiffies(DOCKCHANNEL_RX_TIMEOUT_MS))) { + disable_irq(dockchannel->rx_irq); + return -ETIMEDOUT; + } + + continue; + } + + while (block >= 4) { + put_unaligned_le32(readl_relaxed(dockchannel->data_base + DATA_RX32), p); + p += 4; + left -= 4; + block -= 4; + } + while (block > 0) { + *p++ = readl_relaxed(dockchannel->data_base + DATA_RX8) >> 8; + left--; + block--; + } + } + + return count; +} +EXPORT_SYMBOL(dockchannel_recv); + +int dockchannel_await(struct dockchannel *dockchannel, + void (*callback)(void *cookie, size_t avail), + void *cookie, size_t count) +{ + size_t threshold = min((size_t)dockchannel->fifo_size, count); + + if (!count) { + dockchannel->awaiting = false; + disable_irq(dockchannel->rx_irq); + return 0; + } + + dockchannel->data_available = callback; + dockchannel->cookie = cookie; + dockchannel->awaiting = true; + writel_relaxed(threshold, dockchannel->config_base + CONFIG_RX_THRESH); + enable_irq(dockchannel->rx_irq); + + return threshold; +} +EXPORT_SYMBOL(dockchannel_await); + +struct dockchannel *dockchannel_init(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel *dockchannel; + int ret; + + dockchannel = devm_kzalloc(dev, sizeof(*dockchannel), GFP_KERNEL); + if (!dockchannel) + return ERR_PTR(-ENOMEM); + + dockchannel->dev = dev; + dockchannel->config_base = devm_platform_ioremap_resource_byname(pdev, "config"); + if (IS_ERR(dockchannel->config_base)) + return (__force void *)dockchannel->config_base; + + dockchannel->data_base = devm_platform_ioremap_resource_byname(pdev, "data"); + if (IS_ERR(dockchannel->data_base)) + return (__force void *)dockchannel->data_base; + + ret = of_property_read_u32(dev->of_node, "apple,fifo-size", &dockchannel->fifo_size); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Missing apple,fifo-size property")); + + init_completion(&dockchannel->tx_comp); + init_completion(&dockchannel->rx_comp); + + dockchannel->tx_irq = platform_get_irq_byname(pdev, "tx"); + if (dockchannel->tx_irq < 0) { + return ERR_PTR(dev_err_probe(dev, dockchannel->tx_irq, + "Failed to get TX IRQ")); + } + + dockchannel->rx_irq = platform_get_irq_byname(pdev, "rx"); + if (dockchannel->rx_irq < 0) { + return ERR_PTR(dev_err_probe(dev, dockchannel->rx_irq, + "Failed to get RX IRQ")); + } + + ret = devm_request_irq(dev, dockchannel->tx_irq, dockchannel_tx_irq, IRQF_NO_AUTOEN, + "apple-dockchannel-tx", dockchannel); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Failed to request TX IRQ")); + + ret = devm_request_threaded_irq(dev, dockchannel->rx_irq, dockchannel_rx_irq, + dockchannel_rx_irq_thread, IRQF_NO_AUTOEN, + "apple-dockchannel-rx", dockchannel); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Failed to request RX IRQ")); + + return dockchannel; +} +EXPORT_SYMBOL(dockchannel_init); + + +/* Dockchannel IRQchip */ + +static void dockchannel_irq(struct irq_desc *desc) +{ + unsigned int irq = irq_desc_get_irq(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + struct dockchannel_common *dcc = irq_get_handler_data(irq); + unsigned long flags = readl_relaxed(dcc->irq_base + IRQ_FLAG); + int bit; + + chained_irq_enter(chip, desc); + + for_each_set_bit(bit, &flags, DOCKCHANNEL_MAX_IRQ) + generic_handle_domain_irq(dcc->domain, bit); + + chained_irq_exit(chip, desc); +} + +static void dockchannel_irq_ack(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + + writel_relaxed(BIT(hwirq), dcc->irq_base + IRQ_FLAG); +} + +static void dockchannel_irq_mask(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + u32 val = readl_relaxed(dcc->irq_base + IRQ_MASK); + + writel_relaxed(val & ~BIT(hwirq), dcc->irq_base + IRQ_MASK); +} + +static void dockchannel_irq_unmask(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + u32 val = readl_relaxed(dcc->irq_base + IRQ_MASK); + + writel_relaxed(val | BIT(hwirq), dcc->irq_base + IRQ_MASK); +} + +static const struct irq_chip dockchannel_irqchip = { + .name = "dockchannel-irqc", + .irq_ack = dockchannel_irq_ack, + .irq_mask = dockchannel_irq_mask, + .irq_unmask = dockchannel_irq_unmask, +}; + +static int dockchannel_irq_domain_map(struct irq_domain *d, unsigned int virq, + irq_hw_number_t hw) +{ + irq_set_chip_data(virq, d->host_data); + irq_set_chip_and_handler(virq, &dockchannel_irqchip, handle_level_irq); + + return 0; +} + +static const struct irq_domain_ops dockchannel_irq_domain_ops = { + .xlate = irq_domain_xlate_twocell, + .map = dockchannel_irq_domain_map, +}; + +static int dockchannel_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel_common *dcc; + + dcc = devm_kzalloc(dev, sizeof(*dcc), GFP_KERNEL); + if (!dcc) + return -ENOMEM; + + dcc->dev = dev; + platform_set_drvdata(pdev, dcc); + + dcc->irq_base = devm_platform_ioremap_resource_byname(pdev, "irq"); + if (IS_ERR(dcc->irq_base)) + return PTR_ERR(dcc->irq_base); + + writel_relaxed(0, dcc->irq_base + IRQ_MASK); + writel_relaxed(~0, dcc->irq_base + IRQ_FLAG); + + dcc->domain = irq_domain_add_linear(dev->of_node, DOCKCHANNEL_MAX_IRQ, + &dockchannel_irq_domain_ops, dcc); + if (!dcc->domain) + return -ENOMEM; + + dcc->irq = platform_get_irq(pdev, 0); + if (dcc->irq < 0) + return dev_err_probe(dev, dcc->irq, "Failed to get IRQ"); + + irq_set_handler_data(dcc->irq, dcc); + irq_set_chained_handler(dcc->irq, dockchannel_irq); + + devm_of_platform_populate(dev); + + return 0; +} + +static void dockchannel_remove(struct platform_device *pdev) +{ + struct dockchannel_common *dcc = platform_get_drvdata(pdev); + int hwirq; + + device_for_each_child(&pdev->dev, NULL, of_platform_device_destroy); + + irq_set_chained_handler_and_data(dcc->irq, NULL, NULL); + + for (hwirq = 0; hwirq < DOCKCHANNEL_MAX_IRQ; hwirq++) + irq_dispose_mapping(irq_find_mapping(dcc->domain, hwirq)); + + irq_domain_remove(dcc->domain); + + writel_relaxed(0, dcc->irq_base + IRQ_MASK); + writel_relaxed(~0, dcc->irq_base + IRQ_FLAG); +} + +static const struct of_device_id dockchannel_of_match[] = { + { .compatible = "apple,dockchannel" }, + {}, +}; +MODULE_DEVICE_TABLE(of, dockchannel_of_match); + +static struct platform_driver dockchannel_driver = { + .driver = { + .name = "dockchannel", + .of_match_table = dockchannel_of_match, + }, + .probe = dockchannel_probe, + .remove = dockchannel_remove, +}; +module_platform_driver(dockchannel_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple DockChannel driver"); diff -ruN --no-dereference -x .git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c --- a/drivers/soc/apple/rtkit-helper.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/soc/apple/rtkit-helper.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple Generic RTKit helper coprocessor + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_ASC_CPU_CONTROL 0x44 +#define APPLE_ASC_CPU_CONTROL_RUN BIT(4) + +struct apple_rtkit_helper { + struct device *dev; + struct apple_rtkit *rtk; + + void __iomem *asc_base; + + struct resource *sram; + void __iomem *sram_base; +}; + +static int apple_rtkit_helper_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_rtkit_helper *helper = cookie; + struct resource res = { + .start = bfr->iova, + .end = bfr->iova + bfr->size - 1, + .name = "rtkit_map", + }; + + if (!bfr->iova) { + bfr->buffer = dma_alloc_coherent(helper->dev, bfr->size, + &bfr->iova, GFP_KERNEL); + if (!bfr->buffer) + return -ENOMEM; + return 0; + } + + if (!helper->sram) { + dev_err(helper->dev, + "RTKit buffer request with no SRAM region: %pR", &res); + return -EFAULT; + } + + res.flags = helper->sram->flags; + + if (res.end < res.start || !resource_contains(helper->sram, &res)) { + dev_err(helper->dev, + "RTKit buffer request outside SRAM region: %pR", &res); + return -EFAULT; + } + + bfr->iomem = helper->sram_base + (res.start - helper->sram->start); + bfr->is_mapped = true; + + return 0; +} + +static void apple_rtkit_helper_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +{ + // no-op +} + +static const struct apple_rtkit_ops apple_rtkit_helper_ops = { + .shmem_setup = apple_rtkit_helper_shmem_setup, + .shmem_destroy = apple_rtkit_helper_shmem_destroy, +}; + +static int apple_rtkit_helper_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_rtkit_helper *helper; + int ret; + + /* 44 bits for addresses in standard RTKit requests */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); + if (ret) + return ret; + + helper = devm_kzalloc(dev, sizeof(*helper), GFP_KERNEL); + if (!helper) + return -ENOMEM; + + helper->dev = dev; + platform_set_drvdata(pdev, helper); + + helper->asc_base = devm_platform_ioremap_resource_byname(pdev, "asc"); + if (IS_ERR(helper->asc_base)) + return PTR_ERR(helper->asc_base); + + helper->sram = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sram"); + if (helper->sram) { + helper->sram_base = devm_ioremap_resource(dev, helper->sram); + if (IS_ERR(helper->sram_base)) + return dev_err_probe(dev, PTR_ERR(helper->sram_base), + "Failed to map SRAM region"); + } + + helper->rtk = + devm_apple_rtkit_init(dev, helper, NULL, 0, &apple_rtkit_helper_ops); + if (IS_ERR(helper->rtk)) + return dev_err_probe(dev, PTR_ERR(helper->rtk), + "Failed to intialize RTKit"); + + writel_relaxed(APPLE_ASC_CPU_CONTROL_RUN, + helper->asc_base + APPLE_ASC_CPU_CONTROL); + + /* Works for both wake and boot */ + ret = apple_rtkit_wake(helper->rtk); + if (ret != 0) + return dev_err_probe(dev, ret, "Failed to wake up coprocessor"); + + return 0; +} + +static void apple_rtkit_helper_remove(struct platform_device *pdev) +{ + struct apple_rtkit_helper *helper = platform_get_drvdata(pdev); + + if (apple_rtkit_is_running(helper->rtk)) + apple_rtkit_quiesce(helper->rtk); + + writel_relaxed(0, helper->asc_base + APPLE_ASC_CPU_CONTROL); +} + +static const struct of_device_id apple_rtkit_helper_of_match[] = { + { .compatible = "apple,rtk-helper-asc4" }, + {}, +}; +MODULE_DEVICE_TABLE(of, apple_rtkit_helper_of_match); + +static struct platform_driver apple_rtkit_helper_driver = { + .driver = { + .name = "rtkit-helper", + .of_match_table = apple_rtkit_helper_of_match, + }, + .probe = apple_rtkit_helper_probe, + .remove = apple_rtkit_helper_remove, +}; +module_platform_driver(apple_rtkit_helper_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple RTKit helper driver"); diff -ruN --no-dereference -x .git a/drivers/staging/Kconfig b/drivers/staging/Kconfig --- a/drivers/staging/Kconfig 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/staging/Kconfig 2026-10-13 14:33:30.000000000 +0200 @@ -48,4 +48,6 @@ source "drivers/staging/vme_user/Kconfig" +source "drivers/staging/t2bce/Kconfig" + endif # STAGING diff -ruN --no-dereference -x .git a/drivers/staging/Makefile b/drivers/staging/Makefile --- a/drivers/staging/Makefile 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/staging/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -13,3 +13,4 @@ obj-$(CONFIG_GREYBUS) += greybus/ obj-$(CONFIG_BCM2835_VCHIQ) += vc04_services/ obj-$(CONFIG_XIL_AXIS_FIFO) += axis-fifo/ +obj-$(CONFIG_T2BCE_CORE) += t2bce/ diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/Kconfig b/drivers/staging/t2bce/Kconfig --- a/drivers/staging/t2bce/Kconfig 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/Kconfig 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: GPL-2.0-only +menu "Apple T2 Buffer Copy Engine support" + +config T2BCE_DMA + tristate + help + DMA queue support for the Apple T2 Buffer Copy Engine. + +config T2BCE_CORE + tristate "Apple T2 Buffer Copy Engine core" + depends on X86_64 && PCI + select T2BCE_DMA + help + Core transport driver for the Buffer Copy Engine in Apple T2 Macs. + It provides the mailbox, DMA queue coordination, power management, + and transport API used by the T2 VHCI and audio drivers. + + If compiled as a module, it will be called t2bce_core. + +config T2BCE_VHCI + tristate "Apple T2 virtual USB host controller" + depends on T2BCE_CORE && USB + help + Virtual USB 2.0 host controller for devices exposed by bridgeOS, + including the internal keyboard, trackpad, Touch Bar, and camera. + + If compiled as a module, it will be called t2bce_vhci. + +config T2BCE_AUDIO + tristate "Apple T2 audio support" + depends on T2BCE_CORE && SND + select SND_PCM + select SND_JACK + help + ALSA driver for the audio endpoints exposed by bridgeOS on Apple + T2 Macs. + + If compiled as a module, it will be called t2bce_audio. + +endmenu diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/Makefile b/drivers/staging/t2bce/Makefile --- a/drivers/staging/t2bce/Makefile 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_T2BCE_DMA) += t2bce_dma/ +obj-$(CONFIG_T2BCE_CORE) += t2bce_core/ +obj-$(CONFIG_T2BCE_VHCI) += t2bce_vhci/ +obj-$(CONFIG_T2BCE_AUDIO) += t2bce_audio/ diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_audio/Makefile b/drivers/staging/t2bce/t2bce_audio/Makefile --- a/drivers/staging/t2bce/t2bce_audio/Makefile 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_audio/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_T2BCE_AUDIO) += t2bce_audio.o +t2bce_audio-y := audio.o protocol.o protocol_bce.o pcm.o + +ccflags-y += -DWITHOUT_NVME_PATCH +ccflags-y += -I$(src) +ccflags-y += -I$(src)/../t2bce_core/include diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_audio/audio.c b/drivers/staging/t2bce/t2bce_audio/audio.c --- a/drivers/staging/t2bce/t2bce_audio/audio.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_audio/audio.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,930 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "audio.h" +#include "pcm.h" +#include + +static int t2audio_alsa_index = SNDRV_DEFAULT_IDX1; +static char *t2audio_alsa_id = SNDRV_DEFAULT_STR1; + +static dev_t t2audio_chrdev; +static struct class *t2audio_class; + +static int t2audio_init_cmd(struct t2audio_device *a); +static int t2audio_init_bs(struct t2audio_device *a); +static void t2audio_init_dev(struct t2audio_device *a, t2audio_device_id_t dev_id); +static void t2audio_free_dev(struct t2audio_subdevice *sdev); +static void t2audio_reset_stream(struct t2audio_stream *stream); +static void t2audio_reset_streams(struct t2audio_device *a); +static void t2audio_reset_clock(struct t2audio_device *a); +static void t2audio_resume_work(struct work_struct *ws); +static void t2audio_resume_complete(void *userdata); +static void t2audio_pm_prepare_client(void *userdata); +static void t2audio_pm_shutdown_client(void *userdata); + +static const struct t2bce_core_client_pm_ops t2audio_pm_ops = { + .shutdown = t2audio_pm_shutdown_client, + .pm_prepare = t2audio_pm_prepare_client, +}; + +static int t2audio_probe(struct pci_dev *dev, const struct pci_device_id *id) +{ + struct t2audio_device *t2audio = NULL; + struct t2audio_subdevice *sdev = NULL; + int status = 0; + u32 cfg; + + pr_debug("t2bce_audio: capturing our device\n"); + + if (pci_enable_device(dev)) + return -ENODEV; + if (pci_request_regions(dev, "t2bce_audio")) { + status = -ENODEV; + goto fail; + } + pci_set_master(dev); + + t2audio = kzalloc(sizeof(struct t2audio_device), GFP_KERNEL); + if (!t2audio) { + status = -ENOMEM; + goto fail; + } + + t2audio->bce = t2bce_core_client_get(&dev->dev); + if (IS_ERR(t2audio->bce)) { + status = PTR_ERR(t2audio->bce); + t2audio->bce = NULL; + if (status != -EPROBE_DEFER) + dev_warn(&dev->dev, "t2bce_audio: Failed to get BCE client: %d\n", status); + goto fail; + } + + t2audio->pci = dev; + pci_set_drvdata(dev, t2audio); + t2bce_core_client_set_resume_complete_callback(t2audio->bce, t2audio_resume_complete, t2audio); + t2bce_core_client_set_pm_ops(t2audio->bce, &t2audio_pm_ops, t2audio); + + t2audio->devt = t2audio_chrdev; + t2audio->dev = device_create(t2audio_class, &dev->dev, t2audio->devt, NULL, "t2bce_audio"); + if (IS_ERR_OR_NULL(t2audio->dev)) { + status = PTR_ERR(t2audio->dev); + goto fail; + } + init_completion(&t2audio->remote_alive); + INIT_WORK(&t2audio->resume_work, t2audio_resume_work); + INIT_LIST_HEAD(&t2audio->subdevice_list); + spin_lock_init(&t2audio->clock_lock); + t2audio->clock_samples = 0; + t2audio->clock_offset_valid = false; + + /* Init: set an unknown flag in the bitset */ + if (pci_read_config_dword(dev, 4, &cfg)) + dev_warn(&dev->dev, "t2bce_audio: pci_read_config_dword fail\n"); + if (pci_write_config_dword(dev, 4, cfg | 6u)) + dev_warn(&dev->dev, "t2bce_audio: pci_write_config_dword fail\n"); + + pr_debug("t2bce_audio: bs len = %llx\n", pci_resource_len(dev, 0)); + t2audio->reg_mem_bs_dma = pci_resource_start(dev, 0); + t2audio->reg_mem_bs = pci_iomap(dev, 0, 0); + t2audio->reg_mem_cfg = pci_iomap(dev, 4, 0); + + t2audio->reg_mem_gpr = (u32 __iomem *) ((u8 __iomem *) t2audio->reg_mem_cfg + 0xC000); + + if (IS_ERR_OR_NULL(t2audio->reg_mem_bs) || IS_ERR_OR_NULL(t2audio->reg_mem_cfg)) { + dev_warn(&dev->dev, "t2bce_audio: Failed to pci_iomap required regions\n"); + goto fail; + } + + if (t2audio_bce_init(t2audio)) { + dev_warn(&dev->dev, "t2bce_audio: Failed to init BCE command transport\n"); + goto fail; + } + + if (snd_card_new(t2audio->dev, t2audio_alsa_index, t2audio_alsa_id, THIS_MODULE, 0, &t2audio->card)) { + dev_err(&dev->dev, "t2bce_audio: Failed to create ALSA card\n"); + goto fail; + } + + strcpy(t2audio->card->shortname, "Apple T2 Audio"); + strcpy(t2audio->card->longname, "Apple T2 Audio"); + strcpy(t2audio->card->mixername, "Apple T2 Audio"); + /* Dynamic alsa ids start at 100 */ + t2audio->next_alsa_id = 100; + + if (t2audio_init_cmd(t2audio)) { + dev_err(&dev->dev, "t2bce_audio: Failed to initialize over BCE\n"); + goto fail_snd; + } + + if (t2audio_init_bs(t2audio)) { + dev_err(&dev->dev, "t2bce_audio: Failed to initialize BufferStruct\n"); + goto fail_snd; + } + + if ((status = t2audio_cmd_set_remote_access(t2audio, T2AUDIO_REMOTE_ACCESS_ON))) { + dev_err(&dev->dev, "Failed to set remote access\n"); + goto fail_snd; + } + + if (snd_card_register(t2audio->card)) { + dev_err(&dev->dev, "t2bce_audio: Failed to register ALSA sound device\n"); + goto fail_snd; + } + + list_for_each_entry(sdev, &t2audio->subdevice_list, list) { + struct t2audio_buffer_struct_device *dev = &t2audio->bs->devices[sdev->buf_id]; + + if (sdev->out_stream_cnt == 1 && !strcmp(dev->name, "Speaker")) { + struct snd_pcm_hardware *hw = sdev->out_streams[0].alsa_hw_desc; + + snprintf(t2audio->card->driver, sizeof(t2audio->card->driver) / sizeof(char), "AppleT2x%d", hw->channels_min); + } + } + + return 0; + +fail_snd: + snd_card_free(t2audio->card); +fail: + if (t2audio) { + if (t2audio->dev) + device_destroy(t2audio_class, t2audio->devt); + if (!IS_ERR_OR_NULL(t2audio->reg_mem_bs)) + pci_iounmap(dev, t2audio->reg_mem_bs); + if (!IS_ERR_OR_NULL(t2audio->reg_mem_cfg)) + pci_iounmap(dev, t2audio->reg_mem_cfg); + if (t2audio->bce) + t2bce_core_client_set_pm_ops(t2audio->bce, NULL, NULL); + t2bce_core_client_put(t2audio->bce); + kfree(t2audio); + } + + pci_release_regions(dev); + pci_disable_device(dev); + + if (!status) + status = -EINVAL; + return status; +} + + + +static void t2audio_remove(struct pci_dev *dev) +{ + struct t2audio_subdevice *sdev; + struct t2audio_device *t2audio = pci_get_drvdata(dev); + + cancel_work_sync(&t2audio->resume_work); + snd_card_free(t2audio->card); + while (!list_empty(&t2audio->subdevice_list)) { + sdev = list_first_entry(&t2audio->subdevice_list, struct t2audio_subdevice, list); + list_del(&sdev->list); + t2audio_free_dev(sdev); + } + pci_iounmap(dev, t2audio->reg_mem_bs); + pci_iounmap(dev, t2audio->reg_mem_cfg); + device_destroy(t2audio_class, t2audio->devt); + pci_free_irq_vectors(dev); + pci_release_regions(dev); + pci_disable_device(dev); + t2bce_core_client_set_pm_ops(t2audio->bce, NULL, NULL); + t2bce_core_client_put(t2audio->bce); + kfree(t2audio); +} + +static int t2audio_quiesce(struct t2audio_device *t2audio, bool suspend_pcm) +{ + struct t2audio_subdevice *sdev; + size_t i; + int status; + + if (t2audio->pm_quiesced) + return 0; + + cancel_work_sync(&t2audio->resume_work); + t2audio->resume_deferred = false; + + /* Suspend PCM streams */ + list_for_each_entry(sdev, &t2audio->subdevice_list, list) { + bool stopped_io = false; + + for (i = 0; i < sdev->out_stream_cnt; i++) { + if (!smp_load_acquire(&sdev->out_streams[i].started)) + continue; + stopped_io = true; + smp_store_release(&sdev->out_streams[i].started, 0); + } + + for (i = 0; i < sdev->in_stream_cnt; i++) { + if (!smp_load_acquire(&sdev->in_streams[i].started)) + continue; + stopped_io = true; + smp_store_release(&sdev->in_streams[i].started, 0); + } + + if (stopped_io) + t2audio_cmd_stop_io(sdev->a, sdev->dev_id); + + if (suspend_pcm && sdev->pcm) + snd_pcm_suspend_all(sdev->pcm); + } + + status = t2audio_cmd_set_remote_access(t2audio, T2AUDIO_REMOTE_ACCESS_OFF); + if (status) + dev_warn(t2audio->dev, "Failed to reset remote access\n"); + else + t2audio->pm_quiesced = true; + + return status; +} + +static int t2audio_suspend(struct device *dev) +{ + struct t2audio_device *t2audio = pci_get_drvdata(to_pci_dev(dev)); + int status; + + pr_debug("t2bce_audio: suspend entry\n"); + status = t2audio_quiesce(t2audio, true); + pci_disable_device(t2audio->pci); + pr_info("t2bce_audio: suspend exit status=%d\n", status); + return 0; +} + +static void t2audio_pm_prepare_client(void *userdata) +{ + t2audio_quiesce(userdata, true); +} + +static void t2audio_pm_shutdown_client(void *userdata) +{ + t2audio_quiesce(userdata, false); +} + +static void t2audio_shutdown(struct pci_dev *dev) +{ + struct t2audio_device *t2audio = pci_get_drvdata(dev); + + if (!t2audio) + return; + + /* + * Shutdown is not remove: keep allocations intact, but stop active audio + * IO and revoke remote access while the BCE command transport is still + * alive. The t2bce shutdown path will close the shared bus afterwards. + */ + t2audio_quiesce(t2audio, false); + pci_disable_device(t2audio->pci); +} + +static int t2audio_resume(struct device *dev) +{ + int status; + struct t2audio_device *t2audio = pci_get_drvdata(to_pci_dev(dev)); + bool no_state_resume = t2bce_core_client_no_state_resume(t2audio->bce); + const char *path = no_state_resume ? "no-state" : "stateful"; + + if ((status = pci_enable_device(t2audio->pci))) { + pr_info("t2bce_audio: resume exit status=%d path=%s\n", status, path); + return status; + } + pci_set_master(t2audio->pci); + + /* we are deferring t2audio resume here until vhci is finished*/ + if (!no_state_resume) { + t2audio->resume_deferred = true; + return 0; + } + + if ((status = t2audio_cmd_set_remote_access(t2audio, T2AUDIO_REMOTE_ACCESS_ON))) { + dev_err(t2audio->dev, "Failed to set remote access\n"); + pr_info("t2bce_audio: resume exit status=%d path=%s\n", status, path); + return status; + } + + t2audio->resume_deferred = false; + t2audio->pm_quiesced = false; + t2audio_reset_clock(t2audio); + t2audio_reset_streams(t2audio); + + pr_info("t2bce_audio: resume exit status=0 path=%s\n", path); + return 0; +} + +static void t2audio_resume_work(struct work_struct *ws) +{ + struct t2audio_device *t2audio = container_of(ws, struct t2audio_device, resume_work); + + if (!t2audio->resume_deferred) + return; + + if (t2audio_cmd_set_remote_access(t2audio, T2AUDIO_REMOTE_ACCESS_ON)) { + t2audio->resume_deferred = false; + dev_err(t2audio->dev, "Deferred remote access enable failed\n"); + pr_info("t2bce_audio: resume deferred path failed\n"); + return; + } + + t2audio->resume_deferred = false; + t2audio->pm_quiesced = false; + t2audio_reset_clock(t2audio); + t2audio_reset_streams(t2audio); + pr_info("t2bce_audio: resume deferred path complete\n"); +} + +static void t2audio_resume_complete(void *userdata) +{ + struct t2audio_device *t2audio = userdata; + + if (!t2audio || !t2audio->resume_deferred) + return; + + schedule_work(&t2audio->resume_work); +} + +static void t2audio_reset_stream(struct t2audio_stream *stream) +{ + smp_store_release(&stream->started, 0); + stream->waiting_for_first_ts = true; + stream->remote_timestamp = 0; + stream->timestamp_accept_after = 0; + stream->clock_offset_ns = 0; + stream->timestamp_seed = 0; + stream->clock_offset_valid = false; + stream->frame_min = stream->latency; +} + +static void t2audio_reset_clock(struct t2audio_device *a) +{ + unsigned long flags; + + spin_lock_irqsave(&a->clock_lock, flags); + a->clock_offset_ns = 0; + a->clock_samples = 0; + a->clock_offset_valid = false; + spin_unlock_irqrestore(&a->clock_lock, flags); +} + +static void t2audio_reset_streams(struct t2audio_device *a) +{ + struct t2audio_subdevice *sdev; + size_t i; + + list_for_each_entry(sdev, &a->subdevice_list, list) { + for (i = 0; i < sdev->in_stream_cnt; i++) + t2audio_reset_stream(&sdev->in_streams[i]); + for (i = 0; i < sdev->out_stream_cnt; i++) + t2audio_reset_stream(&sdev->out_streams[i]); + } +} + +static int t2audio_init_cmd(struct t2audio_device *a) +{ + int status; + struct t2audio_send_ctx sctx; + struct t2audio_msg buf; + u64 dev_cnt, dev_i; + t2audio_device_id_t *dev_l; + + if ((status = t2audio_send(a, &sctx, 500, + t2audio_msg_write_alive_notification, 1, 3))) { + dev_err(a->dev, "Sending alive notification failed\n"); + return status; + } + + if (wait_for_completion_timeout(&a->remote_alive, msecs_to_jiffies(500)) == 0) { + dev_err(a->dev, "Timed out waiting for remote\n"); + return -ETIMEDOUT; + } + pr_debug("t2bce_audio: Continuing init\n"); + + buf = t2audio_reply_alloc(); + if ((status = t2audio_cmd_get_device_list(a, &buf, &dev_l, &dev_cnt))) { + dev_err(a->dev, "Failed to get device list\n"); + t2audio_reply_free(&buf); + return status; + } + for (dev_i = 0; dev_i < dev_cnt; ++dev_i) + t2audio_init_dev(a, dev_l[dev_i]); + t2audio_reply_free(&buf); + + return 0; +} + +static void t2audio_init_stream_info(struct t2audio_subdevice *sdev, struct t2audio_stream *strm); +static void t2audio_handle_jack_connection_change(struct t2audio_subdevice *sdev); + +static void t2audio_init_dev(struct t2audio_device *a, t2audio_device_id_t dev_id) +{ + struct t2audio_subdevice *sdev; + struct t2audio_msg buf = t2audio_reply_alloc(); + u64 uid_len, stream_cnt, i; + t2audio_object_id_t *stream_list; + char *uid; + + sdev = kzalloc(sizeof(struct t2audio_subdevice), GFP_KERNEL); + + if (t2audio_cmd_get_property(a, &buf, dev_id, dev_id, T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_GLOBAL, T2AUDIO_PROP_UID, 0), + NULL, 0, (void **) &uid, &uid_len) || uid_len > T2AUDIO_DEVICE_MAX_UID_LEN) { + dev_err(a->dev, "Failed to get device uid for device %llx\n", dev_id); + goto fail; + } + uid[uid_len] = '\0'; + pr_debug("t2bce_audio: Remote device %llx %.*s\n", dev_id, (int) uid_len, uid); + + sdev->a = a; + INIT_LIST_HEAD(&sdev->list); + sdev->dev_id = dev_id; + sdev->buf_id = T2AUDIO_BUFFER_ID_NONE; + strscpy(sdev->uid, uid); + + if (t2audio_cmd_get_primitive_property(a, dev_id, dev_id, + T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_INPUT, T2AUDIO_PROP_LATENCY, 0), NULL, 0, &sdev->in_latency, sizeof(u32))) + dev_warn(a->dev, "Failed to query device input latency\n"); + if (t2audio_cmd_get_primitive_property(a, dev_id, dev_id, + T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_OUTPUT, T2AUDIO_PROP_LATENCY, 0), NULL, 0, &sdev->out_latency, sizeof(u32))) + dev_warn(a->dev, "Failed to query device output latency\n"); + + if (t2audio_cmd_get_input_stream_list(a, &buf, dev_id, &stream_list, &stream_cnt)) { + dev_err(a->dev, "Failed to get input stream list for device %llx\n", dev_id); + goto fail; + } + if (stream_cnt > T2AUDIO_DEVICE_MAX_INPUT_STREAMS) { + dev_warn(a->dev, "Device %s input stream count %llu is larger than the supported count of %u\n", + sdev->uid, stream_cnt, T2AUDIO_DEVICE_MAX_INPUT_STREAMS); + stream_cnt = T2AUDIO_DEVICE_MAX_INPUT_STREAMS; + } + sdev->in_stream_cnt = stream_cnt; + for (i = 0; i < stream_cnt; i++) { + sdev->in_streams[i].id = stream_list[i]; + sdev->in_streams[i].buffer_cnt = 0; + t2audio_init_stream_info(sdev, &sdev->in_streams[i]); + sdev->in_streams[i].latency += sdev->in_latency; + } + + if (t2audio_cmd_get_output_stream_list(a, &buf, dev_id, &stream_list, &stream_cnt)) { + dev_err(a->dev, "Failed to get output stream list for device %llx\n", dev_id); + goto fail; + } + if (stream_cnt > T2AUDIO_DEVICE_MAX_OUTPUT_STREAMS) { + dev_warn(a->dev, "Device %s output stream count %llu is larger than the supported count of %u\n", + sdev->uid, stream_cnt, T2AUDIO_DEVICE_MAX_OUTPUT_STREAMS); + stream_cnt = T2AUDIO_DEVICE_MAX_OUTPUT_STREAMS; + } + sdev->out_stream_cnt = stream_cnt; + for (i = 0; i < stream_cnt; i++) { + sdev->out_streams[i].id = stream_list[i]; + sdev->out_streams[i].buffer_cnt = 0; + t2audio_init_stream_info(sdev, &sdev->out_streams[i]); + sdev->out_streams[i].latency += sdev->out_latency; + } + + if (sdev->is_pcm) + t2audio_create_pcm(sdev); + /* Headphone Jack status */ + if (!strcmp(sdev->uid, "Codec Output")) { + if (snd_jack_new(a->card, sdev->uid, SND_JACK_HEADPHONE, &sdev->jack, true, false)) + dev_warn(a->dev, "Failed to create an attached jack for %s\n", sdev->uid); + t2audio_cmd_property_listener(a, sdev->dev_id, sdev->dev_id, + T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_OUTPUT, T2AUDIO_PROP_JACK_PLUGGED, 0)); + t2audio_handle_jack_connection_change(sdev); + } + + t2audio_reply_free(&buf); + + list_add_tail(&sdev->list, &a->subdevice_list); + return; + +fail: + t2audio_reply_free(&buf); + kfree(sdev); +} + +static void t2audio_init_stream_info(struct t2audio_subdevice *sdev, struct t2audio_stream *strm) +{ + if (t2audio_cmd_get_primitive_property(sdev->a, sdev->dev_id, strm->id, + T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_GLOBAL, T2AUDIO_PROP_PHYS_FORMAT, 0), NULL, 0, + &strm->desc, sizeof(strm->desc))) + dev_warn(sdev->a->dev, "Failed to query stream descriptor\n"); + if (t2audio_cmd_get_primitive_property(sdev->a, sdev->dev_id, strm->id, + T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_GLOBAL, T2AUDIO_PROP_LATENCY, 0), NULL, 0, &strm->latency, sizeof(u32))) + dev_warn(sdev->a->dev, "Failed to query stream latency\n"); + if (strm->desc.format_id == T2AUDIO_FORMAT_LPCM) + sdev->is_pcm = true; +} + +static void t2audio_free_dev(struct t2audio_subdevice *sdev) +{ + size_t i; + for (i = 0; i < sdev->in_stream_cnt; i++) { + if (sdev->in_streams[i].alsa_hw_desc) + kfree(sdev->in_streams[i].alsa_hw_desc); + if (sdev->in_streams[i].buffers) + kfree(sdev->in_streams[i].buffers); + } + for (i = 0; i < sdev->out_stream_cnt; i++) { + if (sdev->out_streams[i].alsa_hw_desc) + kfree(sdev->out_streams[i].alsa_hw_desc); + if (sdev->out_streams[i].buffers) + kfree(sdev->out_streams[i].buffers); + } + kfree(sdev); +} + +static struct t2audio_subdevice *t2audio_find_dev_by_dev_id(struct t2audio_device *a, t2audio_device_id_t dev_id) +{ + struct t2audio_subdevice *sdev; + list_for_each_entry(sdev, &a->subdevice_list, list) { + if (dev_id == sdev->dev_id) + return sdev; + } + return NULL; +} + +static struct t2audio_subdevice *t2audio_find_dev_by_uid(struct t2audio_device *a, const char *uid) +{ + struct t2audio_subdevice *sdev; + list_for_each_entry(sdev, &a->subdevice_list, list) { + if (!strcmp(uid, sdev->uid)) + return sdev; + } + return NULL; +} + +static void t2audio_init_bs_stream(struct t2audio_device *a, struct t2audio_stream *strm, + struct t2audio_buffer_struct_stream *bs_strm); +static void t2audio_init_bs_stream_host(struct t2audio_device *a, struct t2audio_stream *strm, + struct t2audio_buffer_struct_stream *bs_strm); + +static int t2audio_init_bs(struct t2audio_device *a) +{ + int i, j; + struct t2audio_buffer_struct_device *dev; + struct t2audio_subdevice *sdev; + u32 ver, sig, bs_base; + + ver = ioread32(&a->reg_mem_gpr[0]); + if (ver < 3) { + dev_err(a->dev, "t2bce_audio: Bad GPR version (%u)", ver); + return -EINVAL; + } + sig = ioread32(&a->reg_mem_gpr[1]); + if (sig != T2AUDIO_SIG) { + dev_err(a->dev, "t2bce_audio: Bad GPR sig (%x)", sig); + return -EINVAL; + } + bs_base = ioread32(&a->reg_mem_gpr[2]); + a->bs = (struct t2audio_buffer_struct *) ((u8 *) a->reg_mem_bs + bs_base); + if (a->bs->signature != T2AUDIO_SIG) { + dev_err(a->dev, "t2bce_audio: Bad BufferStruct sig (%x)", a->bs->signature); + return -EINVAL; + } + pr_debug("t2bce_audio: BufferStruct ver = %i\n", a->bs->version); + pr_debug("t2bce_audio: Num devices = %i\n", a->bs->num_devices); + for (i = 0; i < a->bs->num_devices; i++) { + dev = &a->bs->devices[i]; + pr_debug("t2bce_audio: Device %i %s\n", i, dev->name); + + sdev = t2audio_find_dev_by_uid(a, dev->name); + if (!sdev) { + dev_err(a->dev, "t2bce_audio: Subdevice not found for BufferStruct device %s\n", dev->name); + continue; + } + sdev->buf_id = (u8) i; + dev->num_input_streams = 0; + for (j = 0; j < dev->num_output_streams; j++) { + pr_debug("t2bce_audio: Device %i Stream %i: Output; Buffer Count = %i\n", i, j, + dev->output_streams[j].num_buffers); + if (j < sdev->out_stream_cnt) + t2audio_init_bs_stream(a, &sdev->out_streams[j], &dev->output_streams[j]); + } + } + + list_for_each_entry(sdev, &a->subdevice_list, list) { + if (sdev->buf_id != T2AUDIO_BUFFER_ID_NONE) + continue; + sdev->buf_id = i; + pr_debug("t2bce_audio: Created device %i %s\n", i, sdev->uid); + strcpy(a->bs->devices[i].name, sdev->uid); + a->bs->devices[i].num_input_streams = 0; + a->bs->devices[i].num_output_streams = 0; + a->bs->num_devices = ++i; + } + list_for_each_entry(sdev, &a->subdevice_list, list) { + if (sdev->in_stream_cnt == 1) { + pr_debug("t2bce_audio: Device %i Host Stream; Input\n", sdev->buf_id); + t2audio_init_bs_stream_host(a, &sdev->in_streams[0], &a->bs->devices[sdev->buf_id].input_streams[0]); + a->bs->devices[sdev->buf_id].num_input_streams = 1; + wmb(); + + if (t2audio_cmd_set_input_stream_address_ranges(a, sdev->dev_id)) { + dev_err(a->dev, "t2bce_audio: Failed to set input stream address ranges\n"); + } + } + } + + return 0; +} + +static void t2audio_init_bs_stream(struct t2audio_device *a, struct t2audio_stream *strm, + struct t2audio_buffer_struct_stream *bs_strm) +{ + size_t i; + strm->buffer_cnt = bs_strm->num_buffers; + if (bs_strm->num_buffers > T2AUDIO_DEVICE_MAX_BUFFER_COUNT) { + dev_warn(a->dev, "BufferStruct buffer count %u exceeds driver limit of %u\n", bs_strm->num_buffers, + T2AUDIO_DEVICE_MAX_BUFFER_COUNT); + strm->buffer_cnt = T2AUDIO_DEVICE_MAX_BUFFER_COUNT; + } + if (!strm->buffer_cnt) + return; + strm->buffers = kmalloc_array(strm->buffer_cnt, sizeof(struct t2audio_dma_buf), GFP_KERNEL); + if (!strm->buffers) { + dev_err(a->dev, "Buffer list allocation failed\n"); + return; + } + for (i = 0; i < strm->buffer_cnt; i++) { + strm->buffers[i].dma_addr = a->reg_mem_bs_dma + (dma_addr_t) bs_strm->buffers[i].address; + strm->buffers[i].ptr = a->reg_mem_bs + bs_strm->buffers[i].address; + strm->buffers[i].size = bs_strm->buffers[i].size; + strm->buffers[i].type = T2AUDIO_DMA_BUF_IOMEM; + } + + if (strm->buffer_cnt == 1) { + strm->alsa_hw_desc = kmalloc(sizeof(struct snd_pcm_hardware), GFP_KERNEL); + if (t2audio_create_hw_info(&strm->desc, strm->alsa_hw_desc, strm->buffers[0].size)) { + kfree(strm->alsa_hw_desc); + strm->alsa_hw_desc = NULL; + } + } +} + +static void t2audio_init_bs_stream_host(struct t2audio_device *a, struct t2audio_stream *strm, + struct t2audio_buffer_struct_stream *bs_strm) +{ + size_t size; + dma_addr_t dma_addr; + void *dma_ptr; + size = strm->desc.bytes_per_packet * 16640; + dma_ptr = dma_alloc_coherent(&a->pci->dev, size, &dma_addr, GFP_KERNEL); + if (!dma_ptr) { + dev_err(a->dev, "dma_alloc_coherent failed\n"); + return; + } + bs_strm->buffers[0].address = dma_addr; + bs_strm->buffers[0].size = size; + bs_strm->num_buffers = 1; + + memset(dma_ptr, 0, size); + + strm->buffer_cnt = 1; + strm->buffers = kmalloc_array(strm->buffer_cnt, sizeof(struct t2audio_dma_buf), GFP_KERNEL); + if (!strm->buffers) { + dev_err(a->dev, "Buffer list allocation failed\n"); + return; + } + strm->buffers[0].dma_addr = dma_addr; + strm->buffers[0].ptr = dma_ptr; + strm->buffers[0].size = size; + strm->buffers[0].type = T2AUDIO_DMA_BUF_COHERENT; + + strm->alsa_hw_desc = kmalloc(sizeof(struct snd_pcm_hardware), GFP_KERNEL); + if (t2audio_create_hw_info(&strm->desc, strm->alsa_hw_desc, strm->buffers[0].size)) { + kfree(strm->alsa_hw_desc); + strm->alsa_hw_desc = NULL; + } +} + +static void t2audio_handle_prop_change(struct t2audio_device *a, struct t2audio_msg *msg); + +void t2audio_handle_notification(struct t2audio_device *a, struct t2audio_msg *msg) +{ + struct t2audio_send_ctx sctx; + struct t2audio_msg_base base; + if (t2audio_msg_read_base(msg, &base)) + return; + switch (base.msg) { + case T2AUDIO_MSG_NOTIFICATION_BOOT: + pr_debug("t2bce_audio: Received boot notification from remote\n"); + + /* Resend the alive notify */ + if (t2audio_send(a, &sctx, 500, + t2audio_msg_write_alive_notification, 1, 3)) { + pr_err("Sending alive notification failed\n"); + } + break; + case T2AUDIO_MSG_NOTIFICATION_ALIVE: + pr_debug("t2bce_audio: Received alive notification from remote\n"); + complete_all(&a->remote_alive); + break; + case T2AUDIO_MSG_PROPERTY_CHANGED: + t2audio_handle_prop_change(a, msg); + break; + default: + pr_debug("t2bce_audio: Unhandled notification %i\n", base.msg); + break; + } +} + +struct t2audio_prop_change_work_struct { + struct work_struct ws; + struct t2audio_device *a; + t2audio_device_id_t dev; + t2audio_object_id_t obj; + struct t2audio_prop_addr prop; +}; + +static void t2audio_handle_jack_connection_change(struct t2audio_subdevice *sdev) +{ + u32 plugged; + if (!sdev->jack) + return; + /* NOTE: Apple made the plug status scoped to the input and output streams. This makes no sense for us, so I just + * always pick the OUTPUT status. */ + if (t2audio_cmd_get_primitive_property(sdev->a, sdev->dev_id, sdev->dev_id, + T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_OUTPUT, T2AUDIO_PROP_JACK_PLUGGED, 0), NULL, 0, &plugged, sizeof(plugged))) { + dev_err(sdev->a->dev, "Failed to get jack enable status\n"); + return; + } + pr_debug("t2bce_audio: Jack is now %s\n", plugged ? "plugged" : "unplugged"); + snd_jack_report(sdev->jack, plugged ? sdev->jack->type : 0); +} + +void t2audio_handle_prop_change_work(struct work_struct *ws) +{ + struct t2audio_prop_change_work_struct *work = container_of(ws, struct t2audio_prop_change_work_struct, ws); + struct t2audio_subdevice *sdev; + + sdev = t2audio_find_dev_by_dev_id(work->a, work->dev); + if (!sdev) { + dev_err(work->a->dev, "Property notification change: device not found\n"); + goto done; + } + pr_debug("t2bce_audio: Property changed for device: %s\n", sdev->uid); + + if (work->prop.scope == T2AUDIO_PROP_SCOPE_OUTPUT && work->prop.selector == T2AUDIO_PROP_JACK_PLUGGED) { + t2audio_handle_jack_connection_change(sdev); + } + +done: + kfree(work); +} + +void t2audio_handle_prop_change(struct t2audio_device *a, struct t2audio_msg *msg) +{ + /* NOTE: This is a scheduled work because this callback will generally need to query device information and this + * is not possible when we are in the reply parsing code's context. */ + struct t2audio_prop_change_work_struct *work; + work = kmalloc(sizeof(struct t2audio_prop_change_work_struct), GFP_KERNEL); + work->a = a; + INIT_WORK(&work->ws, t2audio_handle_prop_change_work); + t2audio_msg_read_property_changed(msg, &work->dev, &work->obj, &work->prop); + schedule_work(&work->ws); +} + +#define t2audio_send_cmd_response(a, sctx, msg, fn, ...) \ + if (t2audio_send_with_tag(a, sctx, ((struct t2audio_msg_header *) msg->data)->tag, 500, fn, ##__VA_ARGS__)) \ + pr_err("t2bce_audio: Failed to reply to a command\n"); + +void t2audio_handle_cmd_timestamp(struct t2audio_device *a, struct t2audio_msg *msg) +{ + ktime_t time_os = ktime_get_boottime(); + unsigned long flags; + struct t2audio_send_ctx sctx; + struct t2audio_subdevice *sdev; + bool clock_ready; + s64 clock_offset_ns; + s64 offset_sample; + u64 devid, timestamp, update_seed; + t2audio_msg_read_update_timestamp(msg, &devid, ×tamp, &update_seed); + + offset_sample = ktime_to_ns(time_os) - (s64)timestamp; + spin_lock_irqsave(&a->clock_lock, flags); + if (offset_sample >= 0) { + if (!a->clock_offset_valid || offset_sample < a->clock_offset_ns) { + a->clock_offset_ns = offset_sample; + a->clock_offset_valid = true; + } + if (a->clock_samples < 2) + a->clock_samples++; + } + clock_offset_ns = a->clock_offset_ns; + clock_ready = a->clock_samples >= 2; + spin_unlock_irqrestore(&a->clock_lock, flags); + pr_debug("t2bce_audio: timestamp dev=%llx t2=%llx host=%lld seed=%llx sample_offset=%lld clock_offset=%lld\n", + devid, timestamp, ktime_to_ns(time_os), update_seed, + offset_sample, clock_offset_ns); + + sdev = t2audio_find_dev_by_dev_id(a, devid); + if (sdev) + t2audio_handle_timestamp(sdev, timestamp, update_seed, clock_offset_ns, + clock_ready); + + t2audio_send_cmd_response(a, &sctx, msg, + t2audio_msg_write_update_timestamp_response); +} + +void t2audio_handle_command(struct t2audio_device *a, struct t2audio_msg *msg) +{ + struct t2audio_msg_base base; + if (t2audio_msg_read_base(msg, &base)) + return; + switch (base.msg) { + case T2AUDIO_MSG_UPDATE_TIMESTAMP: + t2audio_handle_cmd_timestamp(a, msg); + break; + default: + pr_debug("t2bce_audio: Unhandled device command %i\n", base.msg); + break; + } +} + +static struct pci_device_id t2audio_ids[ ] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1803) }, + { 0, }, +}; +MODULE_DEVICE_TABLE(pci, t2audio_ids); + +struct dev_pm_ops t2audio_pci_driver_pm = { + .suspend = t2audio_suspend, + .resume = t2audio_resume +}; +struct pci_driver t2audio_pci_driver = { + .name = "t2bce_audio", + .id_table = t2audio_ids, + .probe = t2audio_probe, + .remove = t2audio_remove, + .shutdown = t2audio_shutdown, + .driver = { + .pm = &t2audio_pci_driver_pm + } +}; + + +static int __init t2audio_module_init(void) +{ + int result; + if ((result = alloc_chrdev_region(&t2audio_chrdev, 0, 1, "t2bce_audio"))) + goto fail_chrdev; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0) + t2audio_class = class_create(THIS_MODULE, "t2bce_audio"); +#else + t2audio_class = class_create("t2bce_audio"); +#endif + if (IS_ERR(t2audio_class)) { + result = PTR_ERR(t2audio_class); + goto fail_class; + } + + result = pci_register_driver(&t2audio_pci_driver); + if (result) + goto fail_drv; + pr_info("t2bce_audio: module initialized\n"); + return 0; + +fail_drv: + pci_unregister_driver(&t2audio_pci_driver); +fail_class: + class_destroy(t2audio_class); +fail_chrdev: + unregister_chrdev_region(t2audio_chrdev, 1); + if (!result) + result = -EINVAL; + pr_info("t2bce_audio: module init failed status=%d\n", result); + return result; +} + +static void __exit t2audio_module_exit(void) +{ + pci_unregister_driver(&t2audio_pci_driver); + class_destroy(t2audio_class); + unregister_chrdev_region(t2audio_chrdev, 1); + pr_info("t2bce_audio: module exited\n"); +} + +struct t2audio_alsa_pcm_id_mapping t2audio_alsa_id_mappings[] = { + {"Speaker", 0}, + {"Digital Mic", 1}, + {"Codec Output", 2}, + {"Codec Input", 3}, + {"Bridge Loopback", 4}, + {} +}; + +module_param_named(index, t2audio_alsa_index, int, 0444); +MODULE_PARM_DESC(index, "Index value for Apple Internal Audio soundcard."); +module_param_named(id, t2audio_alsa_id, charp, 0444); +MODULE_PARM_DESC(id, "ID string for Apple Internal Audio soundcard."); +MODULE_SOFTDEP("pre: t2bce_core"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("André Eikmeyer "); +MODULE_DESCRIPTION("Apple T2 Audio Driver"); +MODULE_VERSION("0.01"); +module_init(t2audio_module_init); +module_exit(t2audio_module_exit); diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_audio/audio.h b/drivers/staging/t2bce/t2bce_audio/audio.h --- a/drivers/staging/t2bce/t2bce_audio/audio.h 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_audio/audio.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,148 @@ +#ifndef T2AUDIO_H +#define T2AUDIO_H + +#include +#include +#include +#include "t2bce_core_transport.h" +#include "protocol_bce.h" +#include "description.h" + +#define T2AUDIO_SIG 0x19870423 + +#define T2AUDIO_DEVICE_MAX_UID_LEN 128 +#define T2AUDIO_DEVICE_MAX_INPUT_STREAMS 1 +#define T2AUDIO_DEVICE_MAX_OUTPUT_STREAMS 1 +#define T2AUDIO_DEVICE_MAX_BUFFER_COUNT 1 + +#define T2AUDIO_BUFFER_ID_NONE 0xffu + +struct snd_card; +struct snd_pcm; +struct snd_pcm_hardware; +struct snd_jack; + +struct __attribute__((packed)) __attribute__((aligned(4))) t2audio_buffer_struct_buffer { + size_t address; + size_t size; + size_t pad[4]; +}; +struct t2audio_buffer_struct_stream { + u8 num_buffers; + struct t2audio_buffer_struct_buffer buffers[100]; + char filler[32]; +}; +struct t2audio_buffer_struct_device { + char name[128]; + u8 num_input_streams; + u8 num_output_streams; + struct t2audio_buffer_struct_stream input_streams[5]; + struct t2audio_buffer_struct_stream output_streams[5]; + char filler[128]; +}; +struct t2audio_buffer_struct { + u32 version; + u32 signature; + u32 flags; + u8 num_devices; + struct t2audio_buffer_struct_device devices[20]; +}; + +struct t2audio_device; + +struct t2audio_deferred_msg { + struct work_struct ws; + struct t2audio_device *a; + struct t2audio_msg msg; +}; + +enum t2audio_dma_buf_type { + T2AUDIO_DMA_BUF_IOMEM, + T2AUDIO_DMA_BUF_COHERENT, +}; + +struct t2audio_dma_buf { + dma_addr_t dma_addr; + void *ptr; + size_t size; + enum t2audio_dma_buf_type type; +}; +struct t2audio_stream { + t2audio_object_id_t id; + size_t buffer_cnt; + struct t2audio_dma_buf *buffers; + + struct t2audio_apple_description desc; + struct snd_pcm_hardware *alsa_hw_desc; + u32 latency; + + bool waiting_for_first_ts; + + ktime_t remote_timestamp; + ktime_t timestamp_accept_after; + s64 clock_offset_ns; + u64 timestamp_seed; + bool clock_offset_valid; + snd_pcm_sframes_t frame_min; + int started; +}; +struct t2audio_subdevice { + struct t2audio_device *a; + struct list_head list; + t2audio_device_id_t dev_id; + u32 in_latency, out_latency; + u8 buf_id; + int alsa_id; + char uid[T2AUDIO_DEVICE_MAX_UID_LEN + 1]; + size_t in_stream_cnt; + struct t2audio_stream in_streams[T2AUDIO_DEVICE_MAX_INPUT_STREAMS]; + size_t out_stream_cnt; + struct t2audio_stream out_streams[T2AUDIO_DEVICE_MAX_OUTPUT_STREAMS]; + bool is_pcm; + struct snd_pcm *pcm; + struct snd_jack *jack; +}; +struct t2audio_alsa_pcm_id_mapping { + const char *name; + int alsa_id; +}; + +struct t2audio_device { + struct pci_dev *pci; + dev_t devt; + struct device *dev; + void __iomem *reg_mem_bs; + dma_addr_t reg_mem_bs_dma; + void __iomem *reg_mem_cfg; + + u32 __iomem *reg_mem_gpr; + + struct t2audio_buffer_struct *bs; + + struct t2bce_core_client *bce; + struct t2audio_bce bcem; + + struct snd_card *card; + + struct list_head subdevice_list; + int next_alsa_id; + + struct completion remote_alive; + struct work_struct resume_work; + bool resume_deferred; + bool pm_quiesced; + + spinlock_t clock_lock; + s64 clock_offset_ns; + u32 clock_samples; + bool clock_offset_valid; +}; + +void t2audio_handle_notification(struct t2audio_device *a, struct t2audio_msg *msg); +void t2audio_handle_prop_change_work(struct work_struct *ws); +void t2audio_handle_cmd_timestamp(struct t2audio_device *a, struct t2audio_msg *msg); +void t2audio_handle_command(struct t2audio_device *a, struct t2audio_msg *msg); + +extern struct t2audio_alsa_pcm_id_mapping t2audio_alsa_id_mappings[]; + +#endif //T2AUDIO_H diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_audio/description.h b/drivers/staging/t2bce/t2bce_audio/description.h --- a/drivers/staging/t2bce/t2bce_audio/description.h 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_audio/description.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,42 @@ +#ifndef T2AUDIO_DESCRIPTION_H +#define T2AUDIO_DESCRIPTION_H + +#include + +struct t2audio_apple_description { + u64 sample_rate_double; + u32 format_id; + u32 format_flags; + u32 bytes_per_packet; + u32 frames_per_packet; + u32 bytes_per_frame; + u32 channels_per_frame; + u32 bits_per_channel; + u32 reserved; +}; + +enum { + T2AUDIO_FORMAT_LPCM = 0x6c70636d // 'lpcm' +}; + +enum { + T2AUDIO_FORMAT_FLAG_FLOAT = 1, + T2AUDIO_FORMAT_FLAG_BIG_ENDIAN = 2, + T2AUDIO_FORMAT_FLAG_SIGNED = 4, + T2AUDIO_FORMAT_FLAG_PACKED = 8, + T2AUDIO_FORMAT_FLAG_ALIGNED_HIGH = 16, + T2AUDIO_FORMAT_FLAG_NON_INTERLEAVED = 32, + T2AUDIO_FORMAT_FLAG_NON_MIXABLE = 64 +}; + +static inline u64 t2audio_double_to_u64(u64 d) +{ + u8 sign = (u8) ((d >> 63) & 1); + s32 exp = (s32) ((d >> 52) & 0x7ff) - 1023; + u64 fr = d & ((1LL << 52) - 1); + if (sign || exp < 0) + return 0; + return (u64) ((1LL << exp) + (fr >> (52 - exp))); +} + +#endif //T2AUDIO_DESCRIPTION_H diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_audio/pcm.c b/drivers/staging/t2bce/t2bce_audio/pcm.c --- a/drivers/staging/t2bce/t2bce_audio/pcm.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_audio/pcm.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,520 @@ +#include "pcm.h" +#include "audio.h" +#include +#include +#include + +static u64 t2audio_get_alsa_fmtbit(struct t2audio_apple_description *desc) +{ + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_FLOAT) { + if (desc->bits_per_channel == 32) { + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_BIG_ENDIAN) + return SNDRV_PCM_FMTBIT_FLOAT_BE; + else + return SNDRV_PCM_FMTBIT_FLOAT_LE; + } else if (desc->bits_per_channel == 64) { + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_BIG_ENDIAN) + return SNDRV_PCM_FMTBIT_FLOAT64_BE; + else + return SNDRV_PCM_FMTBIT_FLOAT64_LE; + } else { + pr_err("t2bce_audio: unsupported bits per channel for float format: %u\n", desc->bits_per_channel); + return 0; + } + } +#define DEFINE_BPC_OPTION(val, b) \ + case val: \ + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_BIG_ENDIAN) { \ + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_SIGNED) \ + return SNDRV_PCM_FMTBIT_S ## b ## BE; \ + else \ + return SNDRV_PCM_FMTBIT_U ## b ## BE; \ + } else { \ + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_SIGNED) \ + return SNDRV_PCM_FMTBIT_S ## b ## LE; \ + else \ + return SNDRV_PCM_FMTBIT_U ## b ## LE; \ + } + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_PACKED) { + switch (desc->bits_per_channel) { + case 8: + case 16: + case 32: + break; + DEFINE_BPC_OPTION(24, 24_3) + default: + pr_err("t2bce_audio: unsupported bits per channel for packed format: %u\n", desc->bits_per_channel); + return 0; + } + } + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_ALIGNED_HIGH) { + switch (desc->bits_per_channel) { + DEFINE_BPC_OPTION(24, 32_) + default: + pr_err("t2bce_audio: unsupported bits per channel for high-aligned format: %u\n", desc->bits_per_channel); + return 0; + } + } + switch (desc->bits_per_channel) { + case 8: + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_SIGNED) + return SNDRV_PCM_FMTBIT_S8; + else + return SNDRV_PCM_FMTBIT_U8; + DEFINE_BPC_OPTION(16, 16_) + DEFINE_BPC_OPTION(24, 24_) + DEFINE_BPC_OPTION(32, 32_) + default: + pr_err("t2bce_audio: unsupported bits per channel: %u\n", desc->bits_per_channel); + return 0; + } +} +int t2audio_create_hw_info(struct t2audio_apple_description *desc, struct snd_pcm_hardware *alsa_hw, + size_t buf_size) +{ + uint rate; + alsa_hw->info = (SNDRV_PCM_INFO_MMAP | + SNDRV_PCM_INFO_BLOCK_TRANSFER | + SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_NO_PERIOD_WAKEUP | + SNDRV_PCM_INFO_DOUBLE); + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_NON_MIXABLE) + pr_warn("t2bce_audio: unsupported hw flag: NON_MIXABLE\n"); + if (!(desc->format_flags & T2AUDIO_FORMAT_FLAG_NON_INTERLEAVED)) + alsa_hw->info |= SNDRV_PCM_INFO_INTERLEAVED; + alsa_hw->formats = t2audio_get_alsa_fmtbit(desc); + if (!alsa_hw->formats) + return -EINVAL; + rate = (uint) t2audio_double_to_u64(desc->sample_rate_double); + alsa_hw->rates = snd_pcm_rate_to_rate_bit(rate); + alsa_hw->rate_min = rate; + alsa_hw->rate_max = rate; + alsa_hw->channels_min = desc->channels_per_frame; + alsa_hw->channels_max = desc->channels_per_frame; + alsa_hw->buffer_bytes_max = buf_size; + alsa_hw->period_bytes_min = desc->bytes_per_packet; + alsa_hw->period_bytes_max = desc->bytes_per_packet; + alsa_hw->periods_min = (uint) (buf_size / desc->bytes_per_packet); + alsa_hw->periods_max = (uint) (buf_size / desc->bytes_per_packet); + pr_debug("t2audio_create_hw_info: format = %llu, rate = %u/%u. channels = %u, periods = %u, period size = %lu\n", + alsa_hw->formats, alsa_hw->rate_min, alsa_hw->rates, alsa_hw->channels_min, alsa_hw->periods_min, + alsa_hw->period_bytes_min); + return 0; +} + +static struct t2audio_stream *t2audio_pcm_stream(struct snd_pcm_substream *substream) +{ + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) + return &sdev->out_streams[substream->number]; + else + return &sdev->in_streams[substream->number]; +} + +static struct t2audio_dma_buf *t2audio_pcm_dma_buf(struct t2audio_stream *stream) +{ + if (!stream->buffer_cnt || !stream->buffers) + return NULL; + + return &stream->buffers[0]; +} + +static void t2audio_dma_memset(struct t2audio_dma_buf *buf, size_t offset, int value, size_t size) +{ + if (!buf || offset >= buf->size) + return; + + size = min(size, buf->size - offset); + switch (buf->type) { + case T2AUDIO_DMA_BUF_IOMEM: + memset_io((u8 __iomem *) buf->ptr + offset, value, size); + break; + case T2AUDIO_DMA_BUF_COHERENT: + memset((u8 *) buf->ptr + offset, value, size); + break; + } +} + +static void t2audio_dma_copy_from(struct t2audio_dma_buf *buf, void *dst, size_t offset, size_t size) +{ + if (!buf || offset >= buf->size) + return; + + size = min(size, buf->size - offset); + switch (buf->type) { + case T2AUDIO_DMA_BUF_IOMEM: + memcpy_fromio(dst, (u8 __iomem *) buf->ptr + offset, size); + break; + case T2AUDIO_DMA_BUF_COHERENT: + memcpy(dst, (u8 *) buf->ptr + offset, size); + break; + } +} + +static void t2audio_dma_copy_to(struct t2audio_dma_buf *buf, size_t offset, const void *src, size_t size) +{ + if (!buf || offset >= buf->size) + return; + + size = min(size, buf->size - offset); + switch (buf->type) { + case T2AUDIO_DMA_BUF_IOMEM: + memcpy_toio((u8 __iomem *) buf->ptr + offset, src, size); + break; + case T2AUDIO_DMA_BUF_COHERENT: + memcpy((u8 *) buf->ptr + offset, src, size); + break; + } +} + +static void t2audio_pcm_zero_frames(struct snd_pcm_substream *substream, + snd_pcm_uframes_t first, snd_pcm_uframes_t frames) +{ + struct t2audio_stream *stream = t2audio_pcm_stream(substream); + struct t2audio_dma_buf *buf = t2audio_pcm_dma_buf(stream); + struct snd_pcm_runtime *runtime = substream->runtime; + size_t offset; + size_t size; + + if (!buf || !runtime || !runtime->buffer_size || !frames) + return; + + first %= runtime->buffer_size; + if (frames > runtime->buffer_size) + frames = runtime->buffer_size; + + offset = frames_to_bytes(runtime, first); + if (first + frames <= runtime->buffer_size) { + size = frames_to_bytes(runtime, frames); + t2audio_dma_memset(buf, offset, 0, size); + return; + } + + size = frames_to_bytes(runtime, runtime->buffer_size - first); + t2audio_dma_memset(buf, offset, 0, size); + t2audio_dma_memset(buf, 0, 0, frames_to_bytes(runtime, frames - (runtime->buffer_size - first))); +} + +static int t2audio_pcm_open(struct snd_pcm_substream *substream) +{ + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + + pr_debug("t2bce_audio: pcm open dev=%s direction=%s substream=%u\n", + sdev->uid, + substream->stream == SNDRV_PCM_STREAM_PLAYBACK ? "playback" : "capture", + substream->number); + substream->runtime->hw = *t2audio_pcm_stream(substream)->alsa_hw_desc; + + return 0; +} + +static int t2audio_pcm_close(struct snd_pcm_substream *substream) +{ + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + + pr_debug("t2bce_audio: pcm close dev=%s direction=%s substream=%u\n", + sdev->uid, + substream->stream == SNDRV_PCM_STREAM_PLAYBACK ? "playback" : "capture", + substream->number); + return 0; +} + +static int t2audio_pcm_prepare(struct snd_pcm_substream *substream) +{ + struct t2audio_stream *stream = t2audio_pcm_stream(substream); + + stream->waiting_for_first_ts = true; + stream->remote_timestamp = 0; + stream->timestamp_accept_after = 0; + stream->clock_offset_ns = 0; + stream->timestamp_seed = 0; + stream->clock_offset_valid = false; + stream->frame_min = stream->latency; + + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK && substream->runtime->buffer_size) + t2audio_pcm_zero_frames(substream, 0, substream->runtime->buffer_size); + + return 0; +} + +static int t2audio_pcm_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *hw_params) +{ + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + struct t2audio_stream *astream = t2audio_pcm_stream(substream); + + pr_debug("t2bce_audio: pcm hw_params dev=%s direction=%s substream=%u\n", + sdev->uid, + substream->stream == SNDRV_PCM_STREAM_PLAYBACK ? "playback" : "capture", + substream->number); + + if (!astream->buffer_cnt || !astream->buffers) + return -EINVAL; + + substream->runtime->dma_area = astream->buffers[0].ptr; + substream->runtime->dma_addr = astream->buffers[0].dma_addr; + substream->runtime->dma_bytes = astream->buffers[0].size; + return 0; +} + +static int t2audio_pcm_hw_free(struct snd_pcm_substream *substream) +{ + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + + pr_debug("t2bce_audio: pcm hw_free dev=%s direction=%s substream=%u\n", + sdev->uid, + substream->stream == SNDRV_PCM_STREAM_PLAYBACK ? "playback" : "capture", + substream->number); + return 0; +} + +static int t2audio_pcm_start(struct snd_pcm_substream *substream) +{ + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + struct t2audio_stream *stream = t2audio_pcm_stream(substream); + struct t2audio_dma_buf *dmabuf = t2audio_pcm_dma_buf(stream); + void *buf = NULL; + size_t s = 0; + ktime_t time_start, time_end; + bool back_buffer; + int status; + + time_start = ktime_get(); + smp_store_release(&stream->started, 0); + + back_buffer = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK); + + if (back_buffer) { + snd_pcm_uframes_t appl_ptr; + + if (!dmabuf) + return -EINVAL; + if (!substream->runtime->buffer_size) + return -EINVAL; + + appl_ptr = substream->runtime->control->appl_ptr % substream->runtime->buffer_size; + s = frames_to_bytes(substream->runtime, appl_ptr); + if (s) { + buf = kmalloc(s, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + t2audio_dma_copy_from(dmabuf, buf, 0, s); + time_end = ktime_get(); + pr_debug("t2bce_audio: Backed up the buffer in %lluns [%li]\n", ktime_to_ns(time_end - time_start), + appl_ptr); + } + } + + status = t2audio_cmd_start_io(sdev->a, sdev->dev_id); + if (back_buffer && buf) + t2audio_dma_copy_to(dmabuf, 0, buf, s); + kfree(buf); + + if (status) + return status; + + /* Timestamps received while START_IO was pending belong to the old epoch. */ + stream->remote_timestamp = 0; + stream->waiting_for_first_ts = true; + stream->timestamp_accept_after = ktime_get_boottime(); + stream->clock_offset_ns = 0; + stream->timestamp_seed = 0; + stream->clock_offset_valid = false; + stream->frame_min = stream->latency; + smp_store_release(&stream->started, 1); + + time_end = ktime_get(); + pr_debug("t2bce_audio: start_io %s %lld us\n", + sdev->uid, ktime_to_us(ktime_sub(time_end, time_start))); + return 0; +} + +static int t2audio_pcm_trigger(struct snd_pcm_substream *substream, int cmd) +{ + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + struct t2audio_stream *stream = t2audio_pcm_stream(substream); + int err; + + /* bridgeOS exposes one ALSA substream per remote stream. */ + if (substream->number != 0) + return 0; + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + pr_debug("t2bce_audio: TRIGGER START %s\n", sdev->uid); + err = t2audio_pcm_start(substream); + if (err) + return err; + break; + case SNDRV_PCM_TRIGGER_STOP: + pr_debug("t2bce_audio: TRIGGER STOP %s\n", sdev->uid); + smp_store_release(&stream->started, 0); + err = t2audio_cmd_stop_io(sdev->a, sdev->dev_id); + stream->remote_timestamp = 0; + stream->waiting_for_first_ts = true; + stream->timestamp_accept_after = 0; + stream->clock_offset_ns = 0; + stream->timestamp_seed = 0; + stream->clock_offset_valid = false; + if (err) + return err; + break; + default: + return -EINVAL; + } + return 0; +} + +static snd_pcm_uframes_t t2audio_pcm_pointer(struct snd_pcm_substream *substream) +{ + struct t2audio_stream *stream = t2audio_pcm_stream(substream); + ktime_t time_from_start; + snd_pcm_sframes_t frames; + snd_pcm_sframes_t buffer_time_length; + + if (!smp_load_acquire(&stream->started) || stream->waiting_for_first_ts) + return 0; + + /* bridgeOS reports coarse timestamps; interpolate the ALSA pointer locally. */ + time_from_start = ktime_get_boottime() - stream->remote_timestamp; + if (ktime_to_ns(time_from_start) < 0) + return 0; + buffer_time_length = NSEC_PER_SEC * substream->runtime->buffer_size / substream->runtime->rate; + frames = (ktime_to_ns(time_from_start) % buffer_time_length) * substream->runtime->buffer_size / buffer_time_length; + if (ktime_to_ns(time_from_start) < buffer_time_length) { + if (frames < stream->frame_min) + frames = stream->frame_min; + else + stream->frame_min = 0; + } else { + if (ktime_to_ns(time_from_start) < 2 * buffer_time_length) + stream->frame_min = frames; + else + stream->frame_min = 0; + } + frames -= stream->latency; + if (frames < 0) + frames += ((-frames - 1) / substream->runtime->buffer_size + 1) * substream->runtime->buffer_size; + frames %= substream->runtime->buffer_size; + return (snd_pcm_uframes_t) frames; +} + +static int t2audio_pcm_mmap(struct snd_pcm_substream *substream, struct vm_area_struct *area) +{ + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + struct t2audio_stream *stream = t2audio_pcm_stream(substream); + struct t2audio_dma_buf *buf; + + if (!stream->buffer_cnt || !stream->buffers) + return -EINVAL; + + buf = &stream->buffers[0]; + switch (buf->type) { + case T2AUDIO_DMA_BUF_IOMEM: + return snd_pcm_lib_mmap_iomem(substream, area); + case T2AUDIO_DMA_BUF_COHERENT: + return dma_mmap_coherent(sdev->a->dev, area, buf->ptr, buf->dma_addr, buf->size); + default: + return -EINVAL; + } +} + +static struct snd_pcm_ops t2audio_pcm_ops = { + .open = t2audio_pcm_open, + .close = t2audio_pcm_close, + .ioctl = snd_pcm_lib_ioctl, + .hw_params = t2audio_pcm_hw_params, + .hw_free = t2audio_pcm_hw_free, + .prepare = t2audio_pcm_prepare, + .trigger = t2audio_pcm_trigger, + .pointer = t2audio_pcm_pointer, + .mmap = t2audio_pcm_mmap +}; + +int t2audio_create_pcm(struct t2audio_subdevice *sdev) +{ + struct snd_pcm *pcm; + struct t2audio_alsa_pcm_id_mapping *id_mapping; + int err; + + if (!sdev->is_pcm || (sdev->in_stream_cnt == 0 && sdev->out_stream_cnt == 0)) { + return -EINVAL; + } + + for (id_mapping = t2audio_alsa_id_mappings; id_mapping->name; id_mapping++) { + if (!strcmp(sdev->uid, id_mapping->name)) { + sdev->alsa_id = id_mapping->alsa_id; + break; + } + } + if (!id_mapping->name) + sdev->alsa_id = sdev->a->next_alsa_id++; + err = snd_pcm_new(sdev->a->card, sdev->uid, sdev->alsa_id, + (int) sdev->out_stream_cnt, (int) sdev->in_stream_cnt, &pcm); + if (err < 0) + return err; + pcm->private_data = sdev; + pcm->nonatomic = 1; + sdev->pcm = pcm; + strcpy(pcm->name, sdev->uid); + snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &t2audio_pcm_ops); + snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_CAPTURE, &t2audio_pcm_ops); + return 0; +} + +static void t2audio_handle_stream_timestamp(struct snd_pcm_substream *substream, + u64 dev_timestamp, u64 update_seed, + s64 clock_offset_ns, bool clock_ready) +{ + unsigned long flags; + ktime_t event_timestamp; + struct t2audio_stream *stream; + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + + stream = t2audio_pcm_stream(substream); + snd_pcm_stream_lock_irqsave(substream, flags); + if (!smp_load_acquire(&stream->started)) { + snd_pcm_stream_unlock_irqrestore(substream, flags); + return; + } + if (!clock_ready) { + snd_pcm_stream_unlock_irqrestore(substream, flags); + return; + } + if (!stream->clock_offset_valid || stream->timestamp_seed != update_seed) { + stream->clock_offset_ns = clock_offset_ns; + stream->timestamp_seed = update_seed; + stream->clock_offset_valid = true; + } + event_timestamp = ns_to_ktime((s64)dev_timestamp + stream->clock_offset_ns); + if (ktime_before(event_timestamp, stream->timestamp_accept_after)) { + pr_debug("t2bce_audio: ignoring pre-start timestamp dev=%s event=%lld accept_after=%lld\n", + sdev->uid, ktime_to_ns(event_timestamp), + ktime_to_ns(stream->timestamp_accept_after)); + snd_pcm_stream_unlock_irqrestore(substream, flags); + return; + } + stream->remote_timestamp = event_timestamp; + if (stream->waiting_for_first_ts) { + stream->waiting_for_first_ts = false; + snd_pcm_stream_unlock_irqrestore(substream, flags); + return; + } + snd_pcm_stream_unlock_irqrestore(substream, flags); + if (!substream->runtime->no_period_wakeup) + snd_pcm_period_elapsed(substream); +} + +void t2audio_handle_timestamp(struct t2audio_subdevice *sdev, u64 dev_timestamp, + u64 update_seed, s64 clock_offset_ns, bool clock_ready) +{ + struct snd_pcm_substream *substream; + + substream = sdev->pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream; + if (substream) + t2audio_handle_stream_timestamp(substream, dev_timestamp, update_seed, + clock_offset_ns, clock_ready); + substream = sdev->pcm->streams[SNDRV_PCM_STREAM_CAPTURE].substream; + if (substream) + t2audio_handle_stream_timestamp(substream, dev_timestamp, update_seed, + clock_offset_ns, clock_ready); +} diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_audio/pcm.h b/drivers/staging/t2bce/t2bce_audio/pcm.h --- a/drivers/staging/t2bce/t2bce_audio/pcm.h 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_audio/pcm.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,17 @@ +#ifndef T2AUDIO_PCM_H +#define T2AUDIO_PCM_H + +#include +#include + +struct t2audio_subdevice; +struct t2audio_apple_description; +struct snd_pcm_hardware; + +int t2audio_create_hw_info(struct t2audio_apple_description *desc, struct snd_pcm_hardware *alsa_hw, size_t buf_size); +int t2audio_create_pcm(struct t2audio_subdevice *sdev); + +void t2audio_handle_timestamp(struct t2audio_subdevice *sdev, u64 dev_timestamp, + u64 update_seed, s64 clock_offset_ns, bool clock_ready); + +#endif //T2AUDIO_PCM_H diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_audio/protocol.c b/drivers/staging/t2bce/t2bce_audio/protocol.c --- a/drivers/staging/t2bce/t2bce_audio/protocol.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_audio/protocol.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,347 @@ +#include "protocol.h" +#include "protocol_bce.h" +#include "audio.h" + +int t2audio_msg_read_base(struct t2audio_msg *msg, struct t2audio_msg_base *base) +{ + if (msg->size < sizeof(struct t2audio_msg_header) + sizeof(struct t2audio_msg_base) * 2) + return -EINVAL; + *base = *((struct t2audio_msg_base *) ((struct t2audio_msg_header *) msg->data + 1)); + return 0; +} + +#define READ_START(type) \ + size_t offset = sizeof(struct t2audio_msg_header) + sizeof(struct t2audio_msg_base); (void)offset; \ + if (((struct t2audio_msg_base *) ((struct t2audio_msg_header *) msg->data + 1))->msg != type) \ + return -EINVAL; +#define READ_DEVID_VAR(devid) *devid = ((struct t2audio_msg_header *) msg->data)->device_id +#define READ_VAL(type) ({ offset += sizeof(type); *((type *) ((u8 *) msg->data + offset - sizeof(type))); }) +#define READ_VAR(type, var) *var = READ_VAL(type) + +int t2audio_msg_read_start_io_response(struct t2audio_msg *msg) +{ + READ_START(T2AUDIO_MSG_START_IO_RESPONSE); + return 0; +} + +int t2audio_msg_read_stop_io_response(struct t2audio_msg *msg) +{ + READ_START(T2AUDIO_MSG_STOP_IO_RESPONSE); + return 0; +} + +int t2audio_msg_read_update_timestamp(struct t2audio_msg *msg, t2audio_device_id_t *devid, + u64 *timestamp, u64 *update_seed) +{ + READ_START(T2AUDIO_MSG_UPDATE_TIMESTAMP); + READ_DEVID_VAR(devid); + READ_VAR(u64, timestamp); + READ_VAR(u64, update_seed); + return 0; +} + +int t2audio_msg_read_get_property_response(struct t2audio_msg *msg, t2audio_object_id_t *obj, + struct t2audio_prop_addr *prop, void **data, u64 *data_size) +{ + READ_START(T2AUDIO_MSG_GET_PROPERTY_RESPONSE); + READ_VAR(t2audio_object_id_t, obj); + READ_VAR(u32, &prop->element); + READ_VAR(u32, &prop->scope); + READ_VAR(u32, &prop->selector); + READ_VAR(u64, data_size); + *data = ((u8 *) msg->data + offset); + /* offset += data_size; */ + return 0; +} + +int t2audio_msg_read_set_property_response(struct t2audio_msg *msg, t2audio_object_id_t *obj) +{ + READ_START(T2AUDIO_MSG_SET_PROPERTY_RESPONSE); + READ_VAR(t2audio_object_id_t, obj); + return 0; +} + +int t2audio_msg_read_property_listener_response(struct t2audio_msg *msg, t2audio_object_id_t *obj, + struct t2audio_prop_addr *prop) +{ + READ_START(T2AUDIO_MSG_PROPERTY_LISTENER_RESPONSE); + READ_VAR(t2audio_object_id_t, obj); + READ_VAR(u32, &prop->element); + READ_VAR(u32, &prop->scope); + READ_VAR(u32, &prop->selector); + return 0; +} + +int t2audio_msg_read_property_changed(struct t2audio_msg *msg, t2audio_device_id_t *devid, t2audio_object_id_t *obj, + struct t2audio_prop_addr *prop) +{ + READ_START(T2AUDIO_MSG_PROPERTY_CHANGED); + READ_DEVID_VAR(devid); + READ_VAR(t2audio_object_id_t, obj); + READ_VAR(u32, &prop->element); + READ_VAR(u32, &prop->scope); + READ_VAR(u32, &prop->selector); + return 0; +} + +int t2audio_msg_read_set_input_stream_address_ranges_response(struct t2audio_msg *msg) +{ + READ_START(T2AUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES_RESPONSE); + return 0; +} + +int t2audio_msg_read_get_input_stream_list_response(struct t2audio_msg *msg, t2audio_object_id_t **str_l, u64 *str_cnt) +{ + READ_START(T2AUDIO_MSG_GET_INPUT_STREAM_LIST_RESPONSE); + READ_VAR(u64, str_cnt); + *str_l = (t2audio_device_id_t *) ((u8 *) msg->data + offset); + /* offset += str_cnt * sizeof(t2audio_object_id_t); */ + return 0; +} + +int t2audio_msg_read_get_output_stream_list_response(struct t2audio_msg *msg, t2audio_object_id_t **str_l, u64 *str_cnt) +{ + READ_START(T2AUDIO_MSG_GET_OUTPUT_STREAM_LIST_RESPONSE); + READ_VAR(u64, str_cnt); + *str_l = (t2audio_device_id_t *) ((u8 *) msg->data + offset); + /* offset += str_cnt * sizeof(t2audio_object_id_t); */ + return 0; +} + +int t2audio_msg_read_set_remote_access_response(struct t2audio_msg *msg) +{ + READ_START(T2AUDIO_MSG_SET_REMOTE_ACCESS_RESPONSE); + return 0; +} + +int t2audio_msg_read_get_device_list_response(struct t2audio_msg *msg, t2audio_device_id_t **dev_l, u64 *dev_cnt) +{ + READ_START(T2AUDIO_MSG_GET_DEVICE_LIST_RESPONSE); + READ_VAR(u64, dev_cnt); + *dev_l = (t2audio_device_id_t *) ((u8 *) msg->data + offset); + /* offset += dev_cnt * sizeof(t2audio_device_id_t); */ + return 0; +} + +#define WRITE_START_OF_TYPE(typev, devid) \ + size_t offset = sizeof(struct t2audio_msg_header); (void) offset; \ + ((struct t2audio_msg_header *) msg->data)->type = (typev); \ + ((struct t2audio_msg_header *) msg->data)->device_id = (devid); +#define WRITE_START_COMMAND(devid) WRITE_START_OF_TYPE(T2AUDIO_MSG_TYPE_COMMAND, devid) +#define WRITE_START_RESPONSE() WRITE_START_OF_TYPE(T2AUDIO_MSG_TYPE_RESPONSE, 0) +#define WRITE_START_NOTIFICATION() WRITE_START_OF_TYPE(T2AUDIO_MSG_TYPE_NOTIFICATION, 0) +#define WRITE_VAL(type, value) { *((type *) ((u8 *) msg->data + offset)) = value; offset += sizeof(value); } +#define WRITE_BIN(value, size) { memcpy((u8 *) msg->data + offset, value, size); offset += size; } +#define WRITE_BASE(type) WRITE_VAL(u32, type) WRITE_VAL(u32, 0) +#define WRITE_END() { msg->size = offset; } + +void t2audio_msg_write_start_io(struct t2audio_msg *msg, t2audio_device_id_t dev) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(T2AUDIO_MSG_START_IO); + WRITE_END(); +} + +void t2audio_msg_write_stop_io(struct t2audio_msg *msg, t2audio_device_id_t dev) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(T2AUDIO_MSG_STOP_IO); + WRITE_END(); +} + +void t2audio_msg_write_get_property(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(T2AUDIO_MSG_GET_PROPERTY); + WRITE_VAL(t2audio_object_id_t, obj); + WRITE_VAL(u32, prop.element); + WRITE_VAL(u32, prop.scope); + WRITE_VAL(u32, prop.selector); + WRITE_VAL(u64, qualifier_size); + WRITE_BIN(qualifier, qualifier_size); + WRITE_END(); +} + +void t2audio_msg_write_set_property(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *data, u64 data_size, void *qualifier, u64 qualifier_size) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(T2AUDIO_MSG_SET_PROPERTY); + WRITE_VAL(t2audio_object_id_t, obj); + WRITE_VAL(u32, prop.element); + WRITE_VAL(u32, prop.scope); + WRITE_VAL(u32, prop.selector); + WRITE_VAL(u64, data_size); + WRITE_BIN(data, data_size); + WRITE_VAL(u64, qualifier_size); + WRITE_BIN(qualifier, qualifier_size); + WRITE_END(); +} + +void t2audio_msg_write_property_listener(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj, + struct t2audio_prop_addr prop) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(T2AUDIO_MSG_PROPERTY_LISTENER); + WRITE_VAL(t2audio_object_id_t, obj); + WRITE_VAL(u32, prop.element); + WRITE_VAL(u32, prop.scope); + WRITE_VAL(u32, prop.selector); + WRITE_END(); +} + +void t2audio_msg_write_set_input_stream_address_ranges(struct t2audio_msg *msg, t2audio_device_id_t devid) +{ + WRITE_START_COMMAND(devid); + WRITE_BASE(T2AUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES); + WRITE_END(); +} + +void t2audio_msg_write_get_input_stream_list(struct t2audio_msg *msg, t2audio_device_id_t devid) +{ + WRITE_START_COMMAND(devid); + WRITE_BASE(T2AUDIO_MSG_GET_INPUT_STREAM_LIST); + WRITE_END(); +} + +void t2audio_msg_write_get_output_stream_list(struct t2audio_msg *msg, t2audio_device_id_t devid) +{ + WRITE_START_COMMAND(devid); + WRITE_BASE(T2AUDIO_MSG_GET_OUTPUT_STREAM_LIST); + WRITE_END(); +} + +void t2audio_msg_write_set_remote_access(struct t2audio_msg *msg, u64 mode) +{ + WRITE_START_COMMAND(0); + WRITE_BASE(T2AUDIO_MSG_SET_REMOTE_ACCESS); + WRITE_VAL(u64, mode); + WRITE_END(); +} + +void t2audio_msg_write_alive_notification(struct t2audio_msg *msg, u32 proto_ver, u32 msg_ver) +{ + WRITE_START_NOTIFICATION(); + WRITE_BASE(T2AUDIO_MSG_NOTIFICATION_ALIVE); + WRITE_VAL(u32, proto_ver); + WRITE_VAL(u32, msg_ver); + WRITE_END(); +} + +void t2audio_msg_write_update_timestamp_response(struct t2audio_msg *msg) +{ + WRITE_START_RESPONSE(); + WRITE_BASE(T2AUDIO_MSG_UPDATE_TIMESTAMP_RESPONSE); + WRITE_END(); +} + +void t2audio_msg_write_get_device_list(struct t2audio_msg *msg) +{ + WRITE_START_COMMAND(0); + WRITE_BASE(T2AUDIO_MSG_GET_DEVICE_LIST); + WRITE_END(); +} + +#define CMD_SHARED_VARS_NO_REPLY \ + int status = 0; \ + struct t2audio_send_ctx sctx; +#define CMD_SHARED_VARS \ + CMD_SHARED_VARS_NO_REPLY \ + struct t2audio_msg reply = t2audio_reply_alloc(); \ + struct t2audio_msg *buf = &reply; +#define CMD_SEND_REQUEST(fn, ...) \ + if ((status = t2audio_send_cmd_sync(a, &sctx, buf, 500, fn, ##__VA_ARGS__))) \ + return status; +#define CMD_DEF_SHARED_AND_SEND(fn, ...) \ + CMD_SHARED_VARS \ + CMD_SEND_REQUEST(fn, ##__VA_ARGS__); +#define CMD_DEF_SHARED_NO_REPLY_AND_SEND(fn, ...) \ + CMD_SHARED_VARS_NO_REPLY \ + CMD_SEND_REQUEST(fn, ##__VA_ARGS__); +#define CMD_HNDL_REPLY_NO_FREE(fn, ...) \ + status = fn(buf, ##__VA_ARGS__); \ + return status; +#define CMD_HNDL_REPLY_AND_FREE(fn, ...) \ + status = fn(buf, ##__VA_ARGS__); \ + t2audio_reply_free(&reply); \ + return status; + +int t2audio_cmd_start_io(struct t2audio_device *a, t2audio_device_id_t devid) +{ + CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_start_io, devid); + CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_start_io_response); +} +int t2audio_cmd_stop_io(struct t2audio_device *a, t2audio_device_id_t devid) +{ + CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_stop_io, devid); + CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_stop_io_response); +} +int t2audio_cmd_get_property(struct t2audio_device *a, struct t2audio_msg *buf, + t2audio_device_id_t devid, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void **data, u64 *data_size) +{ + CMD_DEF_SHARED_NO_REPLY_AND_SEND(t2audio_msg_write_get_property, devid, obj, prop, qualifier, qualifier_size); + CMD_HNDL_REPLY_NO_FREE(t2audio_msg_read_get_property_response, &obj, &prop, data, data_size); +} +int t2audio_cmd_get_primitive_property(struct t2audio_device *a, + t2audio_device_id_t devid, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size) +{ + int status; + struct t2audio_msg reply = t2audio_reply_alloc(); + void *r_data; + u64 r_data_size; + if ((status = t2audio_cmd_get_property(a, &reply, devid, obj, prop, qualifier, qualifier_size, + &r_data, &r_data_size))) + goto finish; + if (r_data_size != data_size) { + status = -EINVAL; + goto finish; + } + memcpy(data, r_data, data_size); +finish: + t2audio_reply_free(&reply); + return status; +} +int t2audio_cmd_set_property(struct t2audio_device *a, t2audio_device_id_t devid, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size) +{ + CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_set_property, devid, obj, prop, data, data_size, + qualifier, qualifier_size); + CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_set_property_response, &obj); +} +int t2audio_cmd_property_listener(struct t2audio_device *a, t2audio_device_id_t devid, t2audio_object_id_t obj, + struct t2audio_prop_addr prop) +{ + CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_property_listener, devid, obj, prop); + CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_property_listener_response, &obj, &prop); +} +int t2audio_cmd_set_input_stream_address_ranges(struct t2audio_device *a, t2audio_device_id_t devid) +{ + CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_set_input_stream_address_ranges, devid); + CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_set_input_stream_address_ranges_response); +} +int t2audio_cmd_get_input_stream_list(struct t2audio_device *a, struct t2audio_msg *buf, t2audio_device_id_t devid, + t2audio_object_id_t **str_l, u64 *str_cnt) +{ + CMD_DEF_SHARED_NO_REPLY_AND_SEND(t2audio_msg_write_get_input_stream_list, devid); + CMD_HNDL_REPLY_NO_FREE(t2audio_msg_read_get_input_stream_list_response, str_l, str_cnt); +} +int t2audio_cmd_get_output_stream_list(struct t2audio_device *a, struct t2audio_msg *buf, t2audio_device_id_t devid, + t2audio_object_id_t **str_l, u64 *str_cnt) +{ + CMD_DEF_SHARED_NO_REPLY_AND_SEND(t2audio_msg_write_get_output_stream_list, devid); + CMD_HNDL_REPLY_NO_FREE(t2audio_msg_read_get_output_stream_list_response, str_l, str_cnt); +} +int t2audio_cmd_set_remote_access(struct t2audio_device *a, u64 mode) +{ + CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_set_remote_access, mode); + CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_set_remote_access_response); +} +int t2audio_cmd_get_device_list(struct t2audio_device *a, struct t2audio_msg *buf, + t2audio_device_id_t **dev_l, u64 *dev_cnt) +{ + CMD_DEF_SHARED_NO_REPLY_AND_SEND(t2audio_msg_write_get_device_list); + CMD_HNDL_REPLY_NO_FREE(t2audio_msg_read_get_device_list_response, dev_l, dev_cnt); +} \ No newline at end of file diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_audio/protocol.h b/drivers/staging/t2bce/t2bce_audio/protocol.h --- a/drivers/staging/t2bce/t2bce_audio/protocol.h 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_audio/protocol.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,147 @@ +#ifndef T2AUDIO_PROTOCOL_H +#define T2AUDIO_PROTOCOL_H + +#include + +struct t2audio_device; + +typedef u64 t2audio_device_id_t; +typedef u64 t2audio_object_id_t; + +struct t2audio_msg { + void *data; + size_t size; +}; + +struct __attribute__((packed)) t2audio_msg_header { + char tag[4]; + u8 type; + t2audio_device_id_t device_id; +}; +struct __attribute__((packed)) t2audio_msg_base { + u32 msg; + u32 status; +}; + +struct t2audio_prop_addr { + u32 scope; + u32 selector; + u32 element; +}; +#define T2AUDIO_PROP(scope, sel, el) (struct t2audio_prop_addr) { scope, sel, el } + +enum { + T2AUDIO_MSG_TYPE_COMMAND = 1, + T2AUDIO_MSG_TYPE_RESPONSE = 2, + T2AUDIO_MSG_TYPE_NOTIFICATION = 3 +}; + +enum { + T2AUDIO_MSG_START_IO = 0, + T2AUDIO_MSG_START_IO_RESPONSE = 1, + T2AUDIO_MSG_STOP_IO = 2, + T2AUDIO_MSG_STOP_IO_RESPONSE = 3, + T2AUDIO_MSG_UPDATE_TIMESTAMP = 4, + T2AUDIO_MSG_GET_PROPERTY = 7, + T2AUDIO_MSG_GET_PROPERTY_RESPONSE = 8, + T2AUDIO_MSG_SET_PROPERTY = 9, + T2AUDIO_MSG_SET_PROPERTY_RESPONSE = 10, + T2AUDIO_MSG_PROPERTY_LISTENER = 11, + T2AUDIO_MSG_PROPERTY_LISTENER_RESPONSE = 12, + T2AUDIO_MSG_PROPERTY_CHANGED = 13, + T2AUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES = 18, + T2AUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES_RESPONSE = 19, + T2AUDIO_MSG_GET_INPUT_STREAM_LIST = 24, + T2AUDIO_MSG_GET_INPUT_STREAM_LIST_RESPONSE = 25, + T2AUDIO_MSG_GET_OUTPUT_STREAM_LIST = 26, + T2AUDIO_MSG_GET_OUTPUT_STREAM_LIST_RESPONSE = 27, + T2AUDIO_MSG_SET_REMOTE_ACCESS = 32, + T2AUDIO_MSG_SET_REMOTE_ACCESS_RESPONSE = 33, + T2AUDIO_MSG_UPDATE_TIMESTAMP_RESPONSE = 34, + + T2AUDIO_MSG_NOTIFICATION_ALIVE = 100, + T2AUDIO_MSG_GET_DEVICE_LIST = 101, + T2AUDIO_MSG_GET_DEVICE_LIST_RESPONSE = 102, + T2AUDIO_MSG_NOTIFICATION_BOOT = 104 +}; + +enum { + T2AUDIO_REMOTE_ACCESS_OFF = 0, + T2AUDIO_REMOTE_ACCESS_ON = 2 +}; + +enum { + T2AUDIO_PROP_SCOPE_GLOBAL = 0x676c6f62, // 'glob' + T2AUDIO_PROP_SCOPE_INPUT = 0x696e7074, // 'inpt' + T2AUDIO_PROP_SCOPE_OUTPUT = 0x6f757470 // 'outp' +}; + +enum { + T2AUDIO_PROP_UID = 0x75696420, // 'uid ' + T2AUDIO_PROP_BOOL_VALUE = 0x6263766c, // 'bcvl' + T2AUDIO_PROP_JACK_PLUGGED = 0x6a61636b, // 'jack' + T2AUDIO_PROP_SEL_VOLUME = 0x64656176, // 'deav' + T2AUDIO_PROP_LATENCY = 0x6c746e63, // 'ltnc' + T2AUDIO_PROP_PHYS_FORMAT = 0x70667420 // 'pft ' +}; + +int t2audio_msg_read_base(struct t2audio_msg *msg, struct t2audio_msg_base *base); + +int t2audio_msg_read_start_io_response(struct t2audio_msg *msg); +int t2audio_msg_read_stop_io_response(struct t2audio_msg *msg); +int t2audio_msg_read_update_timestamp(struct t2audio_msg *msg, t2audio_device_id_t *devid, + u64 *timestamp, u64 *update_seed); +int t2audio_msg_read_get_property_response(struct t2audio_msg *msg, t2audio_object_id_t *obj, + struct t2audio_prop_addr *prop, void **data, u64 *data_size); +int t2audio_msg_read_set_property_response(struct t2audio_msg *msg, t2audio_object_id_t *obj); +int t2audio_msg_read_property_listener_response(struct t2audio_msg *msg,t2audio_object_id_t *obj, + struct t2audio_prop_addr *prop); +int t2audio_msg_read_property_changed(struct t2audio_msg *msg, t2audio_device_id_t *devid, t2audio_object_id_t *obj, + struct t2audio_prop_addr *prop); +int t2audio_msg_read_set_input_stream_address_ranges_response(struct t2audio_msg *msg); +int t2audio_msg_read_get_input_stream_list_response(struct t2audio_msg *msg, t2audio_object_id_t **str_l, u64 *str_cnt); +int t2audio_msg_read_get_output_stream_list_response(struct t2audio_msg *msg, t2audio_object_id_t **str_l, u64 *str_cnt); +int t2audio_msg_read_set_remote_access_response(struct t2audio_msg *msg); +int t2audio_msg_read_get_device_list_response(struct t2audio_msg *msg, t2audio_device_id_t **dev_l, u64 *dev_cnt); + +void t2audio_msg_write_start_io(struct t2audio_msg *msg, t2audio_device_id_t dev); +void t2audio_msg_write_stop_io(struct t2audio_msg *msg, t2audio_device_id_t dev); +void t2audio_msg_write_get_property(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size); +void t2audio_msg_write_set_property(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *data, u64 data_size, void *qualifier, u64 qualifier_size); +void t2audio_msg_write_property_listener(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj, + struct t2audio_prop_addr prop); +void t2audio_msg_write_set_input_stream_address_ranges(struct t2audio_msg *msg, t2audio_device_id_t devid); +void t2audio_msg_write_get_input_stream_list(struct t2audio_msg *msg, t2audio_device_id_t devid); +void t2audio_msg_write_get_output_stream_list(struct t2audio_msg *msg, t2audio_device_id_t devid); +void t2audio_msg_write_set_remote_access(struct t2audio_msg *msg, u64 mode); +void t2audio_msg_write_alive_notification(struct t2audio_msg *msg, u32 proto_ver, u32 msg_ver); +void t2audio_msg_write_update_timestamp_response(struct t2audio_msg *msg); +void t2audio_msg_write_get_device_list(struct t2audio_msg *msg); + + +int t2audio_cmd_start_io(struct t2audio_device *a, t2audio_device_id_t devid); +int t2audio_cmd_stop_io(struct t2audio_device *a, t2audio_device_id_t devid); +int t2audio_cmd_get_property(struct t2audio_device *a, struct t2audio_msg *buf, + t2audio_device_id_t devid, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void **data, u64 *data_size); +int t2audio_cmd_get_primitive_property(struct t2audio_device *a, + t2audio_device_id_t devid, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size); +int t2audio_cmd_set_property(struct t2audio_device *a, t2audio_device_id_t devid, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size); +int t2audio_cmd_property_listener(struct t2audio_device *a, t2audio_device_id_t devid, t2audio_object_id_t obj, + struct t2audio_prop_addr prop); +int t2audio_cmd_set_input_stream_address_ranges(struct t2audio_device *a, t2audio_device_id_t devid); +int t2audio_cmd_get_input_stream_list(struct t2audio_device *a, struct t2audio_msg *buf, t2audio_device_id_t devid, + t2audio_object_id_t **str_l, u64 *str_cnt); +int t2audio_cmd_get_output_stream_list(struct t2audio_device *a, struct t2audio_msg *buf, t2audio_device_id_t devid, + t2audio_object_id_t **str_l, u64 *str_cnt); +int t2audio_cmd_set_remote_access(struct t2audio_device *a, u64 mode); +int t2audio_cmd_get_device_list(struct t2audio_device *a, struct t2audio_msg *buf, + t2audio_device_id_t **dev_l, u64 *dev_cnt); + + + +#endif //T2AUDIO_PROTOCOL_H diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_audio/protocol_bce.c b/drivers/staging/t2bce/t2bce_audio/protocol_bce.c --- a/drivers/staging/t2bce/t2bce_audio/protocol_bce.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_audio/protocol_bce.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,260 @@ +#include "protocol_bce.h" + +#include "audio.h" +#include + +static void t2audio_bce_out_queue_completion(struct t2bce_core_queue_sq *sq); +static void t2audio_bce_in_queue_completion(struct t2bce_core_queue_sq *sq); +static int t2audio_bce_queue_init(struct t2audio_device *dev, struct t2audio_bce_queue *q, const char *name, + enum dma_data_direction direction, t2bce_core_sq_completion cfn); +void t2audio_bce_in_queue_submit_pending(struct t2audio_bce_queue *q, size_t count); +static void t2audio_deferred_msg_work(struct work_struct *ws); + +int t2audio_bce_init(struct t2audio_device *dev) +{ + int status; + struct t2audio_bce *bce = &dev->bcem; + bce->cq = t2bce_core_create_cq(dev->bce, 0x80); + spin_lock_init(&bce->spinlock); + if (!bce->cq) + return -EINVAL; + if ((status = t2audio_bce_queue_init(dev, &bce->qout, "com.apple.BridgeAudio.IntelToARM", DMA_TO_DEVICE, + t2audio_bce_out_queue_completion))) { + return status; + } + if ((status = t2audio_bce_queue_init(dev, &bce->qin, "com.apple.BridgeAudio.ARMToIntel", DMA_FROM_DEVICE, + t2audio_bce_in_queue_completion))) { + return status; + } + t2audio_bce_in_queue_submit_pending(&bce->qin, bce->qin.el_count); + return 0; +} + +int t2audio_bce_queue_init(struct t2audio_device *dev, struct t2audio_bce_queue *q, const char *name, + enum dma_data_direction direction, t2bce_core_sq_completion cfn) +{ + q->cq = dev->bcem.cq; + q->el_size = T2AUDIO_BCE_QUEUE_ELEMENT_SIZE; + q->el_count = T2AUDIO_BCE_QUEUE_ELEMENT_COUNT; + /* Allocate one extra SQ slot because the BCE ring always keeps one entry empty. */ + q->sq = t2bce_core_create_sq(dev->bce, q->cq, name, (u32) (q->el_count + 1), direction, cfn, dev); + if (!q->sq) + return -EINVAL; + + q->data = dma_alloc_coherent(t2bce_core_client_dma_dev(dev->bce), q->el_size * q->el_count, &q->dma_addr, GFP_KERNEL); + if (!q->data) { + t2bce_core_destroy_sq(dev->bce, q->sq); + return -EINVAL; + } + return 0; +} + +static void t2audio_send_create_tag(struct t2audio_bce *b, int *tagn, char tag[4]) +{ + b->tag_num = (b->tag_num + 1) % T2AUDIO_BCE_QUEUE_TAG_COUNT; + *tagn = b->tag_num; + tag[0] = 'S'; + tag[1] = '0' + b->tag_num / 100; + tag[2] = '0' + (b->tag_num / 10) % 10; + tag[3] = '0' + b->tag_num % 10; +} + +int __t2audio_send_prepare(struct t2audio_bce *b, struct t2audio_send_ctx *ctx, char *tag) +{ + int status; + size_t index; + void *dptr; + struct t2audio_msg_header *header; + if ((status = t2bce_core_reserve_submission(b->qout.sq, &ctx->timeout))) + return status; + spin_lock_irqsave(&b->spinlock, ctx->irq_flags); + index = b->qout.data_tail; + dptr = (u8 *) b->qout.data + index * b->qout.el_size; + ctx->msg.data = dptr; + header = dptr; + if (tag) + *((u32 *) header->tag) = *((u32 *) tag); + else + t2audio_send_create_tag(b, &ctx->tag_n, header->tag); + return 0; +} + +void __t2audio_send(struct t2audio_bce *b, struct t2audio_send_ctx *ctx) +{ +#ifdef DEBUG + pr_debug("t2bce_audio: Sending command data\n"); + print_hex_dump(KERN_DEBUG, "t2bce_audio:OUT ", DUMP_PREFIX_NONE, 32, 1, ctx->msg.data, ctx->msg.size, true); +#endif + t2bce_core_set_next_submission_single(b->qout.sq, + b->qout.dma_addr + (dma_addr_t) (ctx->msg.data - b->qout.data), ctx->msg.size); + t2bce_core_submit_to_device(b->qout.sq); + b->qout.data_tail = (b->qout.data_tail + 1) % b->qout.el_count; + spin_unlock_irqrestore(&b->spinlock, ctx->irq_flags); +} + +int __t2audio_send_cmd_sync(struct t2audio_bce *b, struct t2audio_send_ctx *ctx, struct t2audio_msg *reply) +{ + struct t2audio_bce_queue_entry ent; + DECLARE_COMPLETION_ONSTACK(cmpl); + ent.msg = reply; + ent.cmpl = &cmpl; + b->pending_entries[ctx->tag_n] = &ent; + __t2audio_send(b, ctx); /* drops b->spinlock */ + ctx->timeout = wait_for_completion_timeout(&cmpl, ctx->timeout); + if (ctx->timeout == 0) { + /* + * Timed-out replies may still arrive later; remove the waiter before + * returning so a stale reply cannot complete a reused stack object. + */ + spin_lock_irqsave(&b->spinlock, ctx->irq_flags); + if (b->pending_entries[ctx->tag_n] == &ent) + b->pending_entries[ctx->tag_n] = NULL; + spin_unlock_irqrestore(&b->spinlock, ctx->irq_flags); + return -ETIMEDOUT; + } + return 0; +} + +static void t2audio_handle_reply(struct t2audio_bce *b, struct t2audio_msg *reply) +{ + const char *tag; + int tagn; + unsigned long irq_flags; + char tag_zero[5]; + struct t2audio_bce_queue_entry *entry; + + tag = ((struct t2audio_msg_header *) reply->data)->tag; + if (tag[0] != 'S') { + pr_err("t2bce_audio: unexpected reply tag: %.4s\n", tag); + return; + } + *((u32 *) tag_zero) = *((u32 *) tag); + tag_zero[4] = 0; + if (kstrtoint(&tag_zero[1], 10, &tagn)) { + pr_err("t2bce_audio: reply tag parse failed: %.4s\n", tag); + return; + } + + spin_lock_irqsave(&b->spinlock, irq_flags); + entry = b->pending_entries[tagn]; + if (entry) { + if (reply->size < entry->msg->size) + entry->msg->size = reply->size; + memcpy(entry->msg->data, reply->data, entry->msg->size); + complete(entry->cmpl); + + b->pending_entries[tagn] = NULL; + } else { + pr_warn_ratelimited("t2bce_audio: stale reply without queued command tag=%.4s\n", tag); + } + spin_unlock_irqrestore(&b->spinlock, irq_flags); +} + +static void t2audio_bce_out_queue_completion(struct t2bce_core_queue_sq *sq) +{ + while (t2bce_core_next_completion(sq)) { + t2bce_core_notify_submission_complete(sq); + } +} + +static void t2audio_bce_in_queue_handle_msg(struct t2audio_device *a, struct t2audio_msg *msg); + +static void t2audio_deferred_msg_work(struct work_struct *ws) +{ + struct t2audio_deferred_msg *work = container_of(ws, struct t2audio_deferred_msg, ws); + struct t2audio_msg_header *header = work->msg.data; + + if (header->type == T2AUDIO_MSG_TYPE_COMMAND) + t2audio_handle_command(work->a, &work->msg); + else if (header->type == T2AUDIO_MSG_TYPE_NOTIFICATION) + t2audio_handle_notification(work->a, &work->msg); + + kfree(work->msg.data); + kfree(work); +} + +static void t2audio_bce_in_queue_completion(struct t2bce_core_queue_sq *sq) +{ + struct t2audio_msg msg; + struct t2audio_device *dev = t2bce_core_queue_sq_userdata(sq); + struct t2audio_bce_queue *q = &dev->bcem.qin; + struct t2bce_core_sq_completion_data *c; + size_t cnt = 0; + + mb(); + while ((c = t2bce_core_next_completion(sq))) { + msg.data = (u8 *) q->data + q->data_head * q->el_size; + msg.size = c->data_size; +#ifdef DEBUG + pr_debug("t2bce_audio: Received command data %llx\n", c->data_size); + print_hex_dump(KERN_DEBUG, "t2bce_audio:IN ", DUMP_PREFIX_NONE, 32, 1, msg.data, min(msg.size, 128UL), true); +#endif + t2audio_bce_in_queue_handle_msg(dev, &msg); + + q->data_head = (q->data_head + 1) % q->el_count; + + t2bce_core_notify_submission_complete(sq); + ++cnt; + } + t2audio_bce_in_queue_submit_pending(q, cnt); +} + +static void t2audio_bce_in_queue_handle_msg(struct t2audio_device *a, struct t2audio_msg *msg) +{ + struct t2audio_msg_header *header = (struct t2audio_msg_header *) msg->data; + struct t2audio_deferred_msg *work; + + if (msg->size < sizeof(struct t2audio_msg_header)) { + pr_err("t2bce_audio: Msg size smaller than header (%lx)", msg->size); + return; + } + if (header->type == T2AUDIO_MSG_TYPE_RESPONSE) { + t2audio_handle_reply(&a->bcem, msg); + return; + } + + work = kzalloc(sizeof(*work), GFP_ATOMIC); + if (!work) { + pr_err("t2bce_audio: Failed to allocate deferred message work\n"); + return; + } + + work->msg.data = kmemdup(msg->data, msg->size, GFP_ATOMIC); + if (!work->msg.data) { + kfree(work); + pr_err("t2bce_audio: Failed to copy deferred message\n"); + return; + } + + work->a = a; + work->msg.size = msg->size; + INIT_WORK(&work->ws, t2audio_deferred_msg_work); + schedule_work(&work->ws); +} + +void t2audio_bce_in_queue_submit_pending(struct t2audio_bce_queue *q, size_t count) +{ + while (count--) { + if (t2bce_core_reserve_submission(q->sq, NULL)) { + pr_err("t2bce_audio: Failed to reserve an event queue submission\n"); + break; + } + t2bce_core_set_next_submission_single(q->sq, + q->dma_addr + (dma_addr_t) (q->data_tail * q->el_size), q->el_size); + q->data_tail = (q->data_tail + 1) % q->el_count; + } + t2bce_core_submit_to_device(q->sq); +} + +struct t2audio_msg t2audio_reply_alloc(void) +{ + struct t2audio_msg ret; + ret.size = T2AUDIO_BCE_QUEUE_ELEMENT_SIZE; + ret.data = kmalloc(ret.size, GFP_KERNEL); + return ret; +} + +void t2audio_reply_free(struct t2audio_msg *reply) +{ + kfree(reply->data); +} diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_audio/protocol_bce.h b/drivers/staging/t2bce/t2bce_audio/protocol_bce.h --- a/drivers/staging/t2bce/t2bce_audio/protocol_bce.h 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_audio/protocol_bce.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,72 @@ +#ifndef T2AUDIO_PROTOCOL_BCE_H +#define T2AUDIO_PROTOCOL_BCE_H + +#include "protocol.h" +#include "t2bce_core_transport.h" + +#define T2AUDIO_BCE_QUEUE_ELEMENT_SIZE 0x1000 +#define T2AUDIO_BCE_QUEUE_ELEMENT_COUNT 20 + +#define T2AUDIO_BCE_QUEUE_TAG_COUNT 1000 + +struct t2audio_device; + +struct t2audio_bce_queue_entry { + struct t2audio_msg *msg; + struct completion *cmpl; +}; +struct t2audio_bce_queue { + struct t2bce_core_queue_cq *cq; + struct t2bce_core_queue_sq *sq; + void *data; + dma_addr_t dma_addr; + size_t data_head, data_tail; + size_t el_size, el_count; +}; +struct t2audio_bce { + struct t2bce_core_queue_cq *cq; + struct t2audio_bce_queue qin; + struct t2audio_bce_queue qout; + int tag_num; + struct t2audio_bce_queue_entry *pending_entries[T2AUDIO_BCE_QUEUE_TAG_COUNT]; + struct spinlock spinlock; +}; + +struct t2audio_send_ctx { + int status; + int tag_n; + unsigned long irq_flags; + struct t2audio_msg msg; + unsigned long timeout; +}; + +int t2audio_bce_init(struct t2audio_device *dev); +int __t2audio_send_prepare(struct t2audio_bce *b, struct t2audio_send_ctx *ctx, char *tag); +void __t2audio_send(struct t2audio_bce *b, struct t2audio_send_ctx *ctx); +int __t2audio_send_cmd_sync(struct t2audio_bce *b, struct t2audio_send_ctx *ctx, struct t2audio_msg *reply); + +#define t2audio_send_with_tag(a, ctx, tag, tout, fn, ...) ({ \ + (ctx)->timeout = msecs_to_jiffies(tout); \ + (ctx)->status = __t2audio_send_prepare(&(a)->bcem, (ctx), (tag)); \ + if (!(ctx)->status) { \ + fn(&(ctx)->msg, ##__VA_ARGS__); \ + __t2audio_send(&(a)->bcem, (ctx)); \ + } \ + (ctx)->status; \ +}) +#define t2audio_send(a, ctx, tout, fn, ...) t2audio_send_with_tag(a, ctx, NULL, tout, fn, ##__VA_ARGS__) + +#define t2audio_send_cmd_sync(a, ctx, reply, tout, fn, ...) ({ \ + (ctx)->timeout = msecs_to_jiffies(tout); \ + (ctx)->status = __t2audio_send_prepare(&(a)->bcem, (ctx), NULL); \ + if (!(ctx)->status) { \ + fn(&(ctx)->msg, ##__VA_ARGS__); \ + (ctx)->status = __t2audio_send_cmd_sync(&(a)->bcem, (ctx), (reply)); \ + } \ + (ctx)->status; \ +}) + +struct t2audio_msg t2audio_reply_alloc(void); +void t2audio_reply_free(struct t2audio_msg *reply); + +#endif //T2AUDIO_PROTOCOL_BCE_H diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_core/Makefile b/drivers/staging/t2bce/t2bce_core/Makefile --- a/drivers/staging/t2bce/t2bce_core/Makefile 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_core/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_T2BCE_CORE) += t2bce_core.o +t2bce_core-y := t2bce_main.o mailbox.o transport.o + +ccflags-y += -DWITHOUT_NVME_PATCH +ccflags-y += -I$(src)/include +ccflags-y += -I$(src)/../t2bce_dma/include diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_core/include/t2bce_core_transport.h b/drivers/staging/t2bce/t2bce_core/include/t2bce_core_transport.h --- a/drivers/staging/t2bce/t2bce_core/include/t2bce_core_transport.h 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_core/include/t2bce_core_transport.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,83 @@ +#ifndef T2BCE_CORE_TRANSPORT_H +#define T2BCE_CORE_TRANSPORT_H + +#include +#include +#include + +struct t2bce_core_client; +struct t2bce_core_queue_cq; +struct t2bce_core_queue_sq; + +typedef void (*t2bce_core_sq_completion)(struct t2bce_core_queue_sq *sq); +typedef void (*t2bce_core_client_resume_callback)(void *userdata); + +struct t2bce_core_client_pm_ops { + void (*shutdown)(void *userdata); + void (*pm_reset)(void *userdata); + void (*pm_prepare)(void *userdata); + void (*pm_prepare_no_state)(void *userdata); + void (*pm_mark_no_state_resume)(void *userdata); + bool (*pm_is_no_state_resume)(void *userdata); + void (*pm_complete)(void *userdata); +}; + +struct t2bce_core_sq_completion_data { + u32 status; + u64 data_size; + u64 result; +}; + +struct t2bce_core_dma_segment { + u64 addr; + u64 length; +}; + +enum t2bce_completion_status { + T2BCE_COMPLETION_SUCCESS = 0, + T2BCE_COMPLETION_ERROR = 1, + T2BCE_COMPLETION_ABORTED = 2, + T2BCE_COMPLETION_NO_SPACE = 3, + T2BCE_COMPLETION_OVERRUN = 4, +}; + +struct t2bce_core_client *t2bce_core_client_get(struct device *dev); +void t2bce_core_client_put(struct t2bce_core_client *client); + +struct device *t2bce_core_device_get(void); +void t2bce_core_device_put(struct device *dev); +struct device *t2bce_core_client_dma_dev(struct t2bce_core_client *client); +bool t2bce_core_client_no_state_resume(struct t2bce_core_client *client); + +void t2bce_core_client_set_resume_complete_callback(struct t2bce_core_client *client, + t2bce_core_client_resume_callback callback, void *userdata); +void t2bce_core_client_set_pm_ops(struct t2bce_core_client *client, + const struct t2bce_core_client_pm_ops *ops, void *userdata); + +struct t2bce_core_queue_cq *t2bce_core_create_cq(struct t2bce_core_client *client, u32 el_count); +void t2bce_core_destroy_cq(struct t2bce_core_client *client, struct t2bce_core_queue_cq *cq); + +struct t2bce_core_queue_sq *t2bce_core_create_sq(struct t2bce_core_client *client, struct t2bce_core_queue_cq *cq, + const char *name, u32 el_count, enum dma_data_direction direction, + t2bce_core_sq_completion compl, void *userdata); +void t2bce_core_destroy_sq(struct t2bce_core_client *client, struct t2bce_core_queue_sq *sq); + +void *t2bce_core_queue_sq_userdata(struct t2bce_core_queue_sq *sq); + +int t2bce_core_reserve_submission(struct t2bce_core_queue_sq *sq, unsigned long *timeout); +void t2bce_core_cancel_submission_reservation(struct t2bce_core_queue_sq *sq); +void t2bce_core_set_next_submission_single(struct t2bce_core_queue_sq *sq, dma_addr_t addr, size_t size); +void t2bce_core_set_next_submission_segment_list(struct t2bce_core_queue_sq *sq, + dma_addr_t segl_addr, size_t segl_size); +void t2bce_core_submit_to_device(struct t2bce_core_queue_sq *sq); +void t2bce_core_notify_submission_complete(struct t2bce_core_queue_sq *sq); + +struct t2bce_core_sq_completion_data *t2bce_core_next_completion(struct t2bce_core_queue_sq *sq); + +u32 t2bce_core_queue_sq_head(struct t2bce_core_queue_sq *sq); +u32 t2bce_core_queue_sq_tail(struct t2bce_core_queue_sq *sq); +u32 t2bce_core_queue_sq_available(struct t2bce_core_queue_sq *sq); +u32 t2bce_core_queue_sq_capacity(struct t2bce_core_queue_sq *sq); +int t2bce_core_flush_queue(struct t2bce_core_client *client, struct t2bce_core_queue_sq *sq); + +#endif diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_core/mailbox.c b/drivers/staging/t2bce/t2bce_core/mailbox.c --- a/drivers/staging/t2bce/t2bce_core/mailbox.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_core/mailbox.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,234 @@ +#include "mailbox.h" +#include +#include "t2bce.h" +#include + +#define REG_MBOX_OUT_BASE 0x820 +#define REG_MBOX_REPLY_COUNTER 0x108 +#define REG_MBOX_REPLY_BASE 0x810 +#define REG_TIMESTAMP_BASE 0xC000 + +#define BCE_MBOX_TIMEOUT_MS 1000 + +void bce_mailbox_init(struct bce_mailbox *mb, void __iomem *reg_mb) +{ + mb->reg_mb = reg_mb; + mutex_init(&mb->channel_lock); + init_completion(&mb->mb_completion); +} + +int bce_mailbox_channel_pause(struct bce_mailbox *mb) +{ + /* + * System PM needs to exclude new mailbox transactions while the core + * driver is moving bridgeOS into or out of sleep. + */ + mutex_lock(&mb->channel_lock); + return 0; +} + +void bce_mailbox_channel_resume(struct bce_mailbox *mb) +{ + mutex_unlock(&mb->channel_lock); +} + +int bce_mailbox_send_locked_timeout(struct bce_mailbox *mb, u64 msg, u64* recv, unsigned int timeout_ms) +{ + u32 __iomem *regb; + int status = 0; + + if (atomic_cmpxchg(&mb->mb_status, 0, 1) != 0) { + status = -EEXIST; + goto out; + } + reinit_completion(&mb->mb_completion); + + pr_debug("bce_mailbox_send: %llx\n", msg); + regb = (u32*) ((u8*) mb->reg_mb + REG_MBOX_OUT_BASE); + iowrite32((u32) msg, regb); + iowrite32((u32) (msg >> 32), regb + 1); + iowrite32(0, regb + 2); + iowrite32(0, regb + 3); + + wait_for_completion_timeout(&mb->mb_completion, msecs_to_jiffies(timeout_ms)); + if (atomic_read(&mb->mb_status) != 2) { + atomic_set(&mb->mb_status, 0); + status = -ETIMEDOUT; + goto out; + } + + *recv = mb->mb_result; + pr_debug("bce_mailbox_send: reply %llx\n", *recv); + + atomic_set(&mb->mb_status, 0); +out: + return status; +} + +int bce_mailbox_send_timeout(struct bce_mailbox *mb, u64 msg, u64* recv, unsigned int timeout_ms) +{ + int status; + + /* Serialize mailbox traffic; bridgeOS accepts one active command at a time. */ + mutex_lock(&mb->channel_lock); + status = bce_mailbox_send_locked_timeout(mb, msg, recv, timeout_ms); + mutex_unlock(&mb->channel_lock); + return status; +} + +int bce_mailbox_send(struct bce_mailbox *mb, u64 msg, u64* recv) +{ + return bce_mailbox_send_timeout(mb, msg, recv, BCE_MBOX_TIMEOUT_MS); +} + +int bce_mailbox_send_locked(struct bce_mailbox *mb, u64 msg, u64 *recv) +{ + return bce_mailbox_send_locked_timeout(mb, msg, recv, BCE_MBOX_TIMEOUT_MS); +} + +int bce_mailbox_send_no_reply_locked(struct bce_mailbox *mb, u64 msg) +{ + u32 __iomem *regb; + int status = 0; + + if (atomic_cmpxchg(&mb->mb_status, 0, 1) != 0) { + status = -EEXIST; + goto out; + } + + pr_debug("bce_mailbox_send_no_reply: %llx\n", msg); + regb = (u32*) ((u8*) mb->reg_mb + REG_MBOX_OUT_BASE); + iowrite32((u32) msg, regb); + iowrite32((u32) (msg >> 32), regb + 1); + iowrite32(0, regb + 2); + iowrite32(0, regb + 3); + + atomic_set(&mb->mb_status, 0); +out: + return status; +} + +int bce_mailbox_send_no_reply(struct bce_mailbox *mb, u64 msg) +{ + int status; + + /* Fire-and-forget commands share the same mailbox serialization. */ + mutex_lock(&mb->channel_lock); + status = bce_mailbox_send_no_reply_locked(mb, msg); + mutex_unlock(&mb->channel_lock); + return status; +} + +static int bce_mailbox_retrive_response(struct bce_mailbox *mb) +{ + u32 __iomem *regb; + u32 lo, hi; + int count, counter; + u32 res = ioread32((u8*) mb->reg_mb + REG_MBOX_REPLY_COUNTER); + count = (res >> 20) & 0xf; + counter = count; + pr_debug("bce_mailbox_retrive_response count=%i\n", count); + while (counter--) { + regb = (u32*) ((u8*) mb->reg_mb + REG_MBOX_REPLY_BASE); + lo = ioread32(regb); + hi = ioread32(regb + 1); + ioread32(regb + 2); + ioread32(regb + 3); + pr_debug("bce_mailbox_retrive_response %llx\n", ((u64) hi << 32) | lo); + mb->mb_result = ((u64) hi << 32) | lo; + } + return count > 0 ? 0 : -ENODATA; +} + +int bce_mailbox_handle_interrupt(struct bce_mailbox *mb) +{ + int status = bce_mailbox_retrive_response(mb); + if (!status) { + if (atomic_read(&mb->mb_status) == 1) { + atomic_set(&mb->mb_status, 2); + complete(&mb->mb_completion); + } else { + pr_debug("bce_mailbox_handle_interrupt: unsolicited reply %llx\n", mb->mb_result); + atomic_set(&mb->mb_status, 0); + } + } + return status; +} + +static void bce_xhci_pm_tick(struct timer_list *tl); + +void bce_xhci_pm_init(struct bce_xhci_pm *pm, void __iomem *reg) +{ + u32 __iomem *regb; + + spin_lock_init(&pm->stop_sl); + pm->stopped = false; + + pm->reg = reg; + + regb = (u32*) ((u8*) pm->reg + REG_TIMESTAMP_BASE); + + ioread32(regb); + mb(); + + timer_setup(&pm->timer, bce_xhci_pm_tick, 0); +} + +void bce_xhci_pm_start(struct bce_xhci_pm *pm, bool is_initial) +{ + unsigned long flags; + u32 __iomem *regb = (u32*) ((u8*) pm->reg + REG_TIMESTAMP_BASE); + + if (is_initial) { + iowrite32((u32) -4, regb + 2); + iowrite32((u32) -1, regb); + } else { + iowrite32((u32) -3, regb + 2); + iowrite32((u32) -1, regb); + } + + spin_lock_irqsave(&pm->stop_sl, flags); + pm->stopped = false; + spin_unlock_irqrestore(&pm->stop_sl, flags); + mod_timer(&pm->timer, jiffies + msecs_to_jiffies(150)); +} + +void bce_xhci_pm_stop(struct bce_xhci_pm *pm) +{ + unsigned long flags; + u32 __iomem *regb = (u32*) ((u8*) pm->reg + REG_TIMESTAMP_BASE); + + spin_lock_irqsave(&pm->stop_sl, flags); + pm->stopped = true; + spin_unlock_irqrestore(&pm->stop_sl, flags); +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,15,0) + del_timer_sync(&pm->timer); +#else + timer_delete_sync(&pm->timer); +#endif + iowrite32((u32) -2, regb + 2); + iowrite32((u32) -1, regb); +} + +static void bce_xhci_pm_tick(struct timer_list *tl) +{ + struct bce_xhci_pm *pm; + unsigned long flags; + u32 __iomem *regb; + ktime_t bt; + + pm = container_of(tl, struct bce_xhci_pm, timer); + regb = (u32*) ((u8*) pm->reg + REG_TIMESTAMP_BASE); + local_irq_save(flags); + ioread32(regb + 2); + mb(); + bt = ktime_get_boottime(); + iowrite32((u32) bt, regb + 2); + iowrite32((u32) (bt >> 32), regb); + + spin_lock(&pm->stop_sl); + if (!pm->stopped) + mod_timer(&pm->timer, jiffies + msecs_to_jiffies(150)); + spin_unlock(&pm->stop_sl); + local_irq_restore(flags); +} diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_core/mailbox.h b/drivers/staging/t2bce/t2bce_core/mailbox.h --- a/drivers/staging/t2bce/t2bce_core/mailbox.h 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_core/mailbox.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,64 @@ +#ifndef BCE_MAILBOX_H +#define BCE_MAILBOX_H + +#include +#include +#include +#include + +struct bce_mailbox { + void __iomem *reg_mb; + struct mutex channel_lock; + + atomic_t mb_status; // possible statuses: 0 (no msg), 1 (has active msg), 2 (got reply) + struct completion mb_completion; + uint64_t mb_result; +}; + +enum bce_message_type { + BCE_MB_REGISTER_COMMAND_SQ = 0x7, // to-device + BCE_MB_REGISTER_COMMAND_CQ = 0x8, // to-device + BCE_MB_REGISTER_COMMAND_QUEUE_REPLY = 0xB, // to-host + BCE_MB_SET_FW_PROTOCOL_VERSION = 0xC, // both + BCE_MB_SLEEP_NO_STATE = 0x14, // to-device + BCE_MB_RESTORE_NO_STATE = 0x15, // to-device + BCE_MB_SLEEP_LIGHT = 0x16, // to-device, unused; reserved for future runtime-PM work + BCE_MB_SAVE_STATE_AND_SLEEP = 0x17, // to-device + BCE_MB_RESTORE_STATE_AND_WAKE = 0x18, // to-device + BCE_MB_SAVE_STATE_AND_SLEEP_REJECTED = 0x19, // from-device, stateful suspend payload rejected + BCE_MB_SAVE_RESTORE_STATE_COMPLETE = 0x1A, // from-device +}; + +#define BCE_MB_MSG(type, value) (((u64) (type) << 58) | ((value) & 0x3FFFFFFFFFFFFFFLL)) +#define BCE_MB_TYPE(v) ((u32) (v >> 58)) +#define BCE_MB_VALUE(v) (v & 0x3FFFFFFFFFFFFFFLL) + +void bce_mailbox_init(struct bce_mailbox *mb, void __iomem *reg_mb); + +int bce_mailbox_channel_pause(struct bce_mailbox *mb); +void bce_mailbox_channel_resume(struct bce_mailbox *mb); + +int bce_mailbox_send(struct bce_mailbox *mb, u64 msg, u64* recv); +int bce_mailbox_send_timeout(struct bce_mailbox *mb, u64 msg, u64 *recv, unsigned int timeout_ms); +int bce_mailbox_send_no_reply(struct bce_mailbox *mb, u64 msg); +int bce_mailbox_send_locked(struct bce_mailbox *mb, u64 msg, u64 *recv); +int bce_mailbox_send_locked_timeout(struct bce_mailbox *mb, u64 msg, u64 *recv, unsigned int timeout_ms); +int bce_mailbox_send_no_reply_locked(struct bce_mailbox *mb, u64 msg); + +int bce_mailbox_handle_interrupt(struct bce_mailbox *mb); + + +struct bce_xhci_pm { + void __iomem *reg; + struct timer_list timer; + struct spinlock stop_sl; + bool stopped; +}; + +void bce_xhci_pm_init(struct bce_xhci_pm *pm, void __iomem *reg); + +void bce_xhci_pm_start(struct bce_xhci_pm *pm, bool is_initial); + +void bce_xhci_pm_stop(struct bce_xhci_pm *pm); + +#endif //BCEDRIVER_MAILBOX_H diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_core/t2bce.h b/drivers/staging/t2bce/t2bce_core/t2bce.h --- a/drivers/staging/t2bce/t2bce_core/t2bce.h 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_core/t2bce.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,51 @@ +#ifndef APPLE_BCE_H +#define APPLE_BCE_H + +#include +#include +#include +#include +#include +#include "mailbox.h" +#include "t2bce_dma_queue.h" + +#define BC_PROTOCOL_VERSION 0x20001 + +struct t2bce_device { + struct pci_dev *pci, *pci0; + dev_t devt; + struct device *dev; + void __iomem *reg_mem_mb; + void __iomem *reg_mem_dma; + struct t2bce_dma_engine dma; + struct bce_mailbox mbox; + struct bce_xhci_pm xhci_pm; + bool is_being_removed; + + dma_addr_t saved_data_dma_addr; + void *saved_data_dma_ptr; + size_t saved_data_dma_size; + u32 fw_version; + bool stateful_suspend_valid; + bool no_state_fallback; + bool no_state_resume; + bool mailbox_channel_active; + struct mutex pm_lock; + struct mutex clients_lock; + struct list_head clients; + struct srcu_struct clients_srcu; + bool clients_srcu_ready; +}; + +extern struct t2bce_device *global_bce; + +void t2bce_core_notify_resume_complete(struct t2bce_device *bce); +void t2bce_core_clients_shutdown(struct t2bce_device *bce); +void t2bce_core_clients_pm_reset(struct t2bce_device *bce); +void t2bce_core_clients_pm_prepare(struct t2bce_device *bce); +void t2bce_core_clients_pm_prepare_no_state(struct t2bce_device *bce); +void t2bce_core_clients_pm_mark_no_state_resume(struct t2bce_device *bce); +bool t2bce_core_clients_pm_has_no_state_resume(struct t2bce_device *bce); +void t2bce_core_clients_pm_complete(struct t2bce_device *bce); + +#endif //APPLE_BCE_H diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_core/t2bce_main.c b/drivers/staging/t2bce/t2bce_core/t2bce_main.c --- a/drivers/staging/t2bce/t2bce_core/t2bce_main.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_core/t2bce_main.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,770 @@ +#include "t2bce.h" +#include +#include +#include +#include + +/* + * Apple T2 Macs contain a coprocessor serving as a board supervisor. + * The T2 is basically a repurposed iPhone CPU running bridgeOS, + * a stripped down version of WatchOS. Apple reuses the audio codec + * and USB interfaces of this SOC and uses a "Buffer Copy Engine" (BCE) + * for communication between the host OS, the T2 and the VHCI USB devices + * behind the T2 USB hub like internal keyboard, trackpad, touchbar etc.. + * + * This core driver owns the BCE PCI device, mailbox, DMA queue registration + * and system PM ordering. Other modules bind as clients: + * - t2bce_audio exposes the bridgeOS audio endpoints to ALSA. + * - t2bce_vhci exposes the T2-internal virtual USB hub and devices to usbcore. + * - t2bce_dma provides the queue engine used by both. + * + * ACPI describes the host-side PCI devices, but bridgeOS still controls the + * remote endpoints and must be kept in sync during probe, shutdown and system + * suspend/resume. The PM code below mirrors the observed firmware ordering: + * It quiesces clients, drains the mailbox channel, asks bridgeOS to save or discard + * BCE state, then lets clients rebuild or continue according to the selected + * resume path. + */ + +static dev_t bce_chrdev; +static struct class *bce_class; +static const bool bce_use_stateful_sleep = true; + +struct t2bce_device *global_bce; + +static int bce_create_command_queues(struct t2bce_device *bce); +static void bce_free_command_queues(struct t2bce_device *bce); +static irqreturn_t bce_handle_mb_irq(int irq, void *dev); +static irqreturn_t bce_handle_dma_irq(int irq, void *dev); +static int bce_fw_version_handshake(struct t2bce_device *bce); +static int bce_register_command_queue(struct t2bce_device *bce, struct bce_queue_memcfg *cfg, int is_sq); +static int t2bce_dma_register_queue(void *userdata, struct bce_queue_memcfg *cfg, + const char *name, bool isdirout); +static int t2bce_dma_unregister_queue(void *userdata, u16 qid); +static int t2bce_dma_flush_queue(void *userdata, u16 qid); +static int bce_alloc_state_buffer(struct t2bce_device *bce); +static void bce_free_state_buffer(struct t2bce_device *bce); +static int bce_pm_suspend_prepare(struct t2bce_device *bce); +static void bce_pm_suspend_abort(struct t2bce_device *bce); +static void bce_pm_resume_finish(struct t2bce_device *bce); + +static const struct t2bce_dma_engine_ops t2bce_dma_ops = { + .register_queue = t2bce_dma_register_queue, + .unregister_queue = t2bce_dma_unregister_queue, + .flush_queue = t2bce_dma_flush_queue, +}; + +static int bce_alloc_state_buffer(struct t2bce_device *bce) +{ + /* bridgeOS stateful sleep stores BCE state in a persistent 0x2000 buffer. */ + bce->saved_data_dma_size = 0x2000; + bce->saved_data_dma_ptr = dma_alloc_coherent(&bce->pci->dev, bce->saved_data_dma_size, + &bce->saved_data_dma_addr, GFP_KERNEL); + if (!bce->saved_data_dma_ptr) { + bce->saved_data_dma_size = 0; + bce->saved_data_dma_addr = 0; + return -ENOMEM; + } + + return 0; +} + +static void bce_free_state_buffer(struct t2bce_device *bce) +{ + if (!bce->saved_data_dma_ptr) + return; + + dma_free_coherent(&bce->pci->dev, bce->saved_data_dma_size, + bce->saved_data_dma_ptr, bce->saved_data_dma_addr); + bce->saved_data_dma_ptr = NULL; + bce->saved_data_dma_addr = 0; + bce->saved_data_dma_size = 0; +} + +static int t2bce_probe(struct pci_dev *dev, const struct pci_device_id *id) +{ + struct t2bce_device *bce = NULL; + int status = 0; + int nvec; + + pr_debug("t2bce_core: capturing our device\n"); + + if (pci_enable_device(dev)) + return -ENODEV; + if (pci_request_regions(dev, "t2bce_core")) { + status = -ENODEV; + goto fail; + } + pci_set_master(dev); + nvec = pci_alloc_irq_vectors(dev, 1, 8, PCI_IRQ_MSI); + if (nvec < 5) { + status = -EINVAL; + goto fail; + } + + bce = kzalloc(sizeof(struct t2bce_device), GFP_KERNEL); + if (!bce) { + status = -ENOMEM; + goto fail; + } + + bce->pci = dev; + pci_set_drvdata(dev, bce); + + bce->devt = bce_chrdev; + bce->dev = device_create(bce_class, &dev->dev, bce->devt, NULL, "t2bce_core"); + if (IS_ERR_OR_NULL(bce->dev)) { + status = PTR_ERR(bce->dev); + goto fail; + } + + bce->reg_mem_mb = pci_iomap(dev, 4, 0); + bce->reg_mem_dma = pci_iomap(dev, 2, 0); + + if (IS_ERR_OR_NULL(bce->reg_mem_mb) || IS_ERR_OR_NULL(bce->reg_mem_dma)) { + dev_warn(&dev->dev, "t2bce_core: Failed to pci_iomap required regions\n"); + goto fail; + } + + bce_mailbox_init(&bce->mbox, bce->reg_mem_mb); + bce_xhci_pm_init(&bce->xhci_pm, bce->reg_mem_mb); + bce->dma.dma_dev = &dev->dev; + bce->dma.ops = &t2bce_dma_ops; + bce->dma.ops_userdata = bce; + bce->dma.reg_mem_dma = bce->reg_mem_dma; + + spin_lock_init(&bce->dma.queues_lock); + ida_init(&bce->dma.queue_ida); + mutex_init(&bce->pm_lock); + mutex_init(&bce->clients_lock); + INIT_LIST_HEAD(&bce->clients); + status = init_srcu_struct(&bce->clients_srcu); + if (status) + goto fail; + bce->clients_srcu_ready = true; + bce->mailbox_channel_active = true; + + if ((status = pci_request_irq(dev, 0, bce_handle_mb_irq, NULL, dev, "bce_mbox"))) + goto fail; + if ((status = pci_request_irq(dev, 4, NULL, bce_handle_dma_irq, dev, "bce_dma"))) + goto fail_interrupt_0; + + if ((status = dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(37)))) { + dev_warn(&dev->dev, "dma: Setting mask failed\n"); + goto fail_interrupt; + } + + if ((status = bce_alloc_state_buffer(bce))) + goto fail_interrupt; + + /* + * DMA on the BCE function depends on function 0 also being bus master. + * Keep a reference so probe and resume can restore that firmware-visible + * host state. + */ + bce->pci0 = pci_get_slot(dev->bus, PCI_DEVFN(PCI_SLOT(dev->devfn), 0)); +#ifndef WITHOUT_NVME_PATCH + if ((status = pci_enable_device_mem(bce->pci0))) { + dev_warn(&dev->dev, "t2bce_core: failed to enable function 0\n"); + goto fail_dev0; + } +#endif + pci_set_master(bce->pci0); + + /* bridgeOS expects the XHCI PM doorbell before the controller becomes active. */ + bce_xhci_pm_start(&bce->xhci_pm, true); + + if ((status = bce_fw_version_handshake(bce))) + goto fail_ts; + pr_debug("t2bce_core: handshake done\n"); + + if ((status = bce_create_command_queues(bce))) { + pr_err("t2bce_core: Creating command queues failed\n"); + goto fail_ts; + } + + global_bce = bce; + pr_info("t2bce_core: initialized\n"); + + return 0; + +fail_ts: + bce_free_state_buffer(bce); + bce_xhci_pm_stop(&bce->xhci_pm); +#ifndef WITHOUT_NVME_PATCH + pci_disable_device(bce->pci0); +fail_dev0: +#endif + pci_dev_put(bce->pci0); +fail_interrupt: + pci_free_irq(dev, 4, dev); +fail_interrupt_0: + pci_free_irq(dev, 0, dev); +fail: + if (bce && bce->dev) { + device_destroy(bce_class, bce->devt); + + if (!IS_ERR_OR_NULL(bce->reg_mem_mb)) + pci_iounmap(dev, bce->reg_mem_mb); + if (!IS_ERR_OR_NULL(bce->reg_mem_dma)) + pci_iounmap(dev, bce->reg_mem_dma); + + if (bce->clients_srcu_ready) + cleanup_srcu_struct(&bce->clients_srcu); + + kfree(bce); + } + + pci_free_irq_vectors(dev); + pci_release_regions(dev); + pci_disable_device(dev); + + if (!status) + status = -EINVAL; + return status; +} + +static int bce_create_command_queues(struct t2bce_device *bce) +{ + int status; + struct bce_queue_memcfg *cfg; + struct t2bce_dma_engine *dma = &bce->dma; + + dma->cmd_cq = t2bce_dma_alloc_cq(dma, 0, 0x20); + dma->cmd_cmdq = t2bce_dma_alloc_cmdq(dma, 1, 0x20); + if (dma->cmd_cq == NULL || dma->cmd_cmdq == NULL) { + status = -ENOMEM; + goto err; + } + dma->queues[0] = (struct bce_queue *) dma->cmd_cq; + dma->queues[1] = (struct bce_queue *) dma->cmd_cmdq->sq; + + cfg = kzalloc(sizeof(struct bce_queue_memcfg), GFP_KERNEL); + if (!cfg) { + status = -ENOMEM; + goto err; + } + t2bce_dma_get_cq_memcfg(dma->cmd_cq, cfg); + if ((status = bce_register_command_queue(bce, cfg, false))) + goto err; + t2bce_dma_get_sq_memcfg(dma->cmd_cmdq->sq, dma->cmd_cq, cfg); + if ((status = bce_register_command_queue(bce, cfg, true))) + goto err; + kfree(cfg); + + return 0; + +err: + if (dma->cmd_cq) + t2bce_dma_free_cq(dma, dma->cmd_cq); + if (dma->cmd_cmdq) + t2bce_dma_free_cmdq(dma, dma->cmd_cmdq); + return status; +} + +static void bce_free_command_queues(struct t2bce_device *bce) +{ + struct t2bce_dma_engine *dma = &bce->dma; + + t2bce_dma_free_cq(dma, dma->cmd_cq); + t2bce_dma_free_cmdq(dma, dma->cmd_cmdq); + dma->cmd_cq = NULL; + dma->queues[0] = NULL; +} + +static irqreturn_t bce_handle_mb_irq(int irq, void *dev) +{ + struct t2bce_device *bce = pci_get_drvdata(dev); + bce_mailbox_handle_interrupt(&bce->mbox); + return IRQ_HANDLED; +} + +static irqreturn_t bce_handle_dma_irq(int irq, void *dev) +{ + int i; + size_t ce = 0; + struct t2bce_device *bce = pci_get_drvdata(dev); + struct t2bce_dma_engine *dma = &bce->dma; + + spin_lock(&dma->queues_lock); + for (i = 0; i < BCE_MAX_QUEUE_COUNT; i++) + if (dma->queues[i] && dma->queues[i]->type == BCE_QUEUE_CQ) + t2bce_dma_handle_cq_completions_locked(dma, (struct bce_queue_cq *) dma->queues[i], &ce); + spin_unlock(&dma->queues_lock); + t2bce_dma_dispatch_pending_sq_completions(dma, ce); + return IRQ_HANDLED; +} + +static int bce_fw_version_handshake(struct t2bce_device *bce) +{ + u64 result; + u32 fw_version; + int status; + + if ((status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(BCE_MB_SET_FW_PROTOCOL_VERSION, BC_PROTOCOL_VERSION), + &result))) + return status; + fw_version = (u32) BCE_MB_VALUE(result); + if (BCE_MB_TYPE(result) != BCE_MB_SET_FW_PROTOCOL_VERSION || + fw_version > BC_PROTOCOL_VERSION) { + pr_err("t2bce_core: FW version handshake failed %x:%llx\n", BCE_MB_TYPE(result), BCE_MB_VALUE(result)); + return -EINVAL; + } + bce->fw_version = fw_version; + return 0; +} + +static bool bce_stateful_supported(struct t2bce_device *bce) +{ + /* bridgeOS exposes stateful sleep through protocol version 0x20001+. */ + return bce_use_stateful_sleep && bce->fw_version >= BC_PROTOCOL_VERSION; +} + +static int bce_pm_wait_mailbox_idle(struct t2bce_device *bce) +{ + int retries = 100; + + while (atomic_read(&bce->mbox.mb_status) != 0) { + if (!--retries) { + pr_err("t2bce_core: timeout waiting for BCE mailbox channel to quiesce\n"); + return -ETIMEDOUT; + } + usleep_range(5000, 10000); + } + + return 0; +} + +static int bce_pm_channel_pause(struct t2bce_device *bce) +{ + int status; + + status = bce_mailbox_channel_pause(&bce->mbox); + if (status) + return status; + + status = bce_pm_wait_mailbox_idle(bce); + + if (status) + bce_mailbox_channel_resume(&bce->mbox); + if (status) + return status; + + /* Keep new mailbox commands out while bridgeOS is entering sleep. */ + bce->mailbox_channel_active = false; + return 0; +} + +static void bce_pm_channel_resume(struct t2bce_device *bce) +{ + /* Reopen mailbox traffic after bridgeOS completed the wake handshake. */ + bce->mailbox_channel_active = true; + bce_mailbox_channel_resume(&bce->mbox); +} + +static int bce_pm_suspend_prepare(struct t2bce_device *bce) +{ + int status; + + status = bce_pm_channel_pause(bce); + if (status) + return status; + + /* The XHCI PM doorbell is part of the bridgeOS sleep ordering. */ + bce_xhci_pm_stop(&bce->xhci_pm); + return 0; +} + +static void bce_pm_suspend_abort(struct t2bce_device *bce) +{ + /* Failed suspend must unwind the wrapper order back to the running state. */ + bce_xhci_pm_start(&bce->xhci_pm, false); + bce_pm_channel_resume(bce); +} + +static void bce_pm_resume_finish(struct t2bce_device *bce) +{ + /* Match the bridgeOS wake ordering before clients observe resume complete. */ + bce_xhci_pm_start(&bce->xhci_pm, false); + bce_pm_channel_resume(bce); +} + +static int bce_register_command_queue(struct t2bce_device *bce, struct bce_queue_memcfg *cfg, int is_sq) +{ + int status; + int cmd_type; + u64 result; + /* The firmware accepts a TO_DEVICE mapping for this command payload. */ + dma_addr_t a = dma_map_single(&bce->pci->dev, cfg, sizeof(struct bce_queue_memcfg), DMA_TO_DEVICE); + if (dma_mapping_error(&bce->pci->dev, a)) + return -ENOMEM; + cmd_type = is_sq ? BCE_MB_REGISTER_COMMAND_SQ : BCE_MB_REGISTER_COMMAND_CQ; + status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(cmd_type, a), &result); + dma_unmap_single(&bce->pci->dev, a, sizeof(struct bce_queue_memcfg), DMA_TO_DEVICE); + if (status) + return status; + if (BCE_MB_TYPE(result) != BCE_MB_REGISTER_COMMAND_QUEUE_REPLY) + return -EINVAL; + return 0; +} + +static int t2bce_dma_register_queue(void *userdata, struct bce_queue_memcfg *cfg, + const char *name, bool isdirout) +{ + struct t2bce_device *bce = userdata; + + return t2bce_dma_cmd_register_queue(bce->dma.cmd_cmdq, cfg, name, isdirout); +} + +static int t2bce_dma_unregister_queue(void *userdata, u16 qid) +{ + struct t2bce_device *bce = userdata; + + return t2bce_dma_cmd_unregister_memory_queue(bce->dma.cmd_cmdq, qid); +} + +static int t2bce_dma_flush_queue(void *userdata, u16 qid) +{ + struct t2bce_device *bce = userdata; + + return t2bce_dma_cmd_flush_memory_queue(bce->dma.cmd_cmdq, qid); +} + +static void t2bce_remove(struct pci_dev *dev) +{ + struct t2bce_device *bce = pci_get_drvdata(dev); + bce->is_being_removed = true; + bce->dma.is_being_removed = true; + global_bce = NULL; + + bce_free_state_buffer(bce); + bce_xhci_pm_stop(&bce->xhci_pm); +#ifndef WITHOUT_NVME_PATCH + pci_disable_device(bce->pci0); +#endif + pci_dev_put(bce->pci0); + pci_free_irq(dev, 0, dev); + pci_free_irq(dev, 4, dev); + bce_free_command_queues(bce); + pci_iounmap(dev, bce->reg_mem_mb); + pci_iounmap(dev, bce->reg_mem_dma); + device_destroy(bce_class, bce->devt); + pci_free_irq_vectors(dev); + pci_release_regions(dev); + pci_disable_device(dev); + cleanup_srcu_struct(&bce->clients_srcu); + kfree(bce); +} + +static void t2bce_shutdown(struct pci_dev *dev) +{ + struct t2bce_device *bce = pci_get_drvdata(dev); + int status = 0; + + if (!bce) + return; + + mutex_lock(&bce->pm_lock); + bce->is_being_removed = true; + bce->dma.is_being_removed = true; + bce->stateful_suspend_valid = false; + bce->no_state_fallback = false; + bce->no_state_resume = false; + t2bce_core_clients_pm_reset(bce); + + /* + * Do not tear down allocations here; + * just leave the T2 side quiet while command queues and mailbox access are + * still valid: USB HCD off, XHCI PM sentinel written, mailbox drained. + */ + t2bce_core_clients_shutdown(bce); + bce_xhci_pm_stop(&bce->xhci_pm); + + if (bce->mailbox_channel_active) { + status = bce_pm_channel_pause(bce); + if (status) + pr_warn("t2bce_core: shutdown mailbox quiesce failed: %d\n", status); + } + + mutex_unlock(&bce->pm_lock); +} + +static int bce_pm_suspend_fallback_no_state(struct t2bce_device *bce) +{ + pr_debug("t2bce_core: suspend: forcing SLEEP_NO_STATE (no reply expected)\n"); + if (bce_mailbox_send_no_reply_locked(&bce->mbox, BCE_MB_MSG(BCE_MB_SLEEP_NO_STATE, 0))) { + pr_err("t2bce_core: suspend: SLEEP_NO_STATE send failed\n"); + return -EIO; + } + return 0; +} + +static int bce_pm_suspend_try_state(struct t2bce_device *bce) +{ + int status; + u64 resp; + + /* Try the stateful bridgeOS path first; a reject falls back to no-state. */ + bce->stateful_suspend_valid = false; + + if (!bce->saved_data_dma_ptr || !bce->saved_data_dma_addr || !bce->saved_data_dma_size) { + pr_err("t2bce_core: suspend failed (persistent state buffer missing)\n"); + return -ENOMEM; + } + + BUG_ON((bce->saved_data_dma_addr % 4096) != 0); + status = bce_mailbox_send_locked(&bce->mbox, + BCE_MB_MSG(BCE_MB_SAVE_STATE_AND_SLEEP, + (bce->saved_data_dma_addr & ~(4096LLU - 1)) | (bce->saved_data_dma_size / 4096)), + &resp); + if (status) { + pr_err("t2bce_core: suspend failed (mailbox send)\n"); + return status; + } + + if (BCE_MB_TYPE(resp) == BCE_MB_SAVE_RESTORE_STATE_COMPLETE) { + pr_debug("t2bce_core: suspend: remote response: restore state saved\n"); + bce->stateful_suspend_valid = true; + return 0; + } + + if (BCE_MB_TYPE(resp) == BCE_MB_SAVE_STATE_AND_SLEEP_REJECTED) { + pr_err("t2bce_core: remote rejected stateful suspend payload\n"); + return -EAGAIN; + } + return -EINVAL; +} + +static int bce_pm_resume_no_state(struct t2bce_device *bce) +{ + int status; + u64 resp; + + if ((status = bce_mailbox_send_locked(&bce->mbox, BCE_MB_MSG(BCE_MB_RESTORE_NO_STATE, 0), &resp))) { + pr_err("t2bce_core: resume with no state failed (mailbox send)\n"); + return status; + } + if (BCE_MB_TYPE(resp) != BCE_MB_RESTORE_NO_STATE) { + pr_err("t2bce_core: resume with no state failed (invalid device response)\n"); + return -EINVAL; + } + return 0; +} + +static int bce_pm_resume_stateful(struct t2bce_device *bce) +{ + int status; + u64 resp; + + if ((status = bce_mailbox_send_locked(&bce->mbox, BCE_MB_MSG(BCE_MB_RESTORE_STATE_AND_WAKE, + (bce->saved_data_dma_addr & ~(4096LLU - 1)) | (bce->saved_data_dma_size / 4096)), &resp))) { + pr_err("t2bce_core: resume with state failed (mailbox send)\n"); + return status; + } + if (BCE_MB_TYPE(resp) != BCE_MB_SAVE_RESTORE_STATE_COMPLETE) { + pr_err("t2bce_core: resume with state failed (invalid device response)\n"); + return -EINVAL; + } + + return 0; +} + +static int t2bce_suspend(struct device *dev) +{ + struct t2bce_device *bce = pci_get_drvdata(to_pci_dev(dev)); + int status; + + pr_debug("t2bce_core: suspend: entry\n"); + mutex_lock(&bce->pm_lock); + + bce->stateful_suspend_valid = false; + bce->no_state_fallback = false; + bce->no_state_resume = false; + t2bce_core_clients_pm_reset(bce); + t2bce_core_clients_pm_prepare(bce); + + status = bce_pm_suspend_prepare(bce); + if (status) + goto out_unlock; + + if (!bce_stateful_supported(bce)) { + /* No-state resume tears down and rebuilds clients that cannot preserve firmware state. */ + t2bce_core_clients_pm_prepare_no_state(bce); + status = bce_pm_suspend_fallback_no_state(bce); + if (!status) { + bce->no_state_fallback = true; + bce->no_state_resume = true; + t2bce_core_clients_pm_mark_no_state_resume(bce); + } else { + bce_pm_suspend_abort(bce); + } + goto out_unlock; + } + + status = bce_pm_suspend_try_state(bce); + if (!status) + goto out_unlock; + + if (status != -EAGAIN) { + bce_pm_suspend_abort(bce); + goto out_unlock; + } + + /* bridgeOS can reject state save; fall back to the no-state path then. */ + pr_debug("t2bce_core: suspend: stateful path not ready, falling back to no-state\n"); + t2bce_core_clients_pm_prepare_no_state(bce); + status = bce_pm_suspend_fallback_no_state(bce); + if (!status) { + bce->no_state_fallback = true; + bce->no_state_resume = true; + t2bce_core_clients_pm_mark_no_state_resume(bce); + } else { + bce_pm_suspend_abort(bce); + } + +out_unlock: + mutex_unlock(&bce->pm_lock); + pr_info("t2bce_core: suspend: exit status=%d stateful_valid=%d no_state_resume=%d no_state_fallback=%d\n", + status, bce->stateful_suspend_valid, bce->no_state_resume, bce->no_state_fallback); + return status; +} + +static int t2bce_resume(struct device *dev) +{ + struct t2bce_device *bce = pci_get_drvdata(to_pci_dev(dev)); + int status; + bool used_stateful; + + pr_debug("t2bce_core: resume: entry\n"); + mutex_lock(&bce->pm_lock); + + pci_set_master(bce->pci); + pci_set_master(bce->pci0); + + /* Resume follows the suspend result, not a preselected policy. */ + used_stateful = bce_stateful_supported(bce) && bce->stateful_suspend_valid; + pr_debug("t2bce_core: resume path: %s\n", used_stateful ? "stateful" : "no-state"); + if (used_stateful) + status = bce_pm_resume_stateful(bce); + else + status = bce_pm_resume_no_state(bce); + if (status) + goto out_unlock; + + if (used_stateful) + bce->stateful_suspend_valid = false; + + bce_pm_resume_finish(bce); + +out_unlock: + mutex_unlock(&bce->pm_lock); + pr_info("t2bce_core: resume: exit status=%d path=%s stateful_valid=%d no_state_resume=%d no_state_fallback=%d\n", + status, used_stateful ? "stateful" : "no-state", + bce->stateful_suspend_valid, bce->no_state_resume, bce->no_state_fallback); + return status; +} + +static int t2bce_prepare(struct device *dev) +{ + /* + * Force PCI PM to run our real suspend/resume callbacks instead of + * short-cutting an "already runtime suspended" device through complete(). + * The T2 mailbox save/restore handshake lives in those callbacks. + */ + return 0; +} + +static void t2bce_complete(struct device *dev) +{ + struct t2bce_device *bce = pci_get_drvdata(to_pci_dev(dev)); + + pr_debug("t2bce_core: complete: entry no_state_fallback=%d no_state_resume=%d\n", + bce->no_state_fallback, bce->no_state_resume); + if (bce->no_state_fallback && t2bce_core_clients_pm_has_no_state_resume(bce)) { + t2bce_core_clients_pm_complete(bce); + bce->no_state_fallback = false; + } + + t2bce_core_notify_resume_complete(bce); + bce->no_state_resume = false; + pr_debug("t2bce_core: complete: exit\n"); +} + +static struct pci_device_id t2bce_ids[ ] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1801) }, + { 0, }, +}; + +MODULE_DEVICE_TABLE(pci, t2bce_ids); + +struct dev_pm_ops t2bce_pci_driver_pm = { + .prepare = t2bce_prepare, + .suspend = t2bce_suspend, + .resume = t2bce_resume, + .complete = t2bce_complete +}; +struct pci_driver t2bce_pci_driver = { + .name = "t2bce_core", + .id_table = t2bce_ids, + .probe = t2bce_probe, + .remove = t2bce_remove, + .shutdown = t2bce_shutdown, + .driver = { + .pm = &t2bce_pci_driver_pm + } +}; + + +static int __init t2bce_module_init(void) +{ + int result; + + if ((result = alloc_chrdev_region(&bce_chrdev, 0, 1, "t2bce_core"))) + goto fail_chrdev; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0) + bce_class = class_create(THIS_MODULE, "t2bce_core"); +#else + bce_class = class_create("t2bce_core"); +#endif + if (IS_ERR(bce_class)) { + result = PTR_ERR(bce_class); + goto fail_class; + } + result = pci_register_driver(&t2bce_pci_driver); + if (result) + goto fail_drv; + + pr_info("t2bce_core: module initialized\n"); + return 0; + +fail_drv: + pci_unregister_driver(&t2bce_pci_driver); +fail_class: + class_destroy(bce_class); +fail_chrdev: + unregister_chrdev_region(bce_chrdev, 1); + if (!result) + result = -EINVAL; + pr_info("t2bce_core: module init failed status=%d\n", result); + return result; +} +static void __exit t2bce_module_exit(void) +{ + pci_unregister_driver(&t2bce_pci_driver); + + class_destroy(bce_class); + unregister_chrdev_region(bce_chrdev, 1); + pr_info("t2bce_core: module exited\n"); +} + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("André Eikmeyer "); +MODULE_DESCRIPTION("T2 BCE core driver"); +MODULE_VERSION("0.06"); +MODULE_SOFTDEP("post: t2bce_vhci"); +/* + * Legacy compat alias for existing installs. + */ +MODULE_ALIAS("apple-bce"); +module_init(t2bce_module_init); +module_exit(t2bce_module_exit); diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_core/transport.c b/drivers/staging/t2bce/t2bce_core/transport.c --- a/drivers/staging/t2bce/t2bce_core/transport.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_core/transport.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,406 @@ +#include "t2bce_core_transport.h" +#include "t2bce.h" + +#include +#include +#include +#include +#include + +struct t2bce_core_client { + struct t2bce_device *bce; + struct device *dev; + struct device_link *link; + struct list_head list; + t2bce_core_client_resume_callback resume_complete; + void *resume_complete_userdata; + struct t2bce_core_client_pm_ops pm_ops; + void *pm_userdata; +}; + +struct t2bce_sq_ctx { + t2bce_core_sq_completion completion; + void *userdata; +}; + +static struct bce_queue_cq *to_bce_cq(struct t2bce_core_queue_cq *cq) +{ + return (struct bce_queue_cq *) cq; +} + +static struct t2bce_core_queue_cq *to_t2bce_cq(struct bce_queue_cq *cq) +{ + return (struct t2bce_core_queue_cq *) cq; +} + +static struct bce_queue_sq *to_bce_sq(struct t2bce_core_queue_sq *sq) +{ + return (struct bce_queue_sq *) sq; +} + +static struct t2bce_core_queue_sq *to_t2bce_sq(struct bce_queue_sq *sq) +{ + return (struct t2bce_core_queue_sq *) sq; +} + +static void t2bce_sq_completion_adapter(struct bce_queue_sq *sq) +{ + struct t2bce_sq_ctx *ctx = sq->userdata; + + ctx->completion(to_t2bce_sq(sq)); +} + +struct t2bce_core_client *t2bce_core_client_get(struct device *dev) +{ + struct t2bce_device *bce = global_bce; + struct t2bce_core_client *client; + + if (!bce) + return ERR_PTR(-EPROBE_DEFER); + + if (bce->is_being_removed) + return ERR_PTR(-ENODEV); + + client = kzalloc(sizeof(*client), GFP_KERNEL); + if (!client) + return ERR_PTR(-ENOMEM); + + client->bce = bce; + client->dev = dev; + client->link = device_link_add(dev, &bce->pci->dev, + DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_CONSUMER); + if (!client->link) { + kfree(client); + return ERR_PTR(-ENODEV); + } + + mutex_lock(&bce->clients_lock); + list_add_tail_rcu(&client->list, &bce->clients); + mutex_unlock(&bce->clients_lock); + + return client; +} +EXPORT_SYMBOL_GPL(t2bce_core_client_get); + +void t2bce_core_client_put(struct t2bce_core_client *client) +{ + if (!client) + return; + + mutex_lock(&client->bce->clients_lock); + list_del_rcu(&client->list); + mutex_unlock(&client->bce->clients_lock); + + synchronize_srcu(&client->bce->clients_srcu); + + kfree(client); +} +EXPORT_SYMBOL_GPL(t2bce_core_client_put); + +struct device *t2bce_core_device_get(void) +{ + struct t2bce_device *bce = global_bce; + + if (!bce) + return ERR_PTR(-EPROBE_DEFER); + + if (bce->is_being_removed || !bce->dev) + return ERR_PTR(-ENODEV); + + return get_device(bce->dev); +} +EXPORT_SYMBOL_GPL(t2bce_core_device_get); + +void t2bce_core_device_put(struct device *dev) +{ + if (dev) + put_device(dev); +} +EXPORT_SYMBOL_GPL(t2bce_core_device_put); + +struct device *t2bce_core_client_dma_dev(struct t2bce_core_client *client) +{ + return &client->bce->pci->dev; +} +EXPORT_SYMBOL_GPL(t2bce_core_client_dma_dev); + +bool t2bce_core_client_no_state_resume(struct t2bce_core_client *client) +{ + return client->bce->no_state_resume; +} +EXPORT_SYMBOL_GPL(t2bce_core_client_no_state_resume); + +void t2bce_core_client_set_resume_complete_callback(struct t2bce_core_client *client, + t2bce_core_client_resume_callback callback, void *userdata) +{ + if (!callback) { + smp_store_release(&client->resume_complete, NULL); + WRITE_ONCE(client->resume_complete_userdata, NULL); + return; + } + + WRITE_ONCE(client->resume_complete_userdata, userdata); + smp_store_release(&client->resume_complete, callback); +} +EXPORT_SYMBOL_GPL(t2bce_core_client_set_resume_complete_callback); + +void t2bce_core_client_set_pm_ops(struct t2bce_core_client *client, + const struct t2bce_core_client_pm_ops *ops, void *userdata) +{ + if (!ops) { + memset(&client->pm_ops, 0, sizeof(client->pm_ops)); + WRITE_ONCE(client->pm_userdata, NULL); + return; + } + + WRITE_ONCE(client->pm_userdata, userdata); + client->pm_ops = *ops; +} +EXPORT_SYMBOL_GPL(t2bce_core_client_set_pm_ops); + +void t2bce_core_clients_shutdown(struct t2bce_device *bce) +{ + struct t2bce_core_client *client; + int srcu_idx; + + srcu_idx = srcu_read_lock(&bce->clients_srcu); + list_for_each_entry_srcu(client, &bce->clients, list, + srcu_read_lock_held(&bce->clients_srcu)) { + if (client->pm_ops.shutdown) + client->pm_ops.shutdown(READ_ONCE(client->pm_userdata)); + } + srcu_read_unlock(&bce->clients_srcu, srcu_idx); +} + +void t2bce_core_clients_pm_reset(struct t2bce_device *bce) +{ + struct t2bce_core_client *client; + int srcu_idx; + + srcu_idx = srcu_read_lock(&bce->clients_srcu); + list_for_each_entry_srcu(client, &bce->clients, list, + srcu_read_lock_held(&bce->clients_srcu)) { + if (client->pm_ops.pm_reset) + client->pm_ops.pm_reset(READ_ONCE(client->pm_userdata)); + } + srcu_read_unlock(&bce->clients_srcu, srcu_idx); +} + +void t2bce_core_clients_pm_prepare(struct t2bce_device *bce) +{ + struct t2bce_core_client *client; + int srcu_idx; + + srcu_idx = srcu_read_lock(&bce->clients_srcu); + list_for_each_entry_srcu(client, &bce->clients, list, + srcu_read_lock_held(&bce->clients_srcu)) { + if (client->pm_ops.pm_prepare) + client->pm_ops.pm_prepare(READ_ONCE(client->pm_userdata)); + } + srcu_read_unlock(&bce->clients_srcu, srcu_idx); +} + +void t2bce_core_clients_pm_prepare_no_state(struct t2bce_device *bce) +{ + struct t2bce_core_client *client; + int srcu_idx; + + srcu_idx = srcu_read_lock(&bce->clients_srcu); + list_for_each_entry_srcu(client, &bce->clients, list, + srcu_read_lock_held(&bce->clients_srcu)) { + if (client->pm_ops.pm_prepare_no_state) + client->pm_ops.pm_prepare_no_state(READ_ONCE(client->pm_userdata)); + } + srcu_read_unlock(&bce->clients_srcu, srcu_idx); +} + +void t2bce_core_clients_pm_mark_no_state_resume(struct t2bce_device *bce) +{ + struct t2bce_core_client *client; + int srcu_idx; + + srcu_idx = srcu_read_lock(&bce->clients_srcu); + list_for_each_entry_srcu(client, &bce->clients, list, + srcu_read_lock_held(&bce->clients_srcu)) { + if (client->pm_ops.pm_mark_no_state_resume) + client->pm_ops.pm_mark_no_state_resume(READ_ONCE(client->pm_userdata)); + } + srcu_read_unlock(&bce->clients_srcu, srcu_idx); +} + +bool t2bce_core_clients_pm_has_no_state_resume(struct t2bce_device *bce) +{ + struct t2bce_core_client *client; + bool ret = false; + int srcu_idx; + + srcu_idx = srcu_read_lock(&bce->clients_srcu); + list_for_each_entry_srcu(client, &bce->clients, list, + srcu_read_lock_held(&bce->clients_srcu)) { + if (client->pm_ops.pm_is_no_state_resume && + client->pm_ops.pm_is_no_state_resume(READ_ONCE(client->pm_userdata))) { + ret = true; + break; + } + } + srcu_read_unlock(&bce->clients_srcu, srcu_idx); + return ret; +} + +void t2bce_core_clients_pm_complete(struct t2bce_device *bce) +{ + struct t2bce_core_client *client; + int srcu_idx; + + srcu_idx = srcu_read_lock(&bce->clients_srcu); + list_for_each_entry_srcu(client, &bce->clients, list, + srcu_read_lock_held(&bce->clients_srcu)) { + if (client->pm_ops.pm_complete) + client->pm_ops.pm_complete(READ_ONCE(client->pm_userdata)); + } + srcu_read_unlock(&bce->clients_srcu, srcu_idx); +} + +void t2bce_core_notify_resume_complete(struct t2bce_device *bce) +{ + struct t2bce_core_client *client; + int srcu_idx; + + srcu_idx = srcu_read_lock(&bce->clients_srcu); + list_for_each_entry_srcu(client, &bce->clients, list, + srcu_read_lock_held(&bce->clients_srcu)) { + t2bce_core_client_resume_callback callback = smp_load_acquire(&client->resume_complete); + + if (callback) + callback(READ_ONCE(client->resume_complete_userdata)); + } + srcu_read_unlock(&bce->clients_srcu, srcu_idx); +} + +struct t2bce_core_queue_cq *t2bce_core_create_cq(struct t2bce_core_client *client, u32 el_count) +{ + return to_t2bce_cq(t2bce_dma_create_cq(&client->bce->dma, el_count)); +} +EXPORT_SYMBOL_GPL(t2bce_core_create_cq); + +void t2bce_core_destroy_cq(struct t2bce_core_client *client, struct t2bce_core_queue_cq *cq) +{ + t2bce_dma_destroy_cq(&client->bce->dma, to_bce_cq(cq)); +} +EXPORT_SYMBOL_GPL(t2bce_core_destroy_cq); + +struct t2bce_core_queue_sq *t2bce_core_create_sq(struct t2bce_core_client *client, struct t2bce_core_queue_cq *cq, + const char *name, u32 el_count, enum dma_data_direction direction, + t2bce_core_sq_completion compl, void *userdata) +{ + struct t2bce_sq_ctx *ctx; + struct bce_queue_sq *sq; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return NULL; + + ctx->completion = compl; + ctx->userdata = userdata; + + sq = t2bce_dma_create_sq(&client->bce->dma, to_bce_cq(cq), name, el_count, direction, + t2bce_sq_completion_adapter, ctx); + if (!sq) { + kfree(ctx); + return NULL; + } + + return to_t2bce_sq(sq); +} +EXPORT_SYMBOL_GPL(t2bce_core_create_sq); + +void t2bce_core_destroy_sq(struct t2bce_core_client *client, struct t2bce_core_queue_sq *sq) +{ + struct bce_queue_sq *bce_sq = to_bce_sq(sq); + struct t2bce_sq_ctx *ctx = bce_sq->userdata; + + t2bce_dma_destroy_sq(&client->bce->dma, bce_sq); + kfree(ctx); +} +EXPORT_SYMBOL_GPL(t2bce_core_destroy_sq); + +void *t2bce_core_queue_sq_userdata(struct t2bce_core_queue_sq *sq) +{ + struct t2bce_sq_ctx *ctx = to_bce_sq(sq)->userdata; + + return ctx->userdata; +} +EXPORT_SYMBOL_GPL(t2bce_core_queue_sq_userdata); + +int t2bce_core_reserve_submission(struct t2bce_core_queue_sq *sq, unsigned long *timeout) +{ + return t2bce_dma_reserve_submission(to_bce_sq(sq), timeout); +} +EXPORT_SYMBOL_GPL(t2bce_core_reserve_submission); + +void t2bce_core_cancel_submission_reservation(struct t2bce_core_queue_sq *sq) +{ + t2bce_dma_cancel_submission_reservation(to_bce_sq(sq)); +} +EXPORT_SYMBOL_GPL(t2bce_core_cancel_submission_reservation); + +void t2bce_core_set_next_submission_single(struct t2bce_core_queue_sq *sq, dma_addr_t addr, size_t size) +{ + t2bce_dma_set_next_submission_single(to_bce_sq(sq), addr, size); +} +EXPORT_SYMBOL_GPL(t2bce_core_set_next_submission_single); + +void t2bce_core_set_next_submission_segment_list(struct t2bce_core_queue_sq *sq, + dma_addr_t segl_addr, size_t segl_size) +{ + t2bce_dma_set_next_submission_segment_list(to_bce_sq(sq), segl_addr, segl_size); +} +EXPORT_SYMBOL_GPL(t2bce_core_set_next_submission_segment_list); + +void t2bce_core_submit_to_device(struct t2bce_core_queue_sq *sq) +{ + t2bce_dma_submit_to_device(to_bce_sq(sq)); +} +EXPORT_SYMBOL_GPL(t2bce_core_submit_to_device); + +void t2bce_core_notify_submission_complete(struct t2bce_core_queue_sq *sq) +{ + t2bce_dma_notify_submission_complete(to_bce_sq(sq)); +} +EXPORT_SYMBOL_GPL(t2bce_core_notify_submission_complete); + +struct t2bce_core_sq_completion_data *t2bce_core_next_completion(struct t2bce_core_queue_sq *sq) +{ + return (struct t2bce_core_sq_completion_data *) t2bce_dma_next_completion(to_bce_sq(sq)); +} +EXPORT_SYMBOL_GPL(t2bce_core_next_completion); + +u32 t2bce_core_queue_sq_head(struct t2bce_core_queue_sq *sq) +{ + return to_bce_sq(sq)->head; +} +EXPORT_SYMBOL_GPL(t2bce_core_queue_sq_head); + +u32 t2bce_core_queue_sq_tail(struct t2bce_core_queue_sq *sq) +{ + return to_bce_sq(sq)->tail; +} +EXPORT_SYMBOL_GPL(t2bce_core_queue_sq_tail); + +u32 t2bce_core_queue_sq_available(struct t2bce_core_queue_sq *sq) +{ + return atomic_read(&to_bce_sq(sq)->available_commands); +} +EXPORT_SYMBOL_GPL(t2bce_core_queue_sq_available); + +u32 t2bce_core_queue_sq_capacity(struct t2bce_core_queue_sq *sq) +{ + return to_bce_sq(sq)->el_count; +} +EXPORT_SYMBOL_GPL(t2bce_core_queue_sq_capacity); + +int t2bce_core_flush_queue(struct t2bce_core_client *client, struct t2bce_core_queue_sq *sq) +{ + return t2bce_dma_flush_sq(&client->bce->dma, to_bce_sq(sq)); +} +EXPORT_SYMBOL_GPL(t2bce_core_flush_queue); diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_dma/Makefile b/drivers/staging/t2bce/t2bce_dma/Makefile --- a/drivers/staging/t2bce/t2bce_dma/Makefile 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_dma/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_T2BCE_DMA) += t2bce_dma.o +t2bce_dma-y := queue.o + +ccflags-y += -DWITHOUT_NVME_PATCH +ccflags-y += -I$(src)/include diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_dma/include/t2bce_dma_queue.h b/drivers/staging/t2bce/t2bce_dma/include/t2bce_dma_queue.h --- a/drivers/staging/t2bce/t2bce_dma/include/t2bce_dma_queue.h 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_dma/include/t2bce_dma_queue.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,182 @@ +#ifndef T2BCE_DMA_QUEUE_H +#define T2BCE_DMA_QUEUE_H + +#include +#include + +#define BCE_CMD_SIZE 0x40 +#define BCE_MAX_QUEUE_COUNT 0x100 + +#define BCE_QUEUE_USER_MIN 2 +#define BCE_QUEUE_USER_MAX (BCE_MAX_QUEUE_COUNT - 1) + +struct t2bce_dma_segment { + u64 addr; + u64 length; +}; + +struct bce_queue_memcfg; + +struct t2bce_dma_engine_ops { + int (*register_queue)(void *userdata, struct bce_queue_memcfg *cfg, + const char *name, bool isdirout); + int (*unregister_queue)(void *userdata, u16 qid); + int (*flush_queue)(void *userdata, u16 qid); +}; + +enum bce_queue_type { + BCE_QUEUE_CQ, BCE_QUEUE_SQ +}; +struct bce_queue { + int qid; + int type; +}; +struct bce_queue_cq { + int qid; + int type; + u32 el_count; + dma_addr_t dma_handle; + void *data; + + u32 index; +}; +struct bce_queue_sq; +typedef void (*bce_sq_completion)(struct bce_queue_sq *q); +struct bce_sq_completion_data { + u32 status; + u64 data_size; + u64 result; +}; +struct bce_queue_sq { + int qid; + int type; + u32 el_size; + u32 el_count; + dma_addr_t dma_handle; + void *data; + void *userdata; + void __iomem *reg_mem_dma; + + atomic_t available_commands; + struct completion available_command_completion; + atomic_t available_command_completion_waiting_count; + u32 head, tail; + + u32 completion_cidx, completion_tail; + struct bce_sq_completion_data *completion_data; + bool has_pending_completions; + bce_sq_completion completion; +}; + +struct bce_queue_cmdq_result_el { + struct completion cmpl; + u32 status; + u64 result; + u32 slot; + u32 generation; +}; +struct bce_queue_cmdq { + struct bce_queue_sq *sq; + struct spinlock lck; + struct bce_queue_cmdq_result_el **tres; + u32 *slot_gen; +}; + +struct t2bce_dma_engine { + struct device *dma_dev; + const struct t2bce_dma_engine_ops *ops; + void *ops_userdata; + void __iomem *reg_mem_dma; + struct bce_queue *queues[BCE_MAX_QUEUE_COUNT]; + struct spinlock queues_lock; + struct ida queue_ida; + struct bce_queue_cq *cmd_cq; + struct bce_queue_cmdq *cmd_cmdq; + struct bce_queue_sq *int_sq_list[BCE_MAX_QUEUE_COUNT]; + bool is_being_removed; +}; + +struct bce_queue_memcfg { + u16 qid; + u16 el_count; + u16 vector_or_cq; + u16 _pad; + u64 addr; + u64 length; +}; + +enum bce_qe_completion_status { + BCE_COMPLETION_SUCCESS = 0, + BCE_COMPLETION_ERROR = 1, + BCE_COMPLETION_ABORTED = 2, + BCE_COMPLETION_NO_SPACE = 3, + BCE_COMPLETION_OVERRUN = 4 +}; +enum bce_qe_completion_flags { + BCE_COMPLETION_FLAG_PENDING = 0x8000 +}; +struct bce_qe_completion { + u64 result; + u64 data_size; + u16 qid; + u16 completion_index; + u16 status; // bce_qe_completion_status + u16 flags; // bce_qe_completion_flags +}; + +static __always_inline void *t2bce_dma_sq_element(struct bce_queue_sq *q, int i) { + return (void *) ((u8 *) q->data + q->el_size * i); +} +static __always_inline void *t2bce_dma_cq_element(struct bce_queue_cq *q, int i) { + return (void *) ((struct bce_qe_completion *) q->data + i); +} + +static __always_inline struct bce_sq_completion_data *t2bce_dma_next_completion(struct bce_queue_sq *sq) { + struct bce_sq_completion_data *res; + rmb(); + if (sq->completion_cidx == sq->completion_tail) + return NULL; + res = &sq->completion_data[sq->completion_cidx]; + sq->completion_cidx = (sq->completion_cidx + 1) % sq->el_count; + return res; +} + +struct bce_queue_cq *t2bce_dma_alloc_cq(struct t2bce_dma_engine *dma, int qid, u32 el_count); +void t2bce_dma_get_cq_memcfg(struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg); +void t2bce_dma_free_cq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq); +void t2bce_dma_handle_cq_completions_locked(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, size_t *ce); +void t2bce_dma_dispatch_pending_sq_completions(struct t2bce_dma_engine *dma, size_t ce); + +struct bce_queue_sq *t2bce_dma_alloc_sq(struct t2bce_dma_engine *dma, int qid, u32 el_size, u32 el_count, + bce_sq_completion compl, void *userdata); +void t2bce_dma_get_sq_memcfg(struct bce_queue_sq *sq, struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg); +void t2bce_dma_free_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq); +int t2bce_dma_reserve_submission(struct bce_queue_sq *sq, unsigned long *timeout); +void t2bce_dma_cancel_submission_reservation(struct bce_queue_sq *sq); +void t2bce_dma_submit_to_device(struct bce_queue_sq *sq); +void t2bce_dma_notify_submission_complete(struct bce_queue_sq *sq); + +void t2bce_dma_set_next_submission_single(struct bce_queue_sq *sq, dma_addr_t addr, size_t size); +void t2bce_dma_set_next_submission_segment_list(struct bce_queue_sq *sq, + dma_addr_t segl_addr, size_t segl_size); + +struct bce_queue_cmdq *t2bce_dma_alloc_cmdq(struct t2bce_dma_engine *dma, int qid, u32 el_count); +void t2bce_dma_free_cmdq(struct t2bce_dma_engine *dma, struct bce_queue_cmdq *cmdq); + +u32 t2bce_dma_cmd_register_queue(struct bce_queue_cmdq *cmdq, struct bce_queue_memcfg *cfg, const char *name, bool isdirout); +u32 t2bce_dma_cmd_unregister_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid); +u32 t2bce_dma_cmd_flush_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid); + + +/* User API - Creates and registers the queue */ + +struct bce_queue_cq *t2bce_dma_create_cq(struct t2bce_dma_engine *dma, u32 el_count); +struct bce_queue_sq *t2bce_dma_create_sq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, const char *name, u32 el_count, + int direction, bce_sq_completion compl, void *userdata); +struct bce_queue_sq *t2bce_dma_create_sq_with_flags(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, const char *name, + u32 el_count, u16 flags, bce_sq_completion compl, void *userdata); +void t2bce_dma_destroy_cq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq); +void t2bce_dma_destroy_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq); +int t2bce_dma_flush_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq); + +#endif diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_dma/queue.c b/drivers/staging/t2bce/t2bce_dma/queue.c --- a/drivers/staging/t2bce/t2bce_dma/queue.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_dma/queue.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,625 @@ +#include "t2bce_dma_queue.h" +#include +#include +#include +#include +#include +#include + +#define REG_DOORBELL_BASE 0x44000 + +struct bce_qe_submission { + u64 length; + u64 addr; + + u64 segl_addr; + u64 segl_length; +}; + +enum bce_submission_type { + BCE_SUBMISSION_SINGLE, + BCE_SUBMISSION_SEGMENT_LIST, +}; + +struct bce_submission { + enum bce_submission_type type; + union { + struct { + dma_addr_t addr; + size_t size; + } single; + struct { + dma_addr_t addr; + size_t size; + } segment_list; + }; +}; + +enum bce_cmdq_command { + BCE_CMD_REGISTER_MEMORY_QUEUE = 0x20, + BCE_CMD_UNREGISTER_MEMORY_QUEUE = 0x30, + BCE_CMD_FLUSH_MEMORY_QUEUE = 0x40, + BCE_CMD_SET_MEMORY_QUEUE_PROPERTY = 0x50 +}; +struct bce_cmdq_simple_memory_queue_cmd { + u16 cmd; // bce_cmdq_command + u16 flags; + u16 qid; +}; +struct bce_cmdq_register_memory_queue_cmd { + u16 cmd; // bce_cmdq_command + u16 flags; + u16 qid; + u16 _pad; + u16 el_count; + u16 vector_or_cq; + u16 _pad2; + u16 name_len; + char name[0x20]; + u64 addr; + u64 length; +}; + +struct bce_queue_cq *t2bce_dma_alloc_cq(struct t2bce_dma_engine *dma, int qid, u32 el_count) +{ + struct bce_queue_cq *q; + q = kzalloc(sizeof(struct bce_queue_cq), GFP_KERNEL); + if (!q) + return NULL; + q->qid = qid; + q->type = BCE_QUEUE_CQ; + q->el_count = el_count; + q->data = dma_alloc_coherent(dma->dma_dev, el_count * sizeof(struct bce_qe_completion), + &q->dma_handle, GFP_KERNEL); + if (!q->data) { + pr_err("DMA queue memory alloc failed\n"); + kfree(q); + return NULL; + } + return q; +} +EXPORT_SYMBOL_GPL(t2bce_dma_alloc_cq); + +void t2bce_dma_get_cq_memcfg(struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg) +{ + cfg->qid = (u16) cq->qid; + cfg->el_count = (u16) cq->el_count; + cfg->vector_or_cq = 0; + cfg->_pad = 0; + cfg->addr = cq->dma_handle; + cfg->length = cq->el_count * sizeof(struct bce_qe_completion); +} +EXPORT_SYMBOL_GPL(t2bce_dma_get_cq_memcfg); + +void t2bce_dma_free_cq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq) +{ + dma_free_coherent(dma->dma_dev, cq->el_count * sizeof(struct bce_qe_completion), cq->data, cq->dma_handle); + kfree(cq); +} +EXPORT_SYMBOL_GPL(t2bce_dma_free_cq); + +static void t2bce_dma_handle_cq_completion(struct t2bce_dma_engine *dma, struct bce_qe_completion *e, size_t *ce) +{ + struct bce_queue *target; + struct bce_queue_sq *target_sq; + struct bce_sq_completion_data *cmpl; + if (e->qid >= BCE_MAX_QUEUE_COUNT) { + pr_err("Device sent a response for qid (%u) >= BCE_MAX_QUEUE_COUNT\n", e->qid); + return; + } + target = dma->queues[e->qid]; + if (!target || target->type != BCE_QUEUE_SQ) { + pr_err("Device sent a response for qid (%u), which does not exist\n", e->qid); + return; + } + target_sq = (struct bce_queue_sq *) target; + if (target_sq->completion_tail != e->completion_index) { + pr_err("Completion index mismatch; this is likely going to make this driver unusable\n"); + return; + } + if (!target_sq->has_pending_completions) { + target_sq->has_pending_completions = true; + dma->int_sq_list[(*ce)++] = target_sq; + } + cmpl = &target_sq->completion_data[e->completion_index]; + cmpl->status = e->status; + cmpl->data_size = e->data_size; + cmpl->result = e->result; + wmb(); + target_sq->completion_tail = (target_sq->completion_tail + 1) % target_sq->el_count; +} + +void t2bce_dma_handle_cq_completions_locked(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, size_t *ce) +{ + struct bce_qe_completion *e; + e = t2bce_dma_cq_element(cq, cq->index); + if (!(e->flags & BCE_COMPLETION_FLAG_PENDING)) + return; + mb(); + while (true) { + e = t2bce_dma_cq_element(cq, cq->index); + if (!(e->flags & BCE_COMPLETION_FLAG_PENDING)) + break; + pr_debug("t2bce_dma: compl: %i: %i %llx %llx", e->qid, e->status, e->data_size, e->result); + t2bce_dma_handle_cq_completion(dma, e, ce); + e->flags = 0; + cq->index = (cq->index + 1) % cq->el_count; + } + mb(); + iowrite32(cq->index, (u32 *) ((u8 *) dma->reg_mem_dma + REG_DOORBELL_BASE) + cq->qid); +} +EXPORT_SYMBOL_GPL(t2bce_dma_handle_cq_completions_locked); + +void t2bce_dma_dispatch_pending_sq_completions(struct t2bce_dma_engine *dma, size_t ce) +{ + struct bce_queue_sq *sq; + + while (ce) { + --ce; + sq = dma->int_sq_list[ce]; + sq->completion(sq); + sq->has_pending_completions = false; + } +} +EXPORT_SYMBOL_GPL(t2bce_dma_dispatch_pending_sq_completions); + + +struct bce_queue_sq *t2bce_dma_alloc_sq(struct t2bce_dma_engine *dma, int qid, u32 el_size, u32 el_count, + bce_sq_completion compl, void *userdata) +{ + struct bce_queue_sq *q; + q = kzalloc(sizeof(struct bce_queue_sq), GFP_KERNEL); + if (!q) + return NULL; + q->qid = qid; + q->type = BCE_QUEUE_SQ; + q->el_size = el_size; + q->el_count = el_count; + q->data = dma_alloc_coherent(dma->dma_dev, el_count * el_size, + &q->dma_handle, GFP_KERNEL); + q->completion = compl; + q->userdata = userdata; + q->completion_data = kzalloc(sizeof(struct bce_sq_completion_data) * el_count, GFP_KERNEL); + q->reg_mem_dma = dma->reg_mem_dma; + atomic_set(&q->available_commands, el_count - 1); + init_completion(&q->available_command_completion); + atomic_set(&q->available_command_completion_waiting_count, 0); + if (!q->data || !q->completion_data) { + pr_err("DMA queue memory alloc failed\n"); + if (q->data) + dma_free_coherent(dma->dma_dev, el_count * el_size, q->data, q->dma_handle); + kfree(q->completion_data); + kfree(q); + return NULL; + } + return q; +} +EXPORT_SYMBOL_GPL(t2bce_dma_alloc_sq); + +void t2bce_dma_get_sq_memcfg(struct bce_queue_sq *sq, struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg) +{ + cfg->qid = (u16) sq->qid; + cfg->el_count = (u16) sq->el_count; + cfg->vector_or_cq = (u16) cq->qid; + cfg->_pad = 0; + cfg->addr = sq->dma_handle; + cfg->length = sq->el_count * sq->el_size; +} +EXPORT_SYMBOL_GPL(t2bce_dma_get_sq_memcfg); + +void t2bce_dma_free_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq) +{ + dma_free_coherent(dma->dma_dev, sq->el_count * sq->el_size, sq->data, sq->dma_handle); + kfree(sq->completion_data); + kfree(sq); +} +EXPORT_SYMBOL_GPL(t2bce_dma_free_sq); + +int t2bce_dma_reserve_submission(struct bce_queue_sq *sq, unsigned long *timeout) +{ + while (atomic_dec_if_positive(&sq->available_commands) < 0) { + if (!timeout || !*timeout) + return -EAGAIN; + atomic_inc(&sq->available_command_completion_waiting_count); + *timeout = wait_for_completion_timeout(&sq->available_command_completion, *timeout); + if (!*timeout) { + if (atomic_dec_if_positive(&sq->available_command_completion_waiting_count) < 0) + try_wait_for_completion(&sq->available_command_completion); /* consume the pending completion */ + } + } + return 0; +} +EXPORT_SYMBOL_GPL(t2bce_dma_reserve_submission); + +void t2bce_dma_cancel_submission_reservation(struct bce_queue_sq *sq) +{ + atomic_inc(&sq->available_commands); +} +EXPORT_SYMBOL_GPL(t2bce_dma_cancel_submission_reservation); + +static void *t2bce_dma_next_submission(struct bce_queue_sq *sq) +{ + void *ret = t2bce_dma_sq_element(sq, sq->tail); + sq->tail = (sq->tail + 1) % sq->el_count; + return ret; +} + +void t2bce_dma_submit_to_device(struct bce_queue_sq *sq) +{ + mb(); + iowrite32(sq->tail, (u32 *) ((u8 *) sq->reg_mem_dma + REG_DOORBELL_BASE) + sq->qid); +} +EXPORT_SYMBOL_GPL(t2bce_dma_submit_to_device); + +void t2bce_dma_notify_submission_complete(struct bce_queue_sq *sq) +{ + sq->head = (sq->head + 1) % sq->el_count; + atomic_inc(&sq->available_commands); + if (atomic_dec_if_positive(&sq->available_command_completion_waiting_count) >= 0) { + complete(&sq->available_command_completion); + } +} +EXPORT_SYMBOL_GPL(t2bce_dma_notify_submission_complete); + +static void t2bce_dma_write_submission(struct bce_qe_submission *element, const struct bce_submission *submission) +{ + switch (submission->type) { + case BCE_SUBMISSION_SINGLE: + element->addr = submission->single.addr; + element->length = submission->single.size; + element->segl_addr = 0; + element->segl_length = 0; + break; + case BCE_SUBMISSION_SEGMENT_LIST: + element->addr = 0; + element->length = 0; + element->segl_addr = submission->segment_list.addr; + element->segl_length = submission->segment_list.size; + break; + } +} + +void t2bce_dma_set_next_submission_single(struct bce_queue_sq *sq, dma_addr_t addr, size_t size) +{ + struct bce_submission submission = { + .type = BCE_SUBMISSION_SINGLE, + .single = { + .addr = addr, + .size = size, + }, + }; + + t2bce_dma_write_submission(t2bce_dma_next_submission(sq), &submission); +} +EXPORT_SYMBOL_GPL(t2bce_dma_set_next_submission_single); + +void t2bce_dma_set_next_submission_segment_list(struct bce_queue_sq *sq, + dma_addr_t segl_addr, size_t segl_size) +{ + struct bce_submission submission = { + .type = BCE_SUBMISSION_SEGMENT_LIST, + .segment_list = { + .addr = segl_addr, + .size = segl_size, + }, + }; + + t2bce_dma_write_submission(t2bce_dma_next_submission(sq), &submission); +} +EXPORT_SYMBOL_GPL(t2bce_dma_set_next_submission_segment_list); + +static void t2bce_dma_cmdq_completion(struct bce_queue_sq *q); + +struct bce_queue_cmdq *t2bce_dma_alloc_cmdq(struct t2bce_dma_engine *dma, int qid, u32 el_count) +{ + struct bce_queue_cmdq *q; + q = kzalloc(sizeof(struct bce_queue_cmdq), GFP_KERNEL); + if (!q) + return NULL; + q->sq = t2bce_dma_alloc_sq(dma, qid, BCE_CMD_SIZE, el_count, t2bce_dma_cmdq_completion, q); + if (!q->sq) { + kfree(q); + return NULL; + } + spin_lock_init(&q->lck); + q->tres = kzalloc(sizeof(struct bce_queue_cmdq_result_el*) * el_count, GFP_KERNEL); + if (!q->tres) { + t2bce_dma_free_sq(dma, q->sq); + kfree(q); + return NULL; + } + q->slot_gen = kzalloc(sizeof(u32) * el_count, GFP_KERNEL); + if (!q->slot_gen) { + kfree(q->tres); + t2bce_dma_free_sq(dma, q->sq); + kfree(q); + return NULL; + } + return q; +} +EXPORT_SYMBOL_GPL(t2bce_dma_alloc_cmdq); + +void t2bce_dma_free_cmdq(struct t2bce_dma_engine *dma, struct bce_queue_cmdq *cmdq) +{ + t2bce_dma_free_sq(dma, cmdq->sq); + kfree(cmdq->slot_gen); + kfree(cmdq->tres); + kfree(cmdq); +} +EXPORT_SYMBOL_GPL(t2bce_dma_free_cmdq); + +void t2bce_dma_cmdq_completion(struct bce_queue_sq *q) +{ + struct bce_queue_cmdq_result_el *el; + struct bce_queue_cmdq *cmdq = q->userdata; + struct bce_sq_completion_data *result; + + spin_lock(&cmdq->lck); + while ((result = t2bce_dma_next_completion(q))) { + el = cmdq->tres[cmdq->sq->head]; + if (el && el->generation == cmdq->slot_gen[cmdq->sq->head]) { + el->result = result->result; + el->status = result->status; + mb(); + complete(&el->cmpl); + } + cmdq->tres[cmdq->sq->head] = NULL; + t2bce_dma_notify_submission_complete(q); + } + spin_unlock(&cmdq->lck); +} + +static __always_inline void *t2bce_dma_cmd_start(struct bce_queue_cmdq *cmdq, struct bce_queue_cmdq_result_el *res) +{ + void *ret; + unsigned long timeout; + init_completion(&res->cmpl); + mb(); + + timeout = msecs_to_jiffies(1000L * 60 * 5); /* wait for up to ~5 minutes */ + if (t2bce_dma_reserve_submission(cmdq->sq, &timeout)) + return NULL; + + spin_lock(&cmdq->lck); + res->slot = cmdq->sq->tail; + res->generation = cmdq->slot_gen[cmdq->sq->tail]; + cmdq->tres[cmdq->sq->tail] = res; + ret = t2bce_dma_next_submission(cmdq->sq); + return ret; +} + +static __always_inline int t2bce_dma_cmd_finish(struct bce_queue_cmdq *cmdq, struct bce_queue_cmdq_result_el *res) +{ + t2bce_dma_submit_to_device(cmdq->sq); + spin_unlock(&cmdq->lck); + + if (!wait_for_completion_timeout(&res->cmpl, msecs_to_jiffies(5000))) { + pr_err("t2bce_dma: command queue timeout (slot %u)\n", res->slot); + spin_lock(&cmdq->lck); + cmdq->tres[res->slot] = NULL; + cmdq->slot_gen[res->slot]++; + spin_unlock(&cmdq->lck); + t2bce_dma_notify_submission_complete(cmdq->sq); + return -ETIMEDOUT; + } + mb(); + return 0; +} + +u32 t2bce_dma_cmd_register_queue(struct bce_queue_cmdq *cmdq, struct bce_queue_memcfg *cfg, const char *name, bool isdirout) +{ + struct bce_queue_cmdq_result_el res; + struct bce_cmdq_register_memory_queue_cmd *cmd = t2bce_dma_cmd_start(cmdq, &res); + if (!cmd) + return (u32) -1; + cmd->cmd = BCE_CMD_REGISTER_MEMORY_QUEUE; + cmd->flags = (u16) ((name ? 2 : 0) | (isdirout ? 1 : 0)); + cmd->qid = cfg->qid; + cmd->el_count = cfg->el_count; + cmd->vector_or_cq = cfg->vector_or_cq; + memset(cmd->name, 0, sizeof(cmd->name)); + if (name) { + cmd->name_len = (u16) min(strlen(name), (size_t) sizeof(cmd->name)); + memcpy(cmd->name, name, cmd->name_len); + } else { + cmd->name_len = 0; + } + cmd->addr = cfg->addr; + cmd->length = cfg->length; + + if (t2bce_dma_cmd_finish(cmdq, &res)) + return (u32)-1; + return res.status; +} +EXPORT_SYMBOL_GPL(t2bce_dma_cmd_register_queue); + +u32 t2bce_dma_cmd_unregister_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid) +{ + struct bce_queue_cmdq_result_el res; + struct bce_cmdq_simple_memory_queue_cmd *cmd = t2bce_dma_cmd_start(cmdq, &res); + if (!cmd) + return (u32) -1; + cmd->cmd = BCE_CMD_UNREGISTER_MEMORY_QUEUE; + cmd->flags = 0; + cmd->qid = qid; + if (t2bce_dma_cmd_finish(cmdq, &res)) + return (u32)-1; + return res.status; +} +EXPORT_SYMBOL_GPL(t2bce_dma_cmd_unregister_memory_queue); + +u32 t2bce_dma_cmd_flush_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid) +{ + struct bce_queue_cmdq_result_el res; + struct bce_cmdq_simple_memory_queue_cmd *cmd = t2bce_dma_cmd_start(cmdq, &res); + if (!cmd) + return (u32) -1; + cmd->cmd = BCE_CMD_FLUSH_MEMORY_QUEUE; + cmd->flags = 0; + cmd->qid = qid; + if (t2bce_dma_cmd_finish(cmdq, &res)) + return (u32)-1; + return res.status; +} +EXPORT_SYMBOL_GPL(t2bce_dma_cmd_flush_memory_queue); + +static int t2bce_dma_register_queue(struct t2bce_dma_engine *dma, struct bce_queue_memcfg *cfg, + const char *name, bool isdirout) +{ + if (!dma->ops || !dma->ops->register_queue) + return -ENODEV; + + return dma->ops->register_queue(dma->ops_userdata, cfg, name, isdirout); +} + +static int t2bce_dma_unregister_queue(struct t2bce_dma_engine *dma, u16 qid) +{ + if (!dma->ops || !dma->ops->unregister_queue) + return -ENODEV; + + return dma->ops->unregister_queue(dma->ops_userdata, qid); +} + +static int t2bce_dma_flush_queue(struct t2bce_dma_engine *dma, u16 qid) +{ + if (!dma->ops || !dma->ops->flush_queue) + return -ENODEV; + + return dma->ops->flush_queue(dma->ops_userdata, qid); +} + + +struct bce_queue_cq *t2bce_dma_create_cq(struct t2bce_dma_engine *dma, u32 el_count) +{ + struct bce_queue_cq *cq; + struct bce_queue_memcfg cfg; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0) + int qid = ida_simple_get(&dma->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX, GFP_KERNEL); +#else + int qid = ida_alloc_range(&dma->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX - 1, GFP_KERNEL); +#endif + if (qid < 0) + return NULL; + cq = t2bce_dma_alloc_cq(dma, qid, el_count); + if (!cq) + return NULL; + t2bce_dma_get_cq_memcfg(cq, &cfg); + if (t2bce_dma_register_queue(dma, &cfg, NULL, false) != 0) { + pr_err("t2bce_dma: CQ registration failed (%i)", qid); + t2bce_dma_free_cq(dma, cq); +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0) + ida_simple_remove(&dma->queue_ida, (uint) qid); +#else + ida_free(&dma->queue_ida, (uint) qid); +#endif + return NULL; + } + dma->queues[qid] = (struct bce_queue *) cq; + return cq; +} +EXPORT_SYMBOL_GPL(t2bce_dma_create_cq); + +struct bce_queue_sq *t2bce_dma_create_sq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, const char *name, u32 el_count, + int direction, bce_sq_completion compl, void *userdata) +{ + struct bce_queue_sq *sq; + struct bce_queue_memcfg cfg; + int qid; + if (cq == NULL) + return NULL; /* cq can not be null */ + if (name == NULL) + return NULL; /* name can not be null */ + if (direction != DMA_TO_DEVICE && direction != DMA_FROM_DEVICE) + return NULL; /* unsupported direction */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0) + qid = ida_simple_get(&dma->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX, GFP_KERNEL); +#else + qid = ida_alloc_range(&dma->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX - 1, GFP_KERNEL); +#endif + if (qid < 0) + return NULL; + sq = t2bce_dma_alloc_sq(dma, qid, sizeof(struct bce_qe_submission), el_count, compl, userdata); + if (!sq) + return NULL; + t2bce_dma_get_sq_memcfg(sq, cq, &cfg); + if (t2bce_dma_register_queue(dma, &cfg, name, direction != DMA_FROM_DEVICE) != 0) { + pr_err("t2bce_dma: SQ registration failed (%i)", qid); + t2bce_dma_free_sq(dma, sq); +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0) + ida_simple_remove(&dma->queue_ida, (uint) qid); +#else + ida_free(&dma->queue_ida, (uint) qid); +#endif + return NULL; + } + spin_lock(&dma->queues_lock); + dma->queues[qid] = (struct bce_queue *) sq; + spin_unlock(&dma->queues_lock); + return sq; +} +EXPORT_SYMBOL_GPL(t2bce_dma_create_sq); + +struct bce_queue_sq *t2bce_dma_create_sq_with_flags(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, const char *name, + u32 el_count, u16 flags, bce_sq_completion compl, void *userdata) +{ + int direction = (flags & 1) ? DMA_TO_DEVICE : DMA_FROM_DEVICE; + return t2bce_dma_create_sq(dma, cq, name, el_count, direction, compl, userdata); +} +EXPORT_SYMBOL_GPL(t2bce_dma_create_sq_with_flags); + +void t2bce_dma_destroy_cq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq) +{ + if (!dma->is_being_removed && t2bce_dma_unregister_queue(dma, (u16) cq->qid)) + pr_err("t2bce_dma: CQ unregister failed"); + spin_lock(&dma->queues_lock); + dma->queues[cq->qid] = NULL; + spin_unlock(&dma->queues_lock); +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0) + ida_simple_remove(&dma->queue_ida, (uint) cq->qid); +#else + ida_free(&dma->queue_ida, (uint) cq->qid); +#endif + t2bce_dma_free_cq(dma, cq); +} +EXPORT_SYMBOL_GPL(t2bce_dma_destroy_cq); + +void t2bce_dma_destroy_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq) +{ + if (!dma->is_being_removed && t2bce_dma_unregister_queue(dma, (u16) sq->qid)) + pr_err("t2bce_dma: SQ unregister failed"); + spin_lock(&dma->queues_lock); + dma->queues[sq->qid] = NULL; + spin_unlock(&dma->queues_lock); +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0) + ida_simple_remove(&dma->queue_ida, (uint) sq->qid); +#else + ida_free(&dma->queue_ida, (uint) sq->qid); +#endif + t2bce_dma_free_sq(dma, sq); +} +EXPORT_SYMBOL_GPL(t2bce_dma_destroy_sq); + +int t2bce_dma_flush_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq) +{ + return t2bce_dma_flush_queue(dma, (u16) sq->qid); +} +EXPORT_SYMBOL_GPL(t2bce_dma_flush_sq); + +static int __init t2bce_dma_module_init(void) +{ + pr_info("t2bce_dma: module initialized\n"); + return 0; +} + +static void __exit t2bce_dma_module_exit(void) +{ + pr_info("t2bce_dma: module exited\n"); +} + +module_init(t2bce_dma_module_init); +module_exit(t2bce_dma_module_exit); + +MODULE_AUTHOR("André Eikmeyer "); +MODULE_DESCRIPTION("Apple T2 BCE DMA queue engine"); +MODULE_VERSION("0.01"); +MODULE_LICENSE("GPL"); diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_vhci/Makefile b/drivers/staging/t2bce/t2bce_vhci/Makefile --- a/drivers/staging/t2bce/t2bce_vhci/Makefile 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_vhci/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_T2BCE_VHCI) += t2bce_vhci.o +t2bce_vhci-y := vhci.o queue.o transfer.o + +ccflags-y += -DWITHOUT_NVME_PATCH +ccflags-y += -I$(src) +ccflags-y += -I$(src)/../t2bce_core/include diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_vhci/command.h b/drivers/staging/t2bce/t2bce_vhci/command.h --- a/drivers/staging/t2bce/t2bce_vhci/command.h 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_vhci/command.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,206 @@ +#ifndef BCE_VHCI_COMMAND_H +#define BCE_VHCI_COMMAND_H + +#include "queue.h" +#include +#include + +#define BCE_VHCI_CMD_TIMEOUT_SHORT msecs_to_jiffies(2000) +#define BCE_VHCI_CMD_TIMEOUT_LONG msecs_to_jiffies(30000) + +#define BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2 2 +#define BCE_VHCI_BULK_MAX_ACTIVE_URBS (1 << BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2) + +typedef u8 bce_vhci_port_t; +typedef u8 bce_vhci_device_t; + +enum bce_vhci_command { + BCE_VHCI_CMD_CONTROLLER_ENABLE = 1, + BCE_VHCI_CMD_CONTROLLER_DISABLE = 2, + BCE_VHCI_CMD_CONTROLLER_START = 3, + /* Reserved for future controller-level PM work; endpoint PM is used today. */ + BCE_VHCI_CMD_CONTROLLER_PAUSE = 4, + + BCE_VHCI_CMD_PORT_POWER_ON = 0x10, + BCE_VHCI_CMD_PORT_POWER_OFF = 0x11, + BCE_VHCI_CMD_PORT_RESUME = 0x12, + BCE_VHCI_CMD_PORT_SUSPEND = 0x13, + BCE_VHCI_CMD_PORT_RESET = 0x14, + BCE_VHCI_CMD_PORT_DISABLE = 0x15, + BCE_VHCI_CMD_PORT_STATUS = 0x16, + BCE_VHCI_EV_PORT_STATUS_CHANGE = 0x18, + + BCE_VHCI_CMD_DEVICE_CREATE = 0x30, + BCE_VHCI_CMD_DEVICE_DESTROY = 0x31, + + BCE_VHCI_CMD_ENDPOINT_CREATE = 0x40, + BCE_VHCI_CMD_ENDPOINT_DESTROY = 0x41, + BCE_VHCI_CMD_ENDPOINT_SET_STATE = 0x42, + BCE_VHCI_CMD_ENDPOINT_RESET = 0x44, + + /* Device to host only */ + BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE = 0x43, + BCE_VHCI_CMD_TRANSFER_REQUEST = 0x1000, + BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS = 0x1005 +}; + +enum bce_vhci_endpoint_state { + BCE_VHCI_ENDPOINT_ACTIVE = 0, + BCE_VHCI_ENDPOINT_PAUSED = 1, + BCE_VHCI_ENDPOINT_STALLED = 2 +}; + +static inline int bce_vhci_cmd_controller_enable(struct bce_vhci_command_queue *q, u8 busNum, u16 *portMask) +{ + int status; + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_ENABLE; + cmd.param1 = 0x7100u | busNum; + status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); + if (!status) + *portMask = (u16) res.param2; + return status; +} +static inline int bce_vhci_cmd_controller_disable(struct bce_vhci_command_queue *q) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_DISABLE; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} +static inline int bce_vhci_cmd_controller_start(struct bce_vhci_command_queue *q) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_START; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} +static inline int bce_vhci_cmd_controller_pause(struct bce_vhci_command_queue *q) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_PAUSE; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} + +static inline int bce_vhci_cmd_port_power_on(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_POWER_ON; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_port_power_off(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_POWER_OFF; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_port_resume(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_RESUME; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} +static inline int bce_vhci_cmd_port_suspend(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_SUSPEND; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} +static inline int bce_vhci_cmd_port_reset(struct bce_vhci_command_queue *q, bce_vhci_port_t port, u32 timeout) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_RESET; + cmd.param1 = port; + cmd.param2 = timeout; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_port_disable(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_DISABLE; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_port_status(struct bce_vhci_command_queue *q, bce_vhci_port_t port, + u32 clearFlags, u32 *resStatus) +{ + int status; + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_STATUS; + cmd.param1 = port; + cmd.param2 = clearFlags & 0x560000; + status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); + if (status >= 0) + *resStatus = (u32) res.param2; + return status; +} + +static inline int bce_vhci_cmd_device_create(struct bce_vhci_command_queue *q, bce_vhci_port_t port, + bce_vhci_device_t *dev) +{ + int status; + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_DEVICE_CREATE; + cmd.param1 = port; + status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); + if (!status) + *dev = (bce_vhci_device_t) res.param2; + return status; +} +static inline int bce_vhci_cmd_device_destroy(struct bce_vhci_command_queue *q, bce_vhci_device_t dev) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_DEVICE_DESTROY; + cmd.param1 = dev; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} + +static inline int bce_vhci_cmd_endpoint_create(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, + struct usb_endpoint_descriptor *desc) +{ + struct bce_vhci_message cmd, res; + int endpoint_type = usb_endpoint_type(desc); + int maxp = usb_endpoint_maxp(desc); + int maxp_burst = usb_endpoint_maxp_mult(desc) * maxp; + u8 max_active_requests_pow2 = 0; + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_CREATE; + cmd.param1 = dev | ((desc->bEndpointAddress & 0x8Fu) << 8); + if (endpoint_type == USB_ENDPOINT_XFER_BULK) + max_active_requests_pow2 = BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2; + cmd.param2 = endpoint_type | ((max_active_requests_pow2 & 0xf) << 4) | (maxp << 16) | ((u64) maxp_burst << 32); + if (endpoint_type == USB_ENDPOINT_XFER_INT) + cmd.param2 |= (desc->bInterval - 1) << 8; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_endpoint_destroy(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_DESTROY; + cmd.param1 = dev | (endpoint << 8); + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_endpoint_set_state(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint, + enum bce_vhci_endpoint_state newState, enum bce_vhci_endpoint_state *retState) +{ + int status; + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_SET_STATE; + cmd.param1 = dev | (endpoint << 8); + cmd.param2 = (u64) newState; + status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); + if (status != BCE_VHCI_INTERNAL_ERROR && status != BCE_VHCI_NO_POWER) + *retState = (enum bce_vhci_endpoint_state) res.param2; + return status; +} +static inline int bce_vhci_cmd_endpoint_reset(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_RESET; + cmd.param1 = dev | (endpoint << 8); + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} + + +#endif //BCE_VHCI_COMMAND_H diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_vhci/queue.c b/drivers/staging/t2bce/t2bce_vhci/queue.c --- a/drivers/staging/t2bce/t2bce_vhci/queue.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_vhci/queue.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,269 @@ +#include "queue.h" +#include "vhci.h" + +#include + + +static void bce_vhci_message_queue_completion(struct t2bce_core_queue_sq *sq); + +int bce_vhci_message_queue_create(struct bce_vhci *vhci, struct bce_vhci_message_queue *ret, const char *name) +{ + int status; + ret->cq = t2bce_core_create_cq(vhci->client, VHCI_EVENT_QUEUE_EL_COUNT); + if (!ret->cq) + return -EINVAL; + ret->sq = t2bce_core_create_sq(vhci->client, ret->cq, name, VHCI_EVENT_QUEUE_EL_COUNT, DMA_TO_DEVICE, + bce_vhci_message_queue_completion, ret); + if (!ret->sq) { + status = -EINVAL; + goto fail_cq; + } + ret->data = dma_alloc_coherent(t2bce_core_client_dma_dev(vhci->client), + sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, + &ret->dma_addr, GFP_KERNEL); + if (!ret->data) { + status = -EINVAL; + goto fail_sq; + } + return 0; + +fail_sq: + t2bce_core_destroy_sq(vhci->client, ret->sq); + ret->sq = NULL; +fail_cq: + t2bce_core_destroy_cq(vhci->client, ret->cq); + ret->cq = NULL; + return status; +} + +void bce_vhci_message_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_message_queue *q) +{ + if (!q->cq) + return; + dma_free_coherent(t2bce_core_client_dma_dev(vhci->client), + sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, + q->data, q->dma_addr); + t2bce_core_destroy_sq(vhci->client, q->sq); + t2bce_core_destroy_cq(vhci->client, q->cq); +} + +void bce_vhci_message_queue_write(struct bce_vhci_message_queue *q, struct bce_vhci_message *req) +{ + int sidx; + sidx = t2bce_core_queue_sq_tail(q->sq); + pr_debug("t2bce_vhci: Send message: %x s=%x p1=%x p2=%llx\n", req->cmd, req->status, req->param1, req->param2); + q->data[sidx] = *req; + t2bce_core_set_next_submission_single(q->sq, q->dma_addr + sizeof(struct bce_vhci_message) * sidx, + sizeof(struct bce_vhci_message)); + t2bce_core_submit_to_device(q->sq); +} + +static void bce_vhci_message_queue_completion(struct t2bce_core_queue_sq *sq) +{ + while (t2bce_core_next_completion(sq)) + t2bce_core_notify_submission_complete(sq); +} + + + +static void bce_vhci_event_queue_completion(struct t2bce_core_queue_sq *sq); + +int __bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, + t2bce_core_sq_completion compl) +{ + ret->vhci = vhci; + + ret->sq = t2bce_core_create_sq(vhci->client, vhci->ev_cq, name, VHCI_EVENT_QUEUE_EL_COUNT, DMA_FROM_DEVICE, compl, ret); + if (!ret->sq) + return -EINVAL; + ret->data = dma_alloc_coherent(t2bce_core_client_dma_dev(vhci->client), + sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, + &ret->dma_addr, GFP_KERNEL); + if (!ret->data) { + t2bce_core_destroy_sq(vhci->client, ret->sq); + ret->sq = NULL; + return -EINVAL; + } + + init_completion(&ret->queue_empty_completion); + bce_vhci_event_queue_submit_pending(ret, VHCI_EVENT_PENDING_COUNT); + return 0; +} + +int bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, + bce_vhci_event_queue_callback cb) +{ + ret->cb = cb; + return __bce_vhci_event_queue_create(vhci, ret, name, bce_vhci_event_queue_completion); +} + +void bce_vhci_event_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_event_queue *q) +{ + if (!q->sq) + return; + dma_free_coherent(t2bce_core_client_dma_dev(vhci->client), + sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, + q->data, q->dma_addr); + t2bce_core_destroy_sq(vhci->client, q->sq); +} + +static void bce_vhci_event_queue_completion(struct t2bce_core_queue_sq *sq) +{ + struct t2bce_core_sq_completion_data *cd; + struct bce_vhci_event_queue *ev = t2bce_core_queue_sq_userdata(sq); + struct bce_vhci_message *msg; + size_t cnt = 0; + + while ((cd = t2bce_core_next_completion(sq))) { + if (cd->status == T2BCE_COMPLETION_ABORTED) { + t2bce_core_notify_submission_complete(sq); + continue; + } + msg = &ev->data[t2bce_core_queue_sq_head(sq)]; + pr_debug("t2bce_vhci: Got event: %x s=%x p1=%x p2=%llx\n", msg->cmd, msg->status, msg->param1, msg->param2); + ev->cb(ev, msg); + + t2bce_core_notify_submission_complete(sq); + ++cnt; + } + bce_vhci_event_queue_submit_pending(ev, cnt); + if (t2bce_core_queue_sq_available(sq) == t2bce_core_queue_sq_capacity(sq) - 1) + complete(&ev->queue_empty_completion); +} + +void bce_vhci_event_queue_submit_pending(struct bce_vhci_event_queue *q, size_t count) +{ + int idx; + while (count--) { + if (t2bce_core_reserve_submission(q->sq, NULL)) { + pr_err("t2bce_vhci: Failed to reserve an event queue submission\n"); + break; + } + idx = t2bce_core_queue_sq_tail(q->sq); + t2bce_core_set_next_submission_single(q->sq, + q->dma_addr + idx * sizeof(struct bce_vhci_message), sizeof(struct bce_vhci_message)); + } + t2bce_core_submit_to_device(q->sq); +} + +void bce_vhci_event_queue_pause(struct bce_vhci_event_queue *q) +{ + unsigned long timeout; + reinit_completion(&q->queue_empty_completion); + if (t2bce_core_flush_queue(q->vhci->client, q->sq)) + pr_warn("t2bce_vhci: failed to flush event queue\n"); + timeout = msecs_to_jiffies(5000); + while (t2bce_core_queue_sq_available(q->sq) != t2bce_core_queue_sq_capacity(q->sq) - 1) { + timeout = wait_for_completion_timeout(&q->queue_empty_completion, timeout); + if (timeout == 0) { + pr_err("t2bce_vhci: waiting for queue to be flushed timed out\n"); + break; + } + } +} + +void bce_vhci_event_queue_resume(struct bce_vhci_event_queue *q) +{ + if (t2bce_core_queue_sq_available(q->sq) != t2bce_core_queue_sq_capacity(q->sq) - 1) { + pr_err("t2bce_vhci: resume of a queue with pending submissions\n"); + return; + } + bce_vhci_event_queue_submit_pending(q, VHCI_EVENT_PENDING_COUNT); +} + +void bce_vhci_command_queue_create(struct bce_vhci_command_queue *ret, struct bce_vhci_message_queue *mq) +{ + ret->mq = mq; + ret->completion.result = NULL; + init_completion(&ret->completion.completion); + spin_lock_init(&ret->completion_lock); + mutex_init(&ret->mutex); +} + +void bce_vhci_command_queue_destroy(struct bce_vhci_command_queue *cq) +{ + spin_lock(&cq->completion_lock); + if (cq->completion.result) { + memset(cq->completion.result, 0, sizeof(struct bce_vhci_message)); + cq->completion.result->status = BCE_VHCI_ABORT; + complete(&cq->completion.completion); + cq->completion.result = NULL; + } + spin_unlock(&cq->completion_lock); + mutex_lock(&cq->mutex); + mutex_unlock(&cq->mutex); + mutex_destroy(&cq->mutex); +} + +void bce_vhci_command_queue_deliver_completion(struct bce_vhci_command_queue *cq, struct bce_vhci_message *msg) +{ + struct bce_vhci_command_queue_completion *c = &cq->completion; + + spin_lock(&cq->completion_lock); + if (c->result) { + *c->result = *msg; + complete(&c->completion); + c->result = NULL; + } + spin_unlock(&cq->completion_lock); +} + +static int __bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req, + struct bce_vhci_message *res, unsigned long timeout) +{ + int status; + struct bce_vhci_command_queue_completion *c; + struct bce_vhci_message creq; + c = &cq->completion; + + if ((status = t2bce_core_reserve_submission(cq->mq->sq, &timeout))) + return status; + + spin_lock(&cq->completion_lock); + c->result = res; + reinit_completion(&c->completion); + spin_unlock(&cq->completion_lock); + + bce_vhci_message_queue_write(cq->mq, req); + + if (!wait_for_completion_timeout(&c->completion, timeout)) { + /* Ask bridgeOS to cancel the command before reporting timeout. */ + pr_debug("t2bce_vhci: command timed out req=%x\n", req->cmd); + if ((status = t2bce_core_reserve_submission(cq->mq->sq, &timeout))) + return status; + + creq = *req; + creq.cmd |= 0x4000; + bce_vhci_message_queue_write(cq->mq, &creq); + + if (!wait_for_completion_timeout(&c->completion, 1000)) { + pr_err("t2bce_vhci: Possible desync, cmd cancel timed out\n"); + + spin_lock(&cq->completion_lock); + c->result = NULL; + spin_unlock(&cq->completion_lock); + return -ETIMEDOUT; + } + if ((res->cmd & ~0x8000) == creq.cmd) + return -ETIMEDOUT; + /* The original reply arrived after the cancellation was queued. */ + } + + if ((res->cmd & ~0x8000) != req->cmd) { + pr_err("t2bce_vhci: Possible desync, cmd reply mismatch req=%x, res=%x\n", req->cmd, res->cmd); + return -EIO; + } + if (res->status == BCE_VHCI_SUCCESS) + return 0; + return res->status; +} + +int bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req, + struct bce_vhci_message *res, unsigned long timeout) +{ + int status; + mutex_lock(&cq->mutex); + status = __bce_vhci_command_queue_execute(cq, req, res, timeout); + mutex_unlock(&cq->mutex); + return status; +} diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_vhci/queue.h b/drivers/staging/t2bce/t2bce_vhci/queue.h --- a/drivers/staging/t2bce/t2bce_vhci/queue.h 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_vhci/queue.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,76 @@ +#ifndef BCE_VHCI_QUEUE_H +#define BCE_VHCI_QUEUE_H + +#include +#include "t2bce_core_transport.h" + +#define VHCI_EVENT_QUEUE_EL_COUNT 256 +#define VHCI_EVENT_PENDING_COUNT 32 + +struct bce_vhci; +struct bce_vhci_event_queue; + +enum bce_vhci_message_status { + BCE_VHCI_SUCCESS = 1, + BCE_VHCI_ERROR = 2, + BCE_VHCI_USB_PIPE_STALL = 3, + BCE_VHCI_ABORT = 4, + BCE_VHCI_BAD_ARGUMENT = 5, + BCE_VHCI_OVERRUN = 6, + BCE_VHCI_INTERNAL_ERROR = 7, + BCE_VHCI_NO_POWER = 8, + BCE_VHCI_UNSUPPORTED = 9 +}; +struct bce_vhci_message { + u16 cmd; + u16 status; // bce_vhci_message_status + u32 param1; + u64 param2; +}; + +struct bce_vhci_message_queue { + struct t2bce_core_queue_cq *cq; + struct t2bce_core_queue_sq *sq; + struct bce_vhci_message *data; + dma_addr_t dma_addr; +}; +typedef void (*bce_vhci_event_queue_callback)(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg); +struct bce_vhci_event_queue { + struct bce_vhci *vhci; + struct t2bce_core_queue_sq *sq; + struct bce_vhci_message *data; + dma_addr_t dma_addr; + bce_vhci_event_queue_callback cb; + struct completion queue_empty_completion; +}; +struct bce_vhci_command_queue_completion { + struct bce_vhci_message *result; + struct completion completion; +}; +struct bce_vhci_command_queue { + struct bce_vhci_message_queue *mq; + struct bce_vhci_command_queue_completion completion; + struct spinlock completion_lock; + struct mutex mutex; +}; + +int bce_vhci_message_queue_create(struct bce_vhci *vhci, struct bce_vhci_message_queue *ret, const char *name); +void bce_vhci_message_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_message_queue *q); +void bce_vhci_message_queue_write(struct bce_vhci_message_queue *q, struct bce_vhci_message *req); + +int __bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, + t2bce_core_sq_completion compl); +int bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, + bce_vhci_event_queue_callback cb); +void bce_vhci_event_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_event_queue *q); +void bce_vhci_event_queue_submit_pending(struct bce_vhci_event_queue *q, size_t count); +void bce_vhci_event_queue_pause(struct bce_vhci_event_queue *q); +void bce_vhci_event_queue_resume(struct bce_vhci_event_queue *q); + +void bce_vhci_command_queue_create(struct bce_vhci_command_queue *ret, struct bce_vhci_message_queue *mq); +void bce_vhci_command_queue_destroy(struct bce_vhci_command_queue *cq); +int bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req, + struct bce_vhci_message *res, unsigned long timeout); +void bce_vhci_command_queue_deliver_completion(struct bce_vhci_command_queue *cq, struct bce_vhci_message *msg); + +#endif //BCE_VHCI_QUEUE_H diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_vhci/transfer.c b/drivers/staging/t2bce/t2bce_vhci/transfer.c --- a/drivers/staging/t2bce/t2bce_vhci/transfer.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_vhci/transfer.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,1261 @@ +#include "transfer.h" +#include "vhci.h" +#include +#include +#include + +static void bce_vhci_transfer_queue_completion(struct t2bce_core_queue_sq *sq); +static void bce_vhci_transfer_queue_giveback(struct bce_vhci_transfer_queue *q); +static void bce_vhci_transfer_queue_remove_pending(struct bce_vhci_transfer_queue *q); + +static int bce_vhci_urb_init(struct bce_vhci_urb *vurb); +static int bce_vhci_urb_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg); +static int bce_vhci_urb_transfer_completion(struct bce_vhci_urb *urb, struct t2bce_core_sq_completion_data *c); +static void bce_vhci_urb_complete(struct bce_vhci_urb *urb, int status); +static void bce_vhci_urb_free(struct bce_vhci_urb *urb); + +static void bce_vhci_transfer_queue_reset_w(struct work_struct *work); +static void bce_vhci_transfer_queue_resume_w(struct work_struct *work); +static void bce_vhci_transfer_queue_cancel_w(struct work_struct *work); + +void bce_vhci_create_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q, + struct usb_host_endpoint *endp, bce_vhci_device_t dev_addr, enum dma_data_direction dir) +{ + char name[0x21]; + INIT_LIST_HEAD(&q->evq); + INIT_LIST_HEAD(&q->giveback_urb_list); + INIT_LIST_HEAD(&q->cancel_urb_list); + spin_lock_init(&q->urb_lock); + mutex_init(&q->pause_lock); + init_waitqueue_head(&q->sq_out_wait_queue); + atomic_set(&q->sq_out_pending, 0); + q->vhci = vhci; + q->endp = endp; + q->dev_addr = dev_addr; + q->endp_addr = (u8) (endp->desc.bEndpointAddress & 0x8F); + q->state = BCE_VHCI_ENDPOINT_ACTIVE; + q->active = true; + q->stalled = false; + q->paused_by = 0; + q->max_active_requests = 1; + if (usb_endpoint_type(&endp->desc) == USB_ENDPOINT_XFER_BULK) + q->max_active_requests = BCE_VHCI_BULK_MAX_ACTIVE_URBS; + q->remaining_active_requests = q->max_active_requests; + q->cq = t2bce_core_create_cq(vhci->client, 0x100); + INIT_WORK(&q->w_reset, bce_vhci_transfer_queue_reset_w); + INIT_WORK(&q->w_resume, bce_vhci_transfer_queue_resume_w); + INIT_WORK(&q->w_cancel, bce_vhci_transfer_queue_cancel_w); + q->sq_in = NULL; + if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) { + snprintf(name, sizeof(name), "VHC1-%i-%02x", dev_addr, 0x80 | usb_endpoint_num(&endp->desc)); + q->sq_in = t2bce_core_create_sq(vhci->client, q->cq, name, 0x100, DMA_FROM_DEVICE, + bce_vhci_transfer_queue_completion, q); + } + q->sq_out = NULL; + if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) { + snprintf(name, sizeof(name), "VHC1-%i-%02x", dev_addr, usb_endpoint_num(&endp->desc)); + q->sq_out = t2bce_core_create_sq(vhci->client, q->cq, name, 0x100, DMA_TO_DEVICE, + bce_vhci_transfer_queue_completion, q); + } +} + +void bce_vhci_destroy_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q) +{ + cancel_work_sync(&q->w_resume); + cancel_work_sync(&q->w_reset); + cancel_work_sync(&q->w_cancel); + bce_vhci_transfer_queue_giveback(q); + bce_vhci_transfer_queue_remove_pending(q); + if (q->sq_in) + t2bce_core_destroy_sq(vhci->client, q->sq_in); + if (q->sq_out) + t2bce_core_destroy_sq(vhci->client, q->sq_out); + t2bce_core_destroy_cq(vhci->client, q->cq); +} + +static inline bool bce_vhci_transfer_queue_can_init_urb(struct bce_vhci_transfer_queue *q) +{ + return q->remaining_active_requests > 0; +} + +static bool bce_vhci_transfer_queue_defer_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg) +{ + struct bce_vhci_list_message *lm; + lm = kmalloc(sizeof(struct bce_vhci_list_message), GFP_ATOMIC); + if (!lm) { + pr_warn("t2bce_vhci: [%02x] failed to defer transfer queue event cmd=%x status=%x\n", + q->endp_addr, msg->cmd, msg->status); + return false; + } + INIT_LIST_HEAD(&lm->list); + lm->msg = *msg; + list_add_tail(&lm->list, &q->evq); + return true; +} + +static void bce_vhci_transfer_queue_giveback(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + struct urb *urb; + spin_lock_irqsave(&q->urb_lock, flags); + while (!list_empty(&q->giveback_urb_list)) { + urb = list_first_entry(&q->giveback_urb_list, struct urb, urb_list); + list_del(&urb->urb_list); + + spin_unlock_irqrestore(&q->urb_lock, flags); + usb_hcd_giveback_urb(q->vhci->hcd, urb, urb->status); + spin_lock_irqsave(&q->urb_lock, flags); + } + spin_unlock_irqrestore(&q->urb_lock, flags); +} + +static void bce_vhci_transfer_queue_init_pending_urbs(struct bce_vhci_transfer_queue *q); + +static int bce_vhci_transfer_queue_port(struct bce_vhci_transfer_queue *q) +{ + int port; + + for (port = 1; port <= q->vhci->port_count && + port < ARRAY_SIZE(q->vhci->port_to_device); port++) { + if (q->vhci->port_to_device[port] == q->dev_addr) + return port; + } + + return 0; +} + +static bool bce_vhci_transfer_queue_is_ep0(struct bce_vhci_transfer_queue *q) +{ + return q->endp_addr == 0x00; +} + +static size_t bce_vhci_transfer_queue_event_count(struct bce_vhci_transfer_queue *q) +{ + struct bce_vhci_list_message *lm; + size_t count = 0; + + list_for_each_entry(lm, &q->evq, list) + count++; + + return count; +} + +static void bce_vhci_transfer_queue_log_event(struct bce_vhci_transfer_queue *q, + struct bce_vhci_message *msg, const char *stage) +{ + if (!bce_vhci_transfer_queue_is_ep0(q)) + return; + + pr_debug("t2bce_vhci: EP0 event %s dev=%u port=%d cmd=%x msg_status=%x p1=%x p2=%llx active=%u paused_by=%x stalled=%u qstate=%u remaining=%u evq=%zu urb_empty=%d\n", + stage, q->dev_addr, bce_vhci_transfer_queue_port(q), + msg->cmd, msg->status, msg->param1, msg->param2, + q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests, + bce_vhci_transfer_queue_event_count(q), + list_empty(&q->endp->urb_list)); +} + +static void bce_vhci_transfer_queue_log_completion(struct bce_vhci_transfer_queue *q, + struct t2bce_core_queue_sq *sq, struct t2bce_core_sq_completion_data *c, const char *stage) +{ + const char *sq_name = "unknown"; + + if (!bce_vhci_transfer_queue_is_ep0(q)) + return; + + if (sq == q->sq_in) + sq_name = "in"; + else if (sq == q->sq_out) + sq_name = "out"; + + pr_debug("t2bce_vhci: EP0 completion %s dev=%u port=%d sq=%s cstatus=%x data_size=%llu sq_out_pending=%d active=%u paused_by=%x stalled=%u qstate=%u remaining=%u evq=%zu urb_empty=%d\n", + stage, q->dev_addr, bce_vhci_transfer_queue_port(q), + sq_name, c->status, c->data_size, + atomic_read(&q->sq_out_pending), q->active, q->paused_by, + q->stalled, q->state, q->remaining_active_requests, + bce_vhci_transfer_queue_event_count(q), + list_empty(&q->endp->urb_list)); +} + +static void bce_vhci_urb_log_control(struct bce_vhci_urb *urb, const char *stage) +{ + struct bce_vhci_transfer_queue *q = urb->q; + struct usb_ctrlrequest *ctrl; + + if (!urb->is_control || !bce_vhci_transfer_queue_is_ep0(q)) + return; + + ctrl = (struct usb_ctrlrequest *) urb->urb->setup_packet; + if (!ctrl) { + pr_debug("t2bce_vhci: EP0 control %s dev=%u port=%d urb_state=%u no-setup xfer_len=%u actual=%u send=%u recv=%u status=%d active=%u paused_by=%x stalled=%u qstate=%u remaining=%u urb_reject=%d\n", + stage, q->dev_addr, bce_vhci_transfer_queue_port(q), + urb->state, urb->urb->transfer_buffer_length, + urb->urb->actual_length, urb->send_offset, + urb->receive_offset, urb->received_status, q->active, + q->paused_by, q->stalled, q->state, + q->remaining_active_requests, atomic_read(&urb->urb->reject)); + return; + } + + /* Per-stage EP0 control-transfer trace. + * Keep this at pr_debug; enable dynamic debug when diagnosing + * setup/data/status ordering or endpoint recovery issues. + */ + pr_debug("t2bce_vhci: EP0 control %s dev=%u port=%d urb_state=%u dir=%s reqtype=%02x req=%02x value=%04x index=%04x wlen=%u xfer_len=%u actual=%u send=%u recv=%u status=%d active=%u paused_by=%x stalled=%u qstate=%u remaining=%u urb_reject=%d\n", + stage, q->dev_addr, bce_vhci_transfer_queue_port(q), + urb->state, usb_urb_dir_in(urb->urb) ? "in" : "out", + ctrl->bRequestType, ctrl->bRequest, + le16_to_cpu(ctrl->wValue), le16_to_cpu(ctrl->wIndex), + le16_to_cpu(ctrl->wLength), urb->urb->transfer_buffer_length, + urb->urb->actual_length, urb->send_offset, urb->receive_offset, + urb->received_status, q->active, q->paused_by, q->stalled, + q->state, q->remaining_active_requests, + atomic_read(&urb->urb->reject)); +} + +static void bce_vhci_transfer_queue_deliver_pending(struct bce_vhci_transfer_queue *q) +{ + struct urb *urb; + struct bce_vhci_list_message *lm; + + while (!list_empty(&q->endp->urb_list) && !list_empty(&q->evq)) { + urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list); + + lm = list_first_entry(&q->evq, struct bce_vhci_list_message, list); + bce_vhci_transfer_queue_log_event(q, &lm->msg, "deferred-deliver"); + if (bce_vhci_urb_update(urb->hcpriv, &lm->msg) == -EAGAIN) { + bce_vhci_transfer_queue_log_event(q, &lm->msg, "deferred-still-waiting"); + break; + } + bce_vhci_transfer_queue_log_event(q, &lm->msg, "deferred-consumed"); + list_del(&lm->list); + kfree(lm); + } + + /* some of the URBs could have been completed, so initialize more URBs if possible */ + bce_vhci_transfer_queue_init_pending_urbs(q); +} + +static void bce_vhci_transfer_queue_remove_pending(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + struct bce_vhci_list_message *lm; + spin_lock_irqsave(&q->urb_lock, flags); + if (bce_vhci_transfer_queue_is_ep0(q) && !list_empty(&q->evq)) + pr_debug("t2bce_vhci: EP0 event remove-pending dev=%u port=%d evq=%zu active=%u paused_by=%x stalled=%u qstate=%u remaining=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), + bce_vhci_transfer_queue_event_count(q), q->active, + q->paused_by, q->stalled, q->state, + q->remaining_active_requests); + while (!list_empty(&q->evq)) { + lm = list_first_entry(&q->evq, struct bce_vhci_list_message, list); + list_del(&lm->list); + kfree(lm); + } + spin_unlock_irqrestore(&q->urb_lock, flags); +} + +void bce_vhci_transfer_queue_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg) +{ + unsigned long flags; + struct bce_vhci_urb *turb; + struct urb *urb; + spin_lock_irqsave(&q->urb_lock, flags); + bce_vhci_transfer_queue_log_event(q, msg, "incoming"); + /* Paused queues may still complete in-flight work but must not deliver new work. */ + if (!q->active) { + if (bce_vhci_transfer_queue_is_ep0(q) && + msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && + (q->paused_by & BCE_VHCI_PAUSE_INTERNAL_WQ)) { + bce_vhci_transfer_queue_log_event(q, msg, "defer-inactive-internal-pause"); + if (!bce_vhci_transfer_queue_defer_event(q, msg)) + bce_vhci_transfer_queue_log_event(q, msg, "defer-inactive-failed"); + } else { + bce_vhci_transfer_queue_log_event(q, msg, "drop-inactive"); + } + goto complete; + } + bce_vhci_transfer_queue_deliver_pending(q); + + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && + (!list_empty(&q->evq) || list_empty(&q->endp->urb_list))) { + bce_vhci_transfer_queue_log_event(q, msg, "defer-no-urb-or-backlog"); + if (!bce_vhci_transfer_queue_defer_event(q, msg)) + bce_vhci_transfer_queue_log_event(q, msg, "defer-no-urb-or-backlog-failed"); + goto complete; + } + if (list_empty(&q->endp->urb_list)) { + bce_vhci_transfer_queue_log_event(q, msg, "unexpected-empty"); + pr_err("t2bce_vhci: [%02x] Unexpected transfer queue event\n", q->endp_addr); + goto complete; + } + urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list); + turb = urb->hcpriv; + if (bce_vhci_urb_update(turb, msg) == -EAGAIN) { + bce_vhci_transfer_queue_log_event(q, msg, "defer-eagain"); + if (!bce_vhci_transfer_queue_defer_event(q, msg)) + bce_vhci_transfer_queue_log_event(q, msg, "defer-eagain-failed"); + } else { + bce_vhci_transfer_queue_log_event(q, msg, "consumed"); + bce_vhci_transfer_queue_init_pending_urbs(q); + } + +complete: + spin_unlock_irqrestore(&q->urb_lock, flags); + bce_vhci_transfer_queue_giveback(q); +} + +static void bce_vhci_transfer_queue_completion(struct t2bce_core_queue_sq *sq) +{ + unsigned long flags; + struct t2bce_core_sq_completion_data *c; + struct urb *urb; + struct bce_vhci_transfer_queue *q = t2bce_core_queue_sq_userdata(sq); + bool is_sq_out = (sq == q->sq_out); + spin_lock_irqsave(&q->urb_lock, flags); + while ((c = t2bce_core_next_completion(sq))) { + if (c->status == T2BCE_COMPLETION_ABORTED) { /* We flushed the queue */ + bce_vhci_transfer_queue_log_completion(q, sq, c, "aborted"); + pr_debug("t2bce_vhci: [%02x] Got an abort completion\n", q->endp_addr); + if (is_sq_out && atomic_dec_if_positive(&q->sq_out_pending) == 0) + wake_up(&q->sq_out_wait_queue); + t2bce_core_notify_submission_complete(sq); + continue; + } + if (list_empty(&q->endp->urb_list)) { + bce_vhci_transfer_queue_log_completion(q, sq, c, "empty"); + pr_err("t2bce_vhci: [%02x] Got a completion while no requests are pending\n", q->endp_addr); + if (is_sq_out && atomic_dec_if_positive(&q->sq_out_pending) == 0) + wake_up(&q->sq_out_wait_queue); + continue; + } + bce_vhci_transfer_queue_log_completion(q, sq, c, "incoming"); + pr_debug("t2bce_vhci: [%02x] Got a transfer queue completion\n", q->endp_addr); + urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list); + bce_vhci_urb_transfer_completion(urb->hcpriv, c); + if (is_sq_out && atomic_dec_if_positive(&q->sq_out_pending) == 0) + wake_up(&q->sq_out_wait_queue); + t2bce_core_notify_submission_complete(sq); + } + bce_vhci_transfer_queue_deliver_pending(q); + spin_unlock_irqrestore(&q->urb_lock, flags); + bce_vhci_transfer_queue_giveback(q); +} + +int bce_vhci_transfer_queue_do_pause(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + int status; + u8 endp_addr = (u8) (q->endp->desc.bEndpointAddress & 0x8F); + int pending; + long timeout; + spin_lock_irqsave(&q->urb_lock, flags); + q->active = false; + spin_unlock_irqrestore(&q->urb_lock, flags); + if (q->sq_out) { + /* Let pending OUT submissions drain before the endpoint is paused. */ + pending = atomic_read(&q->sq_out_pending); + if (pending > 0) { + timeout = wait_event_timeout(q->sq_out_wait_queue, + atomic_read(&q->sq_out_pending) == 0, + msecs_to_jiffies(2000)); + if (!timeout && atomic_read(&q->sq_out_pending) > 0) + pr_warn("t2bce_vhci: [%02x] pause timeout waiting for %d outputs\n", + q->endp_addr, atomic_read(&q->sq_out_pending)); + } + } + bce_vhci_transfer_queue_remove_pending(q); + if ((status = bce_vhci_cmd_endpoint_set_state( + &q->vhci->cq, q->dev_addr, endp_addr, BCE_VHCI_ENDPOINT_PAUSED, &q->state))) + return status; + if (q->state != BCE_VHCI_ENDPOINT_PAUSED) + return -EINVAL; + if (q->sq_in) + t2bce_core_flush_queue(q->vhci->client, q->sq_in); + if (q->sq_out) + t2bce_core_flush_queue(q->vhci->client, q->sq_out); + return 0; +} + +static void bce_vhci_urb_resume(struct bce_vhci_urb *urb); + +int bce_vhci_transfer_queue_do_resume(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + int status; + struct urb *urb, *urbt; + struct bce_vhci_urb *vurb; + u8 endp_addr = (u8) (q->endp->desc.bEndpointAddress & 0x8F); + if (bce_vhci_transfer_queue_is_ep0(q)) + pr_debug("t2bce_vhci: tq resume begin dev=%u port=%d ep=%02x active=%u paused_by=%x stalled=%u state=%u remaining=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests); + if ((status = bce_vhci_cmd_endpoint_set_state( + &q->vhci->cq, q->dev_addr, endp_addr, BCE_VHCI_ENDPOINT_ACTIVE, &q->state))) { + pr_warn("t2bce_vhci: tq resume set-active failed dev=%u port=%d ep=%02x status=%d ret_state=%u active=%u paused_by=%x stalled=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + status, q->state, q->active, q->paused_by, q->stalled); + return status; + } + if (q->state != BCE_VHCI_ENDPOINT_ACTIVE) { + pr_warn("t2bce_vhci: tq resume set-active returned non-active dev=%u port=%d ep=%02x ret_state=%u active=%u paused_by=%x stalled=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + q->state, q->active, q->paused_by, q->stalled); + return -EINVAL; + } + spin_lock_irqsave(&q->urb_lock, flags); + q->active = true; + list_for_each_entry_safe(urb, urbt, &q->endp->urb_list, urb_list) { + vurb = urb->hcpriv; + if (vurb->state == BCE_VHCI_URB_INIT_PENDING) { + if (!bce_vhci_transfer_queue_can_init_urb(q)) + break; + bce_vhci_urb_init(vurb); + } else { + bce_vhci_urb_resume(vurb); + } + } + bce_vhci_transfer_queue_deliver_pending(q); + spin_unlock_irqrestore(&q->urb_lock, flags); + if (bce_vhci_transfer_queue_is_ep0(q)) + pr_debug("t2bce_vhci: tq resume done dev=%u port=%d ep=%02x active=%u paused_by=%x stalled=%u state=%u remaining=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests); + return 0; +} + +int bce_vhci_transfer_queue_pause(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src) +{ + int ret = 0; + u32 old_paused_by; + + mutex_lock(&q->pause_lock); + old_paused_by = q->paused_by; + if ((q->paused_by & src) != src) { + if (!q->paused_by) + ret = bce_vhci_transfer_queue_do_pause(q); + if (!ret) + q->paused_by |= src; + } + if (bce_vhci_transfer_queue_is_ep0(q) || ret) + pr_debug("t2bce_vhci: tq pause dev=%u port=%d ep=%02x src=%x ret=%d paused_by=%x->%x active=%u stalled=%u state=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + src, ret, old_paused_by, q->paused_by, q->active, + q->stalled, q->state); + mutex_unlock(&q->pause_lock); + return ret; +} + +int bce_vhci_transfer_queue_resume(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src) +{ + int ret = 0; + u32 old_paused_by; + bool did_resume = false; + bool cleared_shutdown = false; + + mutex_lock(&q->pause_lock); + old_paused_by = q->paused_by; + /* + * EP0 queues can carry the old shutdown pause across preserved-state + * resume. System resume owns that transition, so clear the shutdown owner + * here before deciding whether the queue needs a firmware resume. + */ + if (src == BCE_VHCI_PAUSE_SUSPEND && + q->endp_addr == 0x00 && + q->paused_by == (BCE_VHCI_PAUSE_SHUTDOWN | BCE_VHCI_PAUSE_SUSPEND)) { + q->paused_by &= ~BCE_VHCI_PAUSE_SHUTDOWN; + cleared_shutdown = true; + } + if (q->paused_by & src) { + if (!(q->paused_by & ~src)) { + ret = bce_vhci_transfer_queue_do_resume(q); + did_resume = true; + } + if (!ret) + q->paused_by &= ~src; + } + if (src == BCE_VHCI_PAUSE_SUSPEND) + pr_debug("t2bce_vhci: system resume dev=%u ep=%02x src=%x paused_by=%x->%x firmware_resume=%u cleared_shutdown=%u ret=%d state=%x active=%u\n", + q->dev_addr, q->endp_addr, src, old_paused_by, q->paused_by, + did_resume, cleared_shutdown, ret, q->state, q->active); + else if (q->endp_addr == 0x00) + pr_debug("t2bce_vhci: EP0 resume dev=%u src=%x paused_by=%x->%x did_resume=%u ret=%d state=%x active=%u\n", + q->dev_addr, src, old_paused_by, q->paused_by, did_resume, + ret, q->state, q->active); + mutex_unlock(&q->pause_lock); + return ret; +} + +int bce_vhci_transfer_queue_suspend_pause(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + int status; + u8 endp_addr = (u8) (q->endp->desc.bEndpointAddress & 0x8F); + int pending; + long timeout; + int ret = 0; + u32 old_paused_by; + + mutex_lock(&q->pause_lock); + old_paused_by = q->paused_by; + if ((q->paused_by & BCE_VHCI_PAUSE_SUSPEND) == BCE_VHCI_PAUSE_SUSPEND) { + pr_debug("t2bce_vhci: system suspend dev=%u ep=%02x path=already-suspend paused_by=%x state=%x active=%u\n", + q->dev_addr, q->endp_addr, q->paused_by, q->state, q->active); + goto out; + } + if (q->paused_by) { + /* + * Another owner already paused the endpoint. In that case system + * suspend only records its own ownership bit and must not send a + * second firmware pause for the same queue. + */ + q->paused_by |= BCE_VHCI_PAUSE_SUSPEND; + pr_debug("t2bce_vhci: system suspend dev=%u ep=%02x path=mark-only paused_by=%x->%x state=%x active=%u\n", + q->dev_addr, q->endp_addr, old_paused_by, q->paused_by, + q->state, q->active); + goto out; + } + + spin_lock_irqsave(&q->urb_lock, flags); + q->active = false; + spin_unlock_irqrestore(&q->urb_lock, flags); + if (q->sq_out) { + pending = atomic_read(&q->sq_out_pending); + if (pending > 0) { + timeout = wait_event_timeout(q->sq_out_wait_queue, + atomic_read(&q->sq_out_pending) == 0, + msecs_to_jiffies(2000)); + if (!timeout && atomic_read(&q->sq_out_pending) > 0) + pr_warn("t2bce_vhci: [%02x] pause timeout waiting for %d outputs\n", + q->endp_addr, atomic_read(&q->sq_out_pending)); + } + } + /* + * System suspend is a preserved-state pause, not endpoint teardown. + * Deferred TRANSFER_REQUEST events already emitted by the T2 still belong + * to the same endpoint after resume; dropping them can make the first + * post-resume EP0 control URB wait until usbcore's timeout path. + */ + if ((status = bce_vhci_cmd_endpoint_set_state( + &q->vhci->cq, q->dev_addr, endp_addr, BCE_VHCI_ENDPOINT_PAUSED, &q->state))) { + ret = status; + goto out; + } + if (q->state != BCE_VHCI_ENDPOINT_PAUSED) { + ret = -EINVAL; + goto out; + } + q->paused_by |= BCE_VHCI_PAUSE_SUSPEND; + pr_debug("t2bce_vhci: system suspend dev=%u ep=%02x path=firmware-pause paused_by=%x->%x ret=%d state=%x active=%u\n", + q->dev_addr, q->endp_addr, old_paused_by, q->paused_by, + ret, q->state, q->active); + +out: + mutex_unlock(&q->pause_lock); + return ret; +} + +static void bce_vhci_transfer_queue_reset_w(struct work_struct *work) +{ + unsigned long flags; + struct bce_vhci_transfer_queue *q = container_of(work, struct bce_vhci_transfer_queue, w_reset); + + mutex_lock(&q->pause_lock); + spin_lock_irqsave(&q->urb_lock, flags); + pr_debug("t2bce_vhci: tq reset start dev=%u port=%d ep=%02x active=%u paused_by=%x stalled=%u state=%u remaining=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests); + if (!q->stalled) { + spin_unlock_irqrestore(&q->urb_lock, flags); + mutex_unlock(&q->pause_lock); + pr_debug("t2bce_vhci: tq reset skipped dev=%u port=%d ep=%02x stalled=0 active=%u paused_by=%x state=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + q->active, q->paused_by, q->state); + return; + } + q->active = false; + spin_unlock_irqrestore(&q->urb_lock, flags); + q->paused_by |= BCE_VHCI_PAUSE_INTERNAL_WQ; + bce_vhci_transfer_queue_remove_pending(q); + if (q->sq_in) + t2bce_core_flush_queue(q->vhci->client, q->sq_in); + if (q->sq_out) + t2bce_core_flush_queue(q->vhci->client, q->sq_out); + pr_debug("t2bce_vhci: tq reset command dev=%u port=%d ep=%02x\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr); + bce_vhci_cmd_endpoint_reset(&q->vhci->cq, q->dev_addr, (u8) (q->endp->desc.bEndpointAddress & 0x8F)); + spin_lock_irqsave(&q->urb_lock, flags); + q->stalled = false; + spin_unlock_irqrestore(&q->urb_lock, flags); + mutex_unlock(&q->pause_lock); + bce_vhci_transfer_queue_resume(q, BCE_VHCI_PAUSE_INTERNAL_WQ); + pr_debug("t2bce_vhci: tq reset done dev=%u port=%d ep=%02x active=%u paused_by=%x stalled=%u state=%u remaining=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests); +} + +static void bce_vhci_transfer_queue_resume_w(struct work_struct *work) +{ + int ret; + struct bce_vhci_transfer_queue *q = container_of(work, struct bce_vhci_transfer_queue, w_resume); + + pr_debug("t2bce_vhci: tq async resume start dev=%u port=%d ep=%02x active=%u paused_by=%x stalled=%u state=%u remaining=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests); + ret = bce_vhci_transfer_queue_resume(q, BCE_VHCI_PAUSE_INTERNAL_WQ); + pr_debug("t2bce_vhci: tq async resume done dev=%u port=%d ep=%02x ret=%d active=%u paused_by=%x stalled=%u state=%u remaining=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + ret, q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests); +} + +static void bce_vhci_transfer_queue_cancel_w(struct work_struct *work) +{ + unsigned long flags; + struct bce_vhci_transfer_queue *q = container_of(work, struct bce_vhci_transfer_queue, w_cancel); + struct bce_vhci_urb *vurb; + struct urb *urb; + int status; + bool was_active; + + for (;;) { + spin_lock_irqsave(&q->urb_lock, flags); + if (list_empty(&q->cancel_urb_list)) { + spin_unlock_irqrestore(&q->urb_lock, flags); + return; + } + + vurb = list_first_entry(&q->cancel_urb_list, struct bce_vhci_urb, cancel_list); + list_del_init(&vurb->cancel_list); + urb = vurb->urb; + status = vurb->cancel_status; + was_active = vurb->cancel_was_active; + spin_unlock_irqrestore(&q->urb_lock, flags); + + /* Endpoint pause talks to bridgeOS and may sleep. */ + if (was_active) { + pr_debug("t2bce_vhci: [%02x] async cancelling URB\n", q->endp_addr); + bce_vhci_transfer_queue_pause(q, BCE_VHCI_PAUSE_INTERNAL_WQ); + } + + spin_lock_irqsave(&q->urb_lock, flags); + if (was_active) + ++q->remaining_active_requests; + usb_hcd_unlink_urb_from_ep(q->vhci->hcd, urb); + urb->hcpriv = NULL; + spin_unlock_irqrestore(&q->urb_lock, flags); + + usb_hcd_giveback_urb(q->vhci->hcd, urb, status); + + if (was_active) + bce_vhci_transfer_queue_resume(q, BCE_VHCI_PAUSE_INTERNAL_WQ); + + bce_vhci_urb_free(vurb); + } +} + +void bce_vhci_transfer_queue_request_reset(struct bce_vhci_transfer_queue *q) +{ + queue_work(q->vhci->tq_state_wq, &q->w_reset); +} + +static void bce_vhci_transfer_queue_init_pending_urbs(struct bce_vhci_transfer_queue *q) +{ + struct urb *urb, *urbt; + struct bce_vhci_urb *vurb; + list_for_each_entry_safe(urb, urbt, &q->endp->urb_list, urb_list) { + vurb = urb->hcpriv; + if (!bce_vhci_transfer_queue_can_init_urb(q)) + break; + if (vurb->state == BCE_VHCI_URB_INIT_PENDING) + bce_vhci_urb_init(vurb); + } +} + + + +static int bce_vhci_urb_data_start(struct bce_vhci_urb *urb, unsigned long *timeout); + +static bool bce_vhci_urb_uses_sg(struct bce_vhci_urb *urb) +{ + return urb->sg_segments && urb->urb->sg && urb->urb->num_mapped_sgs > 0; +} + +static void bce_vhci_urb_sync_sg_segments_for_cpu(struct bce_vhci_urb *urb) +{ + struct device *dev; + + if (!urb->sg_segments_for_device) + return; + + dev = t2bce_core_client_dma_dev(urb->q->vhci->client); + dma_sync_single_for_cpu(dev, urb->sg_segments_dma, urb->sg_segments_size, + DMA_TO_DEVICE); + urb->sg_segments_for_device = false; +} + +static void bce_vhci_urb_free(struct bce_vhci_urb *urb) +{ + struct device *dev; + + if (!urb) + return; + + if (urb->sg_segments) { + bce_vhci_urb_sync_sg_segments_for_cpu(urb); + dev = t2bce_core_client_dma_dev(urb->q->vhci->client); + dma_unmap_single(dev, urb->sg_segments_dma, urb->sg_segments_size, + DMA_TO_DEVICE); + kfree(urb->sg_segments); + } + + kfree(urb); +} + +static int bce_vhci_urb_init_sg_segments(struct bce_vhci_urb *vurb) +{ + struct device *dev; + + if (!vurb->urb->sg || vurb->urb->num_mapped_sgs <= 0) + return 0; + + if (vurb->urb->num_mapped_sgs > SIZE_MAX / sizeof(*vurb->sg_segments)) + return -EOVERFLOW; + + vurb->sg_segment_count = vurb->urb->num_mapped_sgs; + vurb->sg_segments_size = vurb->sg_segment_count * sizeof(*vurb->sg_segments); + vurb->sg_segments = kcalloc(vurb->sg_segment_count, + sizeof(*vurb->sg_segments), GFP_KERNEL); + if (!vurb->sg_segments) + return -ENOMEM; + + dev = t2bce_core_client_dma_dev(vurb->q->vhci->client); + vurb->sg_segments_dma = dma_map_single(dev, vurb->sg_segments, + vurb->sg_segments_size, DMA_TO_DEVICE); + if (dma_mapping_error(dev, vurb->sg_segments_dma)) { + kfree(vurb->sg_segments); + vurb->sg_segments = NULL; + vurb->sg_segments_size = 0; + vurb->sg_segment_count = 0; + return -ENOMEM; + } + vurb->sg_segments_for_device = true; + bce_vhci_urb_sync_sg_segments_for_cpu(vurb); + + return 0; +} + +static int bce_vhci_urb_set_sg_submission(struct bce_vhci_urb *urb, + struct t2bce_core_queue_sq *sq, u32 offset, u32 size) +{ + struct device *dev = t2bce_core_client_dma_dev(urb->q->vhci->client); + struct scatterlist *sg; + unsigned int i; + unsigned int segment_count = 0; + u32 skip = offset; + u32 remaining = size; + + if (!bce_vhci_urb_uses_sg(urb) || !size) + return -EINVAL; + + /* + * BCE consumes a compact address/length array, not Linux scatterlist + * metadata. Rebuild that array for the requested URB window. + */ + bce_vhci_urb_sync_sg_segments_for_cpu(urb); + + for_each_sg(urb->urb->sg, sg, urb->urb->num_mapped_sgs, i) { + dma_addr_t addr = sg_dma_address(sg); + u32 sg_len = sg_dma_len(sg); + u32 segment_len; + + if (!sg_len) + continue; + if (skip >= sg_len) { + skip -= sg_len; + continue; + } + + addr += skip; + sg_len -= skip; + skip = 0; + + segment_len = min(sg_len, remaining); + urb->sg_segments[segment_count].addr = addr; + urb->sg_segments[segment_count].length = segment_len; + segment_count++; + remaining -= segment_len; + + if (!remaining) + break; + } + + if (remaining || !segment_count) + return -EINVAL; + + dma_sync_single_for_device(dev, urb->sg_segments_dma, + segment_count * sizeof(*urb->sg_segments), DMA_TO_DEVICE); + urb->sg_segments_for_device = true; + t2bce_core_set_next_submission_segment_list(sq, urb->sg_segments_dma, + segment_count * sizeof(*urb->sg_segments)); + return 0; +} + +int bce_vhci_urb_create(struct bce_vhci_transfer_queue *q, struct urb *urb) +{ + unsigned long flags; + int status = 0; + struct bce_vhci_urb *vurb; + bool is_control = (usb_endpoint_num(&urb->ep->desc) == 0); + + if (is_control && q->endp_addr == 0x00 && + !q->active && !q->stalled && + q->paused_by == BCE_VHCI_PAUSE_INTERNAL_WQ) { + pr_warn("t2bce_vhci: EP0 enqueue scheduling async stale internal pause recovery dev=%u port=%d state=%u remaining=%u urb_reject=%d\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->state, + q->remaining_active_requests, atomic_read(&urb->reject)); + queue_work(q->vhci->tq_state_wq, &q->w_resume); + } + + vurb = kzalloc(sizeof(struct bce_vhci_urb), GFP_KERNEL); + if (!vurb) + return -ENOMEM; + + urb->hcpriv = vurb; + + vurb->q = q; + vurb->urb = urb; + vurb->dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + vurb->is_control = is_control; + INIT_LIST_HEAD(&vurb->cancel_list); + status = bce_vhci_urb_init_sg_segments(vurb); + if (status) { + urb->hcpriv = NULL; + bce_vhci_urb_free(vurb); + return status; + } + + bce_vhci_urb_log_control(vurb, "enqueue"); + + spin_lock_irqsave(&q->urb_lock, flags); + if (vurb->is_control && (!q->active || q->stalled || q->paused_by)) + pr_debug("t2bce_vhci: EP0 enqueue dev=%u port=%d active=%u paused_by=%x stalled=%u state=%u remaining=%u urb_reject=%d\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), + q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests, atomic_read(&urb->reject)); + + status = usb_hcd_link_urb_to_ep(q->vhci->hcd, urb); + if (status) { + if (vurb->is_control) + pr_warn("t2bce_vhci: EP0 enqueue link failed dev=%u port=%d status=%d active=%u paused_by=%x stalled=%u state=%u remaining=%u urb_reject=%d\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), status, + q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests, atomic_read(&urb->reject)); + spin_unlock_irqrestore(&q->urb_lock, flags); + urb->hcpriv = NULL; + bce_vhci_urb_free(vurb); + return status; + } + + if (q->active) { + if (bce_vhci_transfer_queue_can_init_urb(vurb->q)) + status = bce_vhci_urb_init(vurb); + else + vurb->state = BCE_VHCI_URB_INIT_PENDING; + } else { + if (q->stalled) + bce_vhci_transfer_queue_request_reset(q); + vurb->state = BCE_VHCI_URB_INIT_PENDING; + } + if (status) { + usb_hcd_unlink_urb_from_ep(q->vhci->hcd, urb); + urb->hcpriv = NULL; + bce_vhci_urb_free(vurb); + } else { + bce_vhci_transfer_queue_deliver_pending(q); + } + spin_unlock_irqrestore(&q->urb_lock, flags); + pr_debug("t2bce_vhci: [%02x] URB enqueued (dir = %s, size = %i)\n", q->endp_addr, + usb_urb_dir_in(urb) ? "IN" : "OUT", urb->transfer_buffer_length); + return status; +} + +static int bce_vhci_urb_init(struct bce_vhci_urb *vurb) +{ + int status = 0; + + if (vurb->q->remaining_active_requests == 0) { + pr_err("t2bce_vhci: cannot init request (remaining_active_requests = 0)\n"); + return -EINVAL; + } + + if (vurb->is_control) { + vurb->state = BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST; + bce_vhci_urb_log_control(vurb, "init-wait-setup"); + } else { + status = bce_vhci_urb_data_start(vurb, NULL); + } + + if (!status) { + --vurb->q->remaining_active_requests; + } + return status; +} + +static void bce_vhci_urb_complete(struct bce_vhci_urb *urb, int status) +{ + struct bce_vhci_transfer_queue *q = urb->q; + struct bce_vhci *vhci = q->vhci; + struct urb *real_urb = urb->urb; + pr_debug("t2bce_vhci: [%02x] URB complete %i\n", q->endp_addr, status); + bce_vhci_urb_log_control(urb, "complete"); + usb_hcd_unlink_urb_from_ep(vhci->hcd, real_urb); + real_urb->hcpriv = NULL; + real_urb->status = status; + if (urb->state != BCE_VHCI_URB_INIT_PENDING) + ++urb->q->remaining_active_requests; + bce_vhci_urb_free(urb); + list_add_tail(&real_urb->urb_list, &q->giveback_urb_list); +} + +int bce_vhci_urb_request_cancel(struct bce_vhci_transfer_queue *q, struct urb *urb, int status) +{ + struct bce_vhci_urb *vurb; + unsigned long flags; + int ret; + enum bce_vhci_urb_state old_state; + + spin_lock_irqsave(&q->urb_lock, flags); + if ((ret = usb_hcd_check_unlink_urb(q->vhci->hcd, urb, status))) { + spin_unlock_irqrestore(&q->urb_lock, flags); + return ret; + } + + vurb = urb->hcpriv; + if (!vurb) { + spin_unlock_irqrestore(&q->urb_lock, flags); + return -ENOENT; + } + + old_state = vurb->state; + bce_vhci_urb_log_control(vurb, "cancel-request"); + + if (old_state == BCE_VHCI_URB_CANCELLED) { + spin_unlock_irqrestore(&q->urb_lock, flags); + pr_debug("t2bce_vhci: URB %p is already cancelled, skipping\n", urb); + return 0; + } + + vurb->state = BCE_VHCI_URB_CANCELLED; + vurb->cancel_status = status; + vurb->cancel_was_active = old_state != BCE_VHCI_URB_INIT_PENDING; + + if (list_empty(&vurb->cancel_list)) + list_add_tail(&vurb->cancel_list, &q->cancel_urb_list); + spin_unlock_irqrestore(&q->urb_lock, flags); + queue_work(q->vhci->tq_state_wq, &q->w_cancel); + + return 0; +} + +static int bce_vhci_urb_data_transfer_in(struct bce_vhci_urb *urb, unsigned long *timeout) +{ + struct bce_vhci_message msg; + u32 tr_len; + int reservation1, reservation2 = -EFAULT; + bool uses_sg; + int status; + + pr_debug("t2bce_vhci: [%02x] DMA from device %llx %x\n", urb->q->endp_addr, + (u64) urb->urb->transfer_dma, urb->urb->transfer_buffer_length); + + /* The firmware request and the inbound DMA submission must be queued as a pair. */ + reservation1 = t2bce_core_reserve_submission(urb->q->vhci->msg_asynchronous.sq, timeout); + if (!reservation1) + reservation2 = t2bce_core_reserve_submission(urb->q->sq_in, timeout); + if (reservation1 || reservation2) { + pr_err("t2bce_vhci: Failed to reserve a submission for URB data transfer\n"); + if (!reservation1) + t2bce_core_cancel_submission_reservation(urb->q->vhci->msg_asynchronous.sq); + return -ENOMEM; + } + + urb->send_offset = urb->receive_offset; + + tr_len = urb->urb->transfer_buffer_length - urb->send_offset; + uses_sg = bce_vhci_urb_uses_sg(urb); + + if (uses_sg) { + status = bce_vhci_urb_set_sg_submission(urb, urb->q->sq_in, + urb->send_offset, tr_len); + if (status) { + t2bce_core_cancel_submission_reservation(urb->q->sq_in); + t2bce_core_cancel_submission_reservation(urb->q->vhci->msg_asynchronous.sq); + pr_err("t2bce_vhci: [%02x] failed to prepare SG IN submission: %d\n", + urb->q->endp_addr, status); + return status; + } + } + + spin_lock(&urb->q->vhci->msg_asynchronous_lock); + msg.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST; + msg.status = 0; + msg.param1 = ((urb->urb->ep->desc.bEndpointAddress & 0x8Fu) << 8) | urb->q->dev_addr; + msg.param2 = tr_len; + bce_vhci_message_queue_write(&urb->q->vhci->msg_asynchronous, &msg); + spin_unlock(&urb->q->vhci->msg_asynchronous_lock); + + if (!uses_sg) + t2bce_core_set_next_submission_single(urb->q->sq_in, + urb->urb->transfer_dma + urb->send_offset, tr_len); + t2bce_core_submit_to_device(urb->q->sq_in); + + urb->state = BCE_VHCI_URB_WAITING_FOR_COMPLETION; + return 0; +} + +static int bce_vhci_urb_data_start(struct bce_vhci_urb *urb, unsigned long *timeout) +{ + if (urb->dir == DMA_TO_DEVICE) { + if (urb->urb->transfer_buffer_length > 0) + urb->state = BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST; + else + urb->state = BCE_VHCI_URB_DATA_TRANSFER_COMPLETE; + bce_vhci_urb_log_control(urb, "data-start-out"); + return 0; + } else { + bce_vhci_urb_log_control(urb, "data-start-in"); + return bce_vhci_urb_data_transfer_in(urb, timeout); + } +} + +static int bce_vhci_urb_send_out_data(struct bce_vhci_urb *urb, dma_addr_t addr, size_t size) +{ + unsigned long timeout = 0; + if (t2bce_core_reserve_submission(urb->q->sq_out, &timeout)) { + pr_err("t2bce_vhci: Failed to reserve a submission for URB data transfer\n"); + return -EPIPE; + } + + pr_debug("t2bce_vhci: [%02x] DMA to device %llx %lx\n", urb->q->endp_addr, (u64) addr, size); + + t2bce_core_set_next_submission_single(urb->q->sq_out, addr, size); + atomic_inc(&urb->q->sq_out_pending); + t2bce_core_submit_to_device(urb->q->sq_out); + return 0; +} + +static int bce_vhci_urb_send_out_transfer_data(struct bce_vhci_urb *urb, + u32 offset, size_t size) +{ + unsigned long timeout = 0; + int status; + + if (t2bce_core_reserve_submission(urb->q->sq_out, &timeout)) { + pr_err("t2bce_vhci: Failed to reserve a submission for URB data transfer\n"); + return -EPIPE; + } + + if (bce_vhci_urb_uses_sg(urb)) { + status = bce_vhci_urb_set_sg_submission(urb, urb->q->sq_out, + offset, size); + if (status) { + t2bce_core_cancel_submission_reservation(urb->q->sq_out); + pr_err("t2bce_vhci: [%02x] failed to prepare SG OUT submission: %d\n", + urb->q->endp_addr, status); + return status; + } + } else { + t2bce_core_set_next_submission_single(urb->q->sq_out, + urb->urb->transfer_dma + offset, size); + } + + atomic_inc(&urb->q->sq_out_pending); + t2bce_core_submit_to_device(urb->q->sq_out); + return 0; +} + +static int bce_vhci_urb_data_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg) +{ + u32 tr_len; + int status; + if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST) { + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST) { + tr_len = min(urb->urb->transfer_buffer_length - urb->send_offset, (u32) msg->param2); + bce_vhci_urb_log_control(urb, "data-request"); + if ((status = bce_vhci_urb_send_out_transfer_data(urb, urb->send_offset, tr_len))) + return status; + urb->send_offset += tr_len; + urb->state = BCE_VHCI_URB_WAITING_FOR_COMPLETION; + bce_vhci_urb_log_control(urb, "data-sent"); + return 0; + } + } + + /* OUT endpoints can receive extra transfer requests while waiting for host data. */ + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && urb->q->sq_out != NULL) + return -EAGAIN; + pr_err("t2bce_vhci: [%02x] %s URB unexpected message (state = %x, msg: %x %x %x %llx)\n", + urb->q->endp_addr, (urb->is_control ? "Control (data update)" : "Data"), urb->state, + msg->cmd, msg->status, msg->param1, msg->param2); + return -EAGAIN; +} + +static int bce_vhci_urb_data_transfer_completion(struct bce_vhci_urb *urb, struct t2bce_core_sq_completion_data *c) +{ + bce_vhci_urb_sync_sg_segments_for_cpu(urb); + + if (urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { + urb->receive_offset += c->data_size; + if (urb->dir == DMA_FROM_DEVICE || urb->receive_offset >= urb->urb->transfer_buffer_length) { + urb->urb->actual_length = (u32) urb->receive_offset; + urb->state = BCE_VHCI_URB_DATA_TRANSFER_COMPLETE; + bce_vhci_urb_log_control(urb, "data-complete"); + if (!urb->is_control) { + bce_vhci_urb_complete(urb, 0); + return -ENOENT; + } + } + } else { + pr_warn_ratelimited("t2bce_vhci: [%02x] data URB unexpected completion state=%x status=%x size=%llu\n", + urb->q->endp_addr, urb->state, c->status, c->data_size); + } + return 0; +} + + +static int bce_vhci_urb_control_check_status(struct bce_vhci_urb *urb) +{ + struct bce_vhci_transfer_queue *q = urb->q; + if (urb->received_status == 0) + return 0; + if (urb->state == BCE_VHCI_URB_DATA_TRANSFER_COMPLETE || + (urb->received_status != BCE_VHCI_SUCCESS && urb->state != BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST && + urb->state != BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION)) { + urb->state = BCE_VHCI_URB_CONTROL_COMPLETE; + if (urb->received_status != BCE_VHCI_SUCCESS) { + if (urb->received_status == 3 && urb->q->endp_addr == 0x00) { + q->active = false; + q->stalled = true; + q->state = BCE_VHCI_ENDPOINT_STALLED; + pr_debug("t2bce_vhci: EP0 status=3 dev=%u state=%x; completing urb and scheduling endpoint reset paused_by=%x remaining=%u\n", + q->dev_addr, urb->state, q->paused_by, + q->remaining_active_requests); + bce_vhci_urb_complete(urb, -EPIPE); + bce_vhci_transfer_queue_request_reset(q); + return -ENOENT; + } + + pr_err("t2bce_vhci: [%02x] URB failed: %x\n", + urb->q->endp_addr, urb->received_status); + urb->q->active = false; + urb->q->stalled = true; + bce_vhci_urb_complete(urb, -EPIPE); + if (!list_empty(&q->endp->urb_list)) + bce_vhci_transfer_queue_request_reset(q); + return -ENOENT; + } + bce_vhci_urb_complete(urb, 0); + return -ENOENT; + } + return 0; +} + +static int bce_vhci_urb_control_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg) +{ + int status; + if (msg->cmd == BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS) { + urb->received_status = msg->status; + bce_vhci_urb_log_control(urb, "status-msg"); + return bce_vhci_urb_control_check_status(urb); + } + + if (urb->state == BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST) { + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST) { + bce_vhci_urb_log_control(urb, "setup-request"); + if (bce_vhci_urb_send_out_data(urb, urb->urb->setup_dma, sizeof(struct usb_ctrlrequest))) { + pr_err("t2bce_vhci: [%02x] Failed to start URB setup transfer\n", urb->q->endp_addr); + /* Keep the old state machine behavior; endpoint recovery is driven by later firmware status. */ + return 0; + } + urb->state = BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION; + bce_vhci_urb_log_control(urb, "setup-sent"); + pr_debug("t2bce_vhci: [%02x] Sent setup %llx\n", urb->q->endp_addr, urb->urb->setup_dma); + return 0; + } + } else if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST || + urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { + if ((status = bce_vhci_urb_data_update(urb, msg))) + return status; + return bce_vhci_urb_control_check_status(urb); + } + + /* OUT endpoints can receive extra transfer requests while waiting for host data. */ + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && urb->q->sq_out != NULL) + return -EAGAIN; + pr_err("t2bce_vhci: [%02x] Control URB unexpected message (state = %x, msg: %x %x %x %llx)\n", urb->q->endp_addr, + urb->state, msg->cmd, msg->status, msg->param1, msg->param2); + return -EAGAIN; +} + +static int bce_vhci_urb_control_transfer_completion(struct bce_vhci_urb *urb, struct t2bce_core_sq_completion_data *c) +{ + int status; + unsigned long timeout; + + if (urb->state == BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION) { + if (c->data_size != sizeof(struct usb_ctrlrequest)) + pr_err("t2bce_vhci: [%02x] transfer complete data size mistmatch for usb_ctrlrequest (%llx instead of %lx)\n", + urb->q->endp_addr, c->data_size, sizeof(struct usb_ctrlrequest)); + + bce_vhci_urb_log_control(urb, "setup-complete"); + timeout = 1000; + status = bce_vhci_urb_data_start(urb, &timeout); + if (status) { + bce_vhci_urb_complete(urb, status); + return -ENOENT; + } + return 0; + } else if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST || + urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { + if ((status = bce_vhci_urb_data_transfer_completion(urb, c))) + return status; + return bce_vhci_urb_control_check_status(urb); + } else { + pr_err("t2bce_vhci: [%02x] Control URB unexpected completion (state = %x)\n", urb->q->endp_addr, urb->state); + } + return 0; +} + +static int bce_vhci_urb_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg) +{ + if (urb->state == BCE_VHCI_URB_INIT_PENDING) + return -EAGAIN; + if (urb->is_control) + return bce_vhci_urb_control_update(urb, msg); + else + return bce_vhci_urb_data_update(urb, msg); +} + +static int bce_vhci_urb_transfer_completion(struct bce_vhci_urb *urb, struct t2bce_core_sq_completion_data *c) +{ + if (urb->is_control) + return bce_vhci_urb_control_transfer_completion(urb, c); + else + return bce_vhci_urb_data_transfer_completion(urb, c); +} + +static void bce_vhci_urb_resume(struct bce_vhci_urb *urb) +{ + int status = 0; + if (urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { + status = bce_vhci_urb_data_transfer_in(urb, NULL); + } + if (status) + bce_vhci_urb_complete(urb, status); +} diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_vhci/transfer.h b/drivers/staging/t2bce/t2bce_vhci/transfer.h --- a/drivers/staging/t2bce/t2bce_vhci/transfer.h 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_vhci/transfer.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,86 @@ +#ifndef BCEDRIVER_TRANSFER_H +#define BCEDRIVER_TRANSFER_H + +#include +#include +#include "queue.h" +#include "command.h" + +struct bce_vhci_list_message { + struct list_head list; + struct bce_vhci_message msg; +}; +enum bce_vhci_pause_source { + BCE_VHCI_PAUSE_INTERNAL_WQ = 1, + BCE_VHCI_PAUSE_FIRMWARE = 2, + BCE_VHCI_PAUSE_SUSPEND = 4, + BCE_VHCI_PAUSE_SHUTDOWN = 8 +}; +struct bce_vhci_transfer_queue { + struct bce_vhci *vhci; + struct usb_host_endpoint *endp; + enum bce_vhci_endpoint_state state; + u32 max_active_requests, remaining_active_requests; + bool active, stalled; + u32 paused_by; + bce_vhci_device_t dev_addr; + u8 endp_addr; + struct t2bce_core_queue_cq *cq; + struct t2bce_core_queue_sq *sq_in; + struct t2bce_core_queue_sq *sq_out; + struct list_head evq; + struct spinlock urb_lock; + struct mutex pause_lock; + struct list_head giveback_urb_list; + wait_queue_head_t sq_out_wait_queue; + atomic_t sq_out_pending; + struct list_head cancel_urb_list; + + struct work_struct w_reset; + struct work_struct w_resume; + struct work_struct w_cancel; +}; +enum bce_vhci_urb_state { + BCE_VHCI_URB_INIT_PENDING, + BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST, + BCE_VHCI_URB_WAITING_FOR_COMPLETION, + BCE_VHCI_URB_DATA_TRANSFER_COMPLETE, + BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST, + BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION, + BCE_VHCI_URB_CONTROL_COMPLETE, + BCE_VHCI_URB_CANCELLED +}; +struct bce_vhci_urb { + struct urb *urb; + struct bce_vhci_transfer_queue *q; + enum dma_data_direction dir; + bool is_control; + enum bce_vhci_urb_state state; + int received_status; + u32 send_offset; + u32 receive_offset; + struct t2bce_core_dma_segment *sg_segments; + dma_addr_t sg_segments_dma; + size_t sg_segments_size; + unsigned int sg_segment_count; + bool sg_segments_for_device; + struct list_head cancel_list; + int cancel_status; + bool cancel_was_active; +}; + +void bce_vhci_create_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q, + struct usb_host_endpoint *endp, bce_vhci_device_t dev_addr, enum dma_data_direction dir); +void bce_vhci_destroy_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q); +void bce_vhci_transfer_queue_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg); +int bce_vhci_transfer_queue_do_pause(struct bce_vhci_transfer_queue *q); +int bce_vhci_transfer_queue_do_resume(struct bce_vhci_transfer_queue *q); +int bce_vhci_transfer_queue_pause(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src); +int bce_vhci_transfer_queue_resume(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src); +int bce_vhci_transfer_queue_suspend_pause(struct bce_vhci_transfer_queue *q); +void bce_vhci_transfer_queue_request_reset(struct bce_vhci_transfer_queue *q); + +int bce_vhci_urb_create(struct bce_vhci_transfer_queue *q, struct urb *urb); +int bce_vhci_urb_request_cancel(struct bce_vhci_transfer_queue *q, struct urb *urb, int status); + +#endif //BCEDRIVER_TRANSFER_H diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_vhci/vhci.c b/drivers/staging/t2bce/t2bce_vhci/vhci.c --- a/drivers/staging/t2bce/t2bce_vhci/vhci.c 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_vhci/vhci.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,1293 @@ +#include "vhci.h" +#include "command.h" +#include +#include +#include +#include +#include +#include +#include + +/* + * bridgeOS exposes several T2-internal devices through a virtual USB 2.0 hub. + * This HCD translates usbcore hub, device and endpoint operations into the + * firmware command/event protocol carried over BCE queues. + */ + +static dev_t bce_vhci_chrdev; +static struct class *bce_vhci_class; +static const struct hc_driver bce_vhci_driver; +static u16 bce_vhci_port_mask = U16_MAX; +static struct bce_vhci *global_vhci; + +enum bce_vhci_port_status_bits { + BCE_VHCI_PORT_STATUS_OVERCURRENT = 0x2, + BCE_VHCI_PORT_STATUS_CONNECTED = 0x4, + BCE_VHCI_PORT_STATUS_RESET = 0x8, + BCE_VHCI_PORT_STATUS_ENABLED = 0x10, + BCE_VHCI_PORT_STATUS_SUSPENDED_LEGACY = 0x60, + BCE_VHCI_PORT_STATUS_SUSPENDED = 0x80, + BCE_VHCI_PORT_STATUS_HIGH_SPEED = 0x200, + BCE_VHCI_PORT_STATUS_C_CONNECTION = 0x40000, +}; + +static int bce_vhci_create_event_queues(struct bce_vhci *vhci); +static void bce_vhci_destroy_event_queues(struct bce_vhci *vhci); +static int bce_vhci_create_message_queues(struct bce_vhci *vhci); +static void bce_vhci_destroy_message_queues(struct bce_vhci *vhci); +static void bce_vhci_handle_firmware_events_w(struct work_struct *ws); +static void bce_vhci_add_hcd_w(struct work_struct *ws); +static void bce_vhci_firmware_event_completion(struct t2bce_core_queue_sq *sq); +static int bce_vhci_start_controller(struct bce_vhci *vhci); +static void bce_vhci_forget_devices(struct bce_vhci *vhci); +static int __bce_vhci_add_hcd(struct bce_vhci *vhci); +static void bce_vhci_shutdown_client(void *userdata); +static void bce_vhci_pm_reset_client(void *userdata); +static void bce_vhci_pm_prepare_client(void *userdata); +static void bce_vhci_pm_prepare_no_state_client(void *userdata); +static void bce_vhci_pm_mark_no_state_resume_client(void *userdata); +static bool bce_vhci_pm_is_no_state_resume_client(void *userdata); +static void bce_vhci_pm_complete_client(void *userdata); + +static const struct t2bce_core_client_pm_ops bce_vhci_pm_ops = { + .shutdown = bce_vhci_shutdown_client, + .pm_reset = bce_vhci_pm_reset_client, + .pm_prepare = bce_vhci_pm_prepare_client, + .pm_prepare_no_state = bce_vhci_pm_prepare_no_state_client, + .pm_mark_no_state_resume = bce_vhci_pm_mark_no_state_resume_client, + .pm_is_no_state_resume = bce_vhci_pm_is_no_state_resume_client, + .pm_complete = bce_vhci_pm_complete_client, +}; + +int bce_vhci_create(struct device *parent, struct bce_vhci *vhci) +{ + int status; + + spin_lock_init(&vhci->hcd_spinlock); + + vhci->vdevt = bce_vhci_chrdev; + vhci->vdev = device_create(bce_vhci_class, parent, vhci->vdevt, NULL, "t2bce_vhci"); + if (IS_ERR_OR_NULL(vhci->vdev)) { + status = PTR_ERR(vhci->vdev); + goto fail_dev; + } + vhci->client = t2bce_core_client_get(vhci->vdev); + if (IS_ERR(vhci->client)) { + status = PTR_ERR(vhci->client); + vhci->client = NULL; + goto fail_client; + } + t2bce_core_client_set_pm_ops(vhci->client, &bce_vhci_pm_ops, vhci); + + if ((status = bce_vhci_create_message_queues(vhci))) + goto fail_mq; + if ((status = bce_vhci_create_event_queues(vhci))) + goto fail_eq; + + vhci->tq_state_wq = alloc_ordered_workqueue("t2bce_vhci-tq-state", 0); + INIT_WORK(&vhci->w_fw_events, bce_vhci_handle_firmware_events_w); + INIT_WORK(&vhci->w_add_hcd, bce_vhci_add_hcd_w); + vhci->port_change_pending = 0; + vhci->stateful_suspended_bounce = 0; + vhci->stateful_resume = false; + + vhci->hcd = usb_create_hcd(&bce_vhci_driver, vhci->vdev, "t2bce_vhci"); + if (!vhci->hcd) { + status = -ENOMEM; + goto fail_hcd; + } + vhci->hcd->self.sysdev = t2bce_core_client_dma_dev(vhci->client); +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0) + vhci->hcd->self.uses_dma = 1; +#endif + *((struct bce_vhci **) vhci->hcd->hcd_priv) = vhci; + vhci->hcd->speed = HCD_USB2; + + if ((status = __bce_vhci_add_hcd(vhci))) + goto fail_hcd; + + pr_info("t2bce_vhci: initialized\n"); + return 0; + +fail_hcd: + bce_vhci_destroy_event_queues(vhci); +fail_eq: + bce_vhci_destroy_message_queues(vhci); +fail_mq: + t2bce_core_client_set_pm_ops(vhci->client, NULL, NULL); + t2bce_core_client_put(vhci->client); + vhci->client = NULL; +fail_client: + device_destroy(bce_vhci_class, vhci->vdevt); +fail_dev: + if (!status) + status = -EINVAL; + return status; +} + +void bce_vhci_destroy(struct bce_vhci *vhci) +{ + cancel_work_sync(&vhci->w_add_hcd); + bce_vhci_remove_hcd(vhci); + bce_vhci_destroy_event_queues(vhci); + bce_vhci_destroy_message_queues(vhci); + t2bce_core_client_set_pm_ops(vhci->client, NULL, NULL); + t2bce_core_client_put(vhci->client); + vhci->client = NULL; + device_destroy(bce_vhci_class, vhci->vdevt); +} + +static int __bce_vhci_add_hcd(struct bce_vhci *vhci) +{ + int status; + + status = usb_add_hcd(vhci->hcd, 0, 0); + if (!status) + vhci->hcd_registered = true; + return status; +} + +int bce_vhci_add_hcd(struct bce_vhci *vhci) +{ + if (vhci->hcd_registered) + return 0; + return __bce_vhci_add_hcd(vhci); +} + +void bce_vhci_remove_hcd(struct bce_vhci *vhci) +{ + cancel_work_sync(&vhci->w_add_hcd); + if (!vhci->hcd_registered) + return; + usb_remove_hcd(vhci->hcd); + vhci->hcd_registered = false; +} + +void bce_vhci_shutdown(struct bce_vhci *vhci) +{ + cancel_work_sync(&vhci->w_add_hcd); + if (vhci->hcd_registered) { + bce_vhci_remove_hcd(vhci); + return; + } + + /* + * Shutdown is not remove: the USB core may already have detached the HCD, + * but the T2-side controller can still be left enabled. Send the controller + * disable directly as a final lights-out command while queues still exist. + */ + bce_vhci_cmd_controller_disable(&vhci->cq); +} + +void bce_vhci_pm_reset(struct bce_vhci *vhci) +{ + vhci->no_state_resume = false; +} + +void bce_vhci_pm_prepare(struct bce_vhci *vhci) +{ + cancel_work_sync(&vhci->w_add_hcd); + vhci->port_change_pending = 0; +} + +void bce_vhci_pm_prepare_no_state(struct bce_vhci *vhci) +{ + bce_vhci_remove_hcd(vhci); +} + +void bce_vhci_pm_mark_no_state_resume(struct bce_vhci *vhci) +{ + vhci->no_state_resume = true; +} + +bool bce_vhci_pm_is_no_state_resume(struct bce_vhci *vhci) +{ + return vhci->no_state_resume; +} + +void bce_vhci_pm_complete(struct bce_vhci *vhci) +{ + if (!vhci->no_state_resume) + return; + + /* Re-add the VHCI HCD after the PM core completed resume ordering. */ + pr_debug("t2bce_vhci: scheduling HCD re-add after no-state wake\n"); + queue_work(vhci->tq_state_wq, &vhci->w_add_hcd); +} + +static void bce_vhci_pm_reset_client(void *userdata) +{ + bce_vhci_pm_reset(userdata); +} + +static void bce_vhci_pm_prepare_client(void *userdata) +{ + bce_vhci_pm_prepare(userdata); +} + +static void bce_vhci_shutdown_client(void *userdata) +{ + bce_vhci_shutdown(userdata); +} + +static void bce_vhci_pm_prepare_no_state_client(void *userdata) +{ + bce_vhci_pm_prepare_no_state(userdata); +} + +static void bce_vhci_pm_mark_no_state_resume_client(void *userdata) +{ + bce_vhci_pm_mark_no_state_resume(userdata); +} + +static bool bce_vhci_pm_is_no_state_resume_client(void *userdata) +{ + return bce_vhci_pm_is_no_state_resume(userdata); +} + +static void bce_vhci_pm_complete_client(void *userdata) +{ + bce_vhci_pm_complete(userdata); +} + +static void bce_vhci_add_hcd_w(struct work_struct *ws) +{ + struct bce_vhci *vhci = container_of(ws, struct bce_vhci, w_add_hcd); + int status; + + pr_debug("t2bce_vhci: deferred HCD add after no-state wake\n"); + status = bce_vhci_add_hcd(vhci); + if (status) { + pr_err("t2bce_vhci: deferred HCD add failed: %d\n", status); + return; + } + + vhci->no_state_resume = false; +} + +struct bce_vhci *bce_vhci_from_hcd(struct usb_hcd *hcd) +{ + return *((struct bce_vhci **) hcd->hcd_priv); +} + +int bce_vhci_start(struct usb_hcd *hcd) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + return bce_vhci_start_controller(vhci); +} + +void bce_vhci_stop(struct usb_hcd *hcd) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + bce_vhci_cmd_controller_disable(&vhci->cq); +} + +static int bce_vhci_start_controller(struct bce_vhci *vhci) +{ + int status; + u16 port_mask = 0; + bce_vhci_port_t port_no = 0; + + if ((status = bce_vhci_cmd_controller_enable(&vhci->cq, 1, &port_mask))) + return status; + vhci->port_mask = port_mask; + vhci->port_power_mask = 0; + if ((status = bce_vhci_cmd_controller_start(&vhci->cq))) + return status; + while (port_mask) { + port_no += 1; + port_mask >>= 1; + } + vhci->port_count = port_no; + return 0; +} + +static void bce_vhci_forget_devices(struct bce_vhci *vhci) +{ + int i, j; + bce_vhci_device_t devid; + struct bce_vhci_device *dev; + + for (i = 0; i < 16; i++) { + devid = vhci->port_to_device[i]; + if (!devid) + continue; + + dev = vhci->devices[devid]; + if (!dev) { + vhci->port_to_device[i] = 0; + continue; + } + + for (j = 0; j < 32; j++) { + if (!(dev->tq_mask & BIT(j))) + continue; + if (dev->tq[j].endp) + dev->tq[j].endp->hcpriv = NULL; + bce_vhci_destroy_transfer_queue(vhci, &dev->tq[j]); + } + + dev->tq_mask = 0; + vhci->devices[devid] = NULL; + vhci->port_to_device[i] = 0; + kfree(dev); + } +} + +static int bce_vhci_hub_status_data(struct usb_hcd *hcd, char *buf) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + unsigned long pending; + + /* Only ports reported by firmware as changed should wake the hub state machine. */ + pending = READ_ONCE(vhci->port_change_pending); + if (!pending) + return 0; + + memset(buf, 0, 2); + buf[0] = (char)((pending << 1) & 0xff); + buf[1] = (char)((pending >> 7) & 0xff); + return buf[0] || buf[1]; +} + +static int bce_vhci_reset_device(struct bce_vhci *vhci, int index, u16 timeout); + +static int bce_vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, u16 wIndex, char *buf, u16 wLength) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + int status; + struct usb_hub_descriptor *hd; + struct usb_hub_status *hs; + struct usb_port_status *ps; + u32 port_status; + u32 raw_port_status; + if (typeReq == GetHubDescriptor && wLength >= sizeof(struct usb_hub_descriptor)) { + hd = (struct usb_hub_descriptor *) buf; + memset(hd, 0, sizeof(*hd)); + hd->bDescLength = sizeof(struct usb_hub_descriptor); + hd->bDescriptorType = USB_DT_HUB; + hd->bNbrPorts = (u8) vhci->port_count; + hd->wHubCharacteristics = HUB_CHAR_INDV_PORT_LPSM | HUB_CHAR_INDV_PORT_OCPM; + hd->bPwrOn2PwrGood = 0; + hd->bHubContrCurrent = 0; + return 0; + } else if (typeReq == GetHubStatus && wLength >= sizeof(struct usb_hub_status)) { + hs = (struct usb_hub_status *) buf; + memset(hs, 0, sizeof(*hs)); + hs->wHubStatus = 0; + hs->wHubChange = 0; + return 0; + } else if (typeReq == GetPortStatus && wLength >= 4 /* usb 2.0 */) { + ps = (struct usb_port_status *) buf; + ps->wPortStatus = 0; + ps->wPortChange = 0; + + if (vhci->port_power_mask & BIT(wIndex)) + ps->wPortStatus |= USB_PORT_STAT_POWER; + + if (!(bce_vhci_port_mask & BIT(wIndex))) + return 0; + + if ((status = bce_vhci_cmd_port_status(&vhci->cq, (u8) wIndex, 0, &port_status))) + return status; + raw_port_status = port_status; + + /* + * During stateful resume the T2 can briefly report a disconnect bounce + * for an already known virtual port. Keep usbcore from treating that + * transient as a real unplug; the follow-up status/event decides the + * actual port state. + */ + if (vhci->stateful_resume && + (port_status & BCE_VHCI_PORT_STATUS_C_CONNECTION) && + !(port_status & BCE_VHCI_PORT_STATUS_CONNECTED) && + wIndex < ARRAY_SIZE(vhci->port_to_device) && + vhci->port_to_device[wIndex]) { + port_status |= BCE_VHCI_PORT_STATUS_CONNECTED | + BCE_VHCI_PORT_STATUS_ENABLED | + BCE_VHCI_PORT_STATUS_SUSPENDED; + pr_debug("t2bce_vhci: hub masked stateful disconnect bounce port=%u raw=%x masked_raw=%x\n", + wIndex, raw_port_status, port_status); + } + if (port_status & BCE_VHCI_PORT_STATUS_ENABLED) + ps->wPortStatus |= USB_PORT_STAT_ENABLE; + if (port_status & BCE_VHCI_PORT_STATUS_CONNECTED) + ps->wPortStatus |= USB_PORT_STAT_CONNECTION; + if (port_status & BCE_VHCI_PORT_STATUS_OVERCURRENT) + ps->wPortStatus |= USB_PORT_STAT_OVERCURRENT; + if (port_status & BCE_VHCI_PORT_STATUS_RESET) + ps->wPortStatus |= USB_PORT_STAT_RESET; + if (port_status & BCE_VHCI_PORT_STATUS_HIGH_SPEED) + ps->wPortStatus |= USB_PORT_STAT_HIGH_SPEED; + if (port_status & (BCE_VHCI_PORT_STATUS_SUSPENDED | + BCE_VHCI_PORT_STATUS_SUSPENDED_LEGACY)) { + ps->wPortStatus |= USB_PORT_STAT_SUSPEND; + if (vhci->stateful_resume && (port_status & BCE_VHCI_PORT_STATUS_CONNECTED)) + ps->wPortStatus |= USB_PORT_STAT_ENABLE; + } + + /* + * These ports are fixed T2-internal devices, not user-pluggable ports. + * For stateful resume, suppress firmware connection-change bounces so + * usbcore does not re-enumerate solely because the T2 toggled its + * internal representation while restoring state. + */ + if (vhci->stateful_resume && (port_status & BCE_VHCI_PORT_STATUS_C_CONNECTION)) { + u32 cleared_status = 0; + + clear_bit(wIndex - 1, &vhci->port_change_pending); + status = bce_vhci_cmd_port_status(&vhci->cq, (u8) wIndex, 0x40000, &cleared_status); + pr_debug("t2bce_vhci: hub suppressed C_CONNECTION port=%u raw=%x clear_status=%d cleared_raw=%x\n", + wIndex, port_status, status, cleared_status); + } else if (port_status & BCE_VHCI_PORT_STATUS_C_CONNECTION) { + ps->wPortChange |= USB_PORT_STAT_C_CONNECTION; + } + pr_debug("t2bce_vhci: hub GetPortStatus port=%u raw=%x status=%x change=%x\n", + wIndex, port_status, ps->wPortStatus, ps->wPortChange); + return 0; + } else if (typeReq == SetPortFeature) { + if (wValue == USB_PORT_FEAT_POWER) { + status = bce_vhci_cmd_port_power_on(&vhci->cq, (u8) wIndex); + /* usbcore tracks logical power state separately from bridgeOS. */ + if (!status) + vhci->port_power_mask |= BIT(wIndex); + return status; + } + if (wValue == USB_PORT_FEAT_RESET) { + pr_debug("t2bce_vhci: hub SetPortFeature RESET port=%u\n", wIndex); + return bce_vhci_reset_device(vhci, wIndex, wValue); + } + if (wValue == USB_PORT_FEAT_SUSPEND) { + /* Trace the usbcore hub state machine against the firmware port commands. */ + pr_debug("t2bce_vhci: hub SetPortFeature SUSPEND port=%u\n", wIndex); + if (vhci->system_suspending) { + pr_debug("t2bce_vhci: hub SetPortFeature SUSPEND port=%u skipped during system suspend\n", + wIndex); + return 0; + } + status = bce_vhci_cmd_port_suspend(&vhci->cq, (u8) wIndex); + pr_debug("t2bce_vhci: hub SetPortFeature SUSPEND port=%u -> %d\n", wIndex, status); + return status; + } + } else if (typeReq == ClearPortFeature) { + if (wValue == USB_PORT_FEAT_ENABLE) + return bce_vhci_cmd_port_disable(&vhci->cq, (u8) wIndex); + if (wValue == USB_PORT_FEAT_POWER) { + status = bce_vhci_cmd_port_power_off(&vhci->cq, (u8) wIndex); + if (!status) + vhci->port_power_mask &= ~BIT(wIndex); + return status; + } + if (wValue == USB_PORT_FEAT_C_CONNECTION) { + pr_debug("t2bce_vhci: hub ClearPortFeature C_CONNECTION port=%u\n", wIndex); + clear_bit(wIndex - 1, &vhci->port_change_pending); + return bce_vhci_cmd_port_status(&vhci->cq, (u8) wIndex, 0x40000, &port_status); + } + if (wValue == USB_PORT_FEAT_C_RESET) { + pr_debug("t2bce_vhci: hub ClearPortFeature C_RESET port=%u\n", wIndex); + return 0; + } + if (wValue == USB_PORT_FEAT_SUSPEND) { + pr_debug("t2bce_vhci: hub ClearPortFeature SUSPEND port=%u\n", wIndex); + status = bce_vhci_cmd_port_resume(&vhci->cq, (u8) wIndex); + pr_debug("t2bce_vhci: hub ClearPortFeature SUSPEND port=%u -> %d\n", wIndex, status); + return status; + } + } + pr_err("t2bce_vhci: bce_vhci_hub_control unhandled request: %x %i %i [bufl=%i]\n", typeReq, wValue, wIndex, wLength); + dump_stack(); + return -EIO; +} + +static int bce_vhci_enable_device(struct usb_hcd *hcd, struct usb_device *udev) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + struct bce_vhci_device *vdev; + bce_vhci_device_t devid; + pr_debug("t2bce_vhci: enable_device port=%u\n", udev->portnum); + + if (vhci->port_to_device[udev->portnum]) + return 0; + + /* bridgeOS requires a firmware device id before endpoints are created. */ + if (bce_vhci_cmd_device_create(&vhci->cq, udev->portnum, &devid)) + return -EIO; + + pr_debug("t2bce_vhci: device_create port=%u dev=%u\n", udev->portnum, devid); + + vdev = kzalloc(sizeof(struct bce_vhci_device), GFP_KERNEL); + vhci->port_to_device[udev->portnum] = devid; + vhci->devices[devid] = vdev; + + bce_vhci_create_transfer_queue(vhci, &vdev->tq[0], &udev->ep0, devid, DMA_BIDIRECTIONAL); + udev->ep0.hcpriv = &vdev->tq[0]; + vdev->tq_mask |= BIT(0); + + bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &udev->ep0.desc); + return 0; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,8,0) +static int bce_vhci_address_device(struct usb_hcd *hcd, struct usb_device *udev) +#else +static int bce_vhci_address_device(struct usb_hcd *hcd, struct usb_device *udev, unsigned int timeout_ms) +#endif +{ + /* + * Older kernels create the firmware device from enable_device(); newer + * kernels use address_device() for the same bridgeOS operation. The + * firmware command path does not currently consume the usbcore timeout. + */ + return bce_vhci_enable_device(hcd, udev); +} + +static void bce_vhci_free_device(struct usb_hcd *hcd, struct usb_device *udev) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + int i; + bce_vhci_device_t devid; + struct bce_vhci_device *dev; + pr_debug("t2bce_vhci: free_device port=%u\n", udev->portnum); + if (!vhci->port_to_device[udev->portnum]) + return; + devid = vhci->port_to_device[udev->portnum]; + dev = vhci->devices[devid]; + for (i = 0; i < 32; i++) { + if (dev->tq_mask & BIT(i)) { + bce_vhci_transfer_queue_pause(&dev->tq[i], BCE_VHCI_PAUSE_SHUTDOWN); + bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, dev->tq[i].endp_addr); + if (dev->tq[i].endp) + dev->tq[i].endp->hcpriv = NULL; + bce_vhci_destroy_transfer_queue(vhci, &dev->tq[i]); + } + } + vhci->devices[devid] = NULL; + vhci->port_to_device[udev->portnum] = 0; + bce_vhci_cmd_device_destroy(&vhci->cq, devid); + kfree(dev); +} + +static int bce_vhci_reset_device(struct bce_vhci *vhci, int index, u16 timeout) +{ + struct bce_vhci_device *dev = NULL; + bce_vhci_device_t devid; + int i; + int status; + enum dma_data_direction dir; + pr_debug("t2bce_vhci: reset_device port=%d\n", index); + + devid = vhci->port_to_device[index]; + if (devid) { + dev = vhci->devices[devid]; + + for (i = 0; i < 32; i++) { + if (dev->tq_mask & BIT(i)) { + bce_vhci_transfer_queue_pause(&dev->tq[i], BCE_VHCI_PAUSE_SHUTDOWN); + bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, dev->tq[i].endp_addr); + if (dev->tq[i].endp) + dev->tq[i].endp->hcpriv = NULL; + bce_vhci_destroy_transfer_queue(vhci, &dev->tq[i]); + } + } + vhci->devices[devid] = NULL; + vhci->port_to_device[index] = 0; + bce_vhci_cmd_device_destroy(&vhci->cq, devid); + } + status = bce_vhci_cmd_port_reset(&vhci->cq, (u8) index, timeout); + + if (dev) { + if ((status = bce_vhci_cmd_device_create(&vhci->cq, index, &devid))) + return status; + vhci->devices[devid] = dev; + vhci->port_to_device[index] = devid; + + for (i = 0; i < 32; i++) { + if (dev->tq_mask & BIT(i)) { + dir = usb_endpoint_dir_in(&dev->tq[i].endp->desc) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + if (i == 0) + dir = DMA_BIDIRECTIONAL; + bce_vhci_create_transfer_queue(vhci, &dev->tq[i], dev->tq[i].endp, devid, dir); + dev->tq[i].endp->hcpriv = &dev->tq[i]; + bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &dev->tq[i].endp->desc); + } + } + } + + return status; +} + +static int bce_vhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev) +{ + return 0; +} + +static int bce_vhci_get_frame_number(struct usb_hcd *hcd) +{ + return 0; +} + +static int bce_vhci_suspend_quiesce(struct usb_hcd *hcd) +{ + int i, j; + int status; + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + struct bce_vhci_transfer_queue *tq; + struct urb *urb; + struct bce_vhci_urb *vurb; + unsigned long flags; + + pr_debug("t2bce_vhci: suspend started\n"); + + pr_debug("t2bce_vhci: suspend endpoints\n"); + for (i = 0; i < 16; i++) { + if (!vhci->port_to_device[i]) + continue; + for (j = 0; j < 32; j++) { + if (!(vhci->devices[vhci->port_to_device[i]]->tq_mask & BIT(j))) + continue; + tq = &vhci->devices[vhci->port_to_device[i]]->tq[j]; + if (tq->endp_addr == 0x00 || (usb_endpoint_xfer_int(&tq->endp->desc) && + usb_endpoint_dir_in(&tq->endp->desc))) { + spin_lock_irqsave(&tq->urb_lock, flags); + if (!list_empty(&tq->endp->urb_list)) { + urb = list_first_entry(&tq->endp->urb_list, struct urb, urb_list); + vurb = urb->hcpriv; + pr_debug("t2bce_vhci: suspend tq pre: dev=%u ep=%02x active=%u paused_by=%x stalled=%u has_urb=1 first_state=%x\n", + tq->dev_addr, tq->endp_addr, tq->active, tq->paused_by, tq->stalled, + vurb ? vurb->state : 0xffffffff); + } else { + pr_debug("t2bce_vhci: suspend tq pre: dev=%u ep=%02x active=%u paused_by=%x stalled=%u has_urb=0\n", + tq->dev_addr, tq->endp_addr, tq->active, tq->paused_by, tq->stalled); + } + spin_unlock_irqrestore(&tq->urb_lock, flags); + } + + bce_vhci_transfer_queue_suspend_pause(tq); + + if (tq->endp_addr == 0x00 || (usb_endpoint_xfer_int(&tq->endp->desc) && + usb_endpoint_dir_in(&tq->endp->desc))) { + spin_lock_irqsave(&tq->urb_lock, flags); + if (!list_empty(&tq->endp->urb_list)) { + urb = list_first_entry(&tq->endp->urb_list, struct urb, urb_list); + vurb = urb->hcpriv; + pr_debug("t2bce_vhci: suspend tq post: dev=%u ep=%02x active=%u paused_by=%x stalled=%u has_urb=1 first_state=%x\n", + tq->dev_addr, tq->endp_addr, tq->active, tq->paused_by, tq->stalled, + vurb ? vurb->state : 0xffffffff); + } else { + pr_debug("t2bce_vhci: suspend tq post: dev=%u ep=%02x active=%u paused_by=%x stalled=%u has_urb=0\n", + tq->dev_addr, tq->endp_addr, tq->active, tq->paused_by, tq->stalled); + } + spin_unlock_irqrestore(&tq->urb_lock, flags); + } + } + } + + /* + * Only occupied virtual ports are suspended. bridgeOS rejects holes in + * the virtual hub, and that rejection would abort system PM. + */ + pr_debug("t2bce_vhci: suspend occupied ports\n"); + for (i = 1; i < ARRAY_SIZE(vhci->port_to_device); i++) { + if (!vhci->port_to_device[i]) + continue; + + status = bce_vhci_cmd_port_suspend(&vhci->cq, (u8)i); + pr_debug("t2bce_vhci: suspend port %d -> %d\n", i, status); + if (status) + return status; + } + + /* Keep Linux-side event queues quiet before the BCE state-save transition. */ + bce_vhci_event_queue_pause(&vhci->ev_commands); + bce_vhci_event_queue_pause(&vhci->ev_system); + bce_vhci_event_queue_pause(&vhci->ev_isochronous); + bce_vhci_event_queue_pause(&vhci->ev_interrupt); + bce_vhci_event_queue_pause(&vhci->ev_asynchronous); + pr_debug("t2bce_vhci: suspend done\n"); + return 0; +} + +static int bce_vhci_suspend_prepare(struct usb_hcd *hcd) +{ + /* + * VHCI must quiesce usbcore-visible state before the core BCE mailbox + * suspend command asks bridgeOS to save or discard queue state. + */ + return bce_vhci_suspend_quiesce(hcd); +} + +static int bce_vhci_bus_suspend(struct usb_hcd *hcd) +{ + int status; + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + + pr_debug("t2bce_vhci: bus_suspend entry\n"); + vhci->port_change_pending = 0; + vhci->system_suspending = true; + status = bce_vhci_suspend_prepare(hcd); + if (status) + vhci->system_suspending = false; + pr_info("t2bce_vhci: bus_suspend exit status=%d\n", status); + return status; +} + +static int bce_vhci_resume_no_state(struct usb_hcd *hcd) +{ + int i; + int status; + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + + bce_vhci_event_queue_resume(&vhci->ev_system); + bce_vhci_event_queue_resume(&vhci->ev_isochronous); + bce_vhci_event_queue_resume(&vhci->ev_interrupt); + bce_vhci_event_queue_resume(&vhci->ev_asynchronous); + bce_vhci_event_queue_resume(&vhci->ev_commands); + + pr_debug("t2bce_vhci: no-state resume, rebuilding controller state\n"); + vhci->stateful_resume = false; + bce_vhci_forget_devices(vhci); + status = bce_vhci_start_controller(vhci); + if (status) { + pr_info("t2bce_vhci: no-state resume exit status=%d\n", status); + return status; + } + + pr_debug("t2bce_vhci: no-state resume, powering off all ports\n"); + for (i = 1; i <= vhci->port_count; i++) { + status = bce_vhci_cmd_port_power_off(&vhci->cq, (u8) i); + pr_debug("t2bce_vhci: no-state resume, port %d power_off -> %d\n", i, status); + } + + msleep(100); + + pr_debug("t2bce_vhci: no-state resume, powering on all ports\n"); + for (i = 1; i <= vhci->port_count; i++) { + status = bce_vhci_cmd_port_power_on(&vhci->cq, (u8) i); + pr_debug("t2bce_vhci: no-state resume, port %d power_on -> %d\n", i, status); + } + + pr_debug("t2bce_vhci: no-state resume, notifying usbcore about lost power\n"); + usb_root_hub_lost_power(hcd->self.root_hub); + vhci->no_state_resume = false; + pr_info("t2bce_vhci: no-state resume exit status=%d\n", status); + return status; +} + +static int bce_vhci_resume_all_queues(struct bce_vhci *vhci) +{ + int devid; + int endp; + int status; + + for (devid = 1; devid < ARRAY_SIZE(vhci->devices); devid++) { + if (!vhci->devices[devid]) + continue; + + for (endp = 0; endp < 32; endp++) { + if (!(vhci->devices[devid]->tq_mask & BIT(endp))) + continue; + + if (vhci->devices[devid]->tq[endp].endp_addr == 0x00) + pr_debug("t2bce_vhci: resume_all EP0 pre dev=%d paused_by=%x state=%x active=%u\n", + devid, vhci->devices[devid]->tq[endp].paused_by, + vhci->devices[devid]->tq[endp].state, + vhci->devices[devid]->tq[endp].active); + status = bce_vhci_transfer_queue_resume(&vhci->devices[devid]->tq[endp], + BCE_VHCI_PAUSE_SUSPEND); + if (vhci->devices[devid]->tq[endp].endp_addr == 0x00) + pr_debug("t2bce_vhci: resume_all EP0 post dev=%d status=%d paused_by=%x state=%x active=%u\n", + devid, status, vhci->devices[devid]->tq[endp].paused_by, + vhci->devices[devid]->tq[endp].state, + vhci->devices[devid]->tq[endp].active); + if (status) { + pr_err("t2bce_vhci: stateful resume queue failed: dev=%d ep=%02x status=%d\n", + devid, vhci->devices[devid]->tq[endp].endp_addr, status); + return status; + } + } + } + + return 0; +} + +static int bce_vhci_resume_stateful(struct usb_hcd *hcd) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + int status; + + /* Linux preserved-state resume path. This is kept separate from the + * no-state rebuild path below. */ + vhci->port_change_pending = 0; + vhci->stateful_suspended_bounce = 0; + vhci->stateful_resume = true; + + bce_vhci_event_queue_resume(&vhci->ev_system); + bce_vhci_event_queue_resume(&vhci->ev_isochronous); + bce_vhci_event_queue_resume(&vhci->ev_interrupt); + bce_vhci_event_queue_resume(&vhci->ev_asynchronous); + + pr_debug("t2bce_vhci: stateful resume queues before firmware command events\n"); + status = bce_vhci_resume_all_queues(vhci); + pr_debug("t2bce_vhci: stateful resume enabling firmware command events status=%d\n", status); + bce_vhci_event_queue_resume(&vhci->ev_commands); + pr_info("t2bce_vhci: stateful resume exit status=%d\n", status); + return status; +} + +static int bce_vhci_bus_resume(struct usb_hcd *hcd) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + int status; + + pr_debug("t2bce_vhci: bus_resume entry no_state_resume=%d\n", vhci->no_state_resume); + vhci->port_change_pending = 0; + vhci->system_suspending = false; + pr_debug("t2bce_vhci: resume started\n"); + + if (vhci->no_state_resume) + status = bce_vhci_resume_no_state(hcd); + else + status = bce_vhci_resume_stateful(hcd); + + pr_info("t2bce_vhci: bus_resume exit status=%d no_state_resume=%d\n", status, vhci->no_state_resume); + return status; +} + +static int bce_vhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags) +{ + struct bce_vhci_transfer_queue *q = urb->ep->hcpriv; + if (!q) + return -ENOENT; + pr_debug("t2bce_vhci: urb_enqueue dev=%u ep=%02x\n", + q->dev_addr, urb->ep->desc.bEndpointAddress); + return bce_vhci_urb_create(q, urb); +} + +static int bce_vhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status) +{ + struct bce_vhci_transfer_queue *q = urb->ep->hcpriv; + if (!q) + return -ENOENT; + pr_debug("t2bce_vhci: urb_dequeue ep=%02x\n", urb->ep->desc.bEndpointAddress); + return bce_vhci_urb_request_cancel(q, urb, status); +} + +static void bce_vhci_endpoint_reset(struct usb_hcd *hcd, struct usb_host_endpoint *ep) +{ + struct bce_vhci_transfer_queue *q = ep->hcpriv; + pr_debug("t2bce_vhci: endpoint_reset\n"); + if (q) + bce_vhci_transfer_queue_request_reset(q); +} + +static u8 bce_vhci_endpoint_index(u8 addr) +{ + if (addr & 0x80) + return (u8) (0x10 + (addr & 0xf)); + return (u8) (addr & 0xf); +} + +static int bce_vhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev, struct usb_host_endpoint *endp) +{ + u8 endp_index = bce_vhci_endpoint_index(endp->desc.bEndpointAddress); + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + bce_vhci_device_t devid = vhci->port_to_device[udev->portnum]; + struct bce_vhci_device *vdev = vhci->devices[devid]; + pr_debug("t2bce_vhci: add_endpoint port=%u dev=%u ep_index=%u\n", + udev->portnum, devid, endp_index); + + if (udev->bus->root_hub == udev) /* The USB hub */ + return 0; + if (vdev == NULL) + return -ENODEV; + if (vdev->tq_mask & BIT(endp_index)) { + endp->hcpriv = &vdev->tq[endp_index]; + return 0; + } + + bce_vhci_create_transfer_queue(vhci, &vdev->tq[endp_index], endp, devid, + usb_endpoint_dir_in(&endp->desc) ? DMA_FROM_DEVICE : DMA_TO_DEVICE); + endp->hcpriv = &vdev->tq[endp_index]; + vdev->tq_mask |= BIT(endp_index); + + bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &endp->desc); + return 0; +} + +static int bce_vhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev, struct usb_host_endpoint *endp) +{ + u8 endp_index = bce_vhci_endpoint_index(endp->desc.bEndpointAddress); + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + bce_vhci_device_t devid = vhci->port_to_device[udev->portnum]; + struct bce_vhci_transfer_queue *q = endp->hcpriv; + struct bce_vhci_device *vdev; + pr_debug("t2bce_vhci: drop_endpoint port=%u ep_index=%u\n", udev->portnum, endp_index); + if (!devid || !vhci->devices[devid]) { + endp->hcpriv = NULL; + return 0; + } + vdev = vhci->devices[devid]; + if (!q) { + if (vdev->tq_mask & BIT(endp_index)) { + pr_err("t2bce_vhci: endpoint missing hcpriv while still tracked\n"); + q = &vdev->tq[endp_index]; + } else { + return 0; + } + } + + bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, (u8) (endp->desc.bEndpointAddress & 0x8Fu)); + vdev->tq_mask &= ~BIT(endp_index); + bce_vhci_destroy_transfer_queue(vhci, q); + endp->hcpriv = NULL; + return 0; +} + +static int bce_vhci_create_message_queues(struct bce_vhci *vhci) +{ + if (bce_vhci_message_queue_create(vhci, &vhci->msg_commands, "VHC1HostCommands") || + bce_vhci_message_queue_create(vhci, &vhci->msg_system, "VHC1HostSystemEvents") || + bce_vhci_message_queue_create(vhci, &vhci->msg_isochronous, "VHC1HostIsochronousEvents") || + bce_vhci_message_queue_create(vhci, &vhci->msg_interrupt, "VHC1HostInterruptEvents") || + bce_vhci_message_queue_create(vhci, &vhci->msg_asynchronous, "VHC1HostAsynchronousEvents")) { + bce_vhci_destroy_message_queues(vhci); + return -EINVAL; + } + spin_lock_init(&vhci->msg_asynchronous_lock); + bce_vhci_command_queue_create(&vhci->cq, &vhci->msg_commands); + return 0; +} + +static void bce_vhci_destroy_message_queues(struct bce_vhci *vhci) +{ + bce_vhci_command_queue_destroy(&vhci->cq); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_commands); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_system); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_isochronous); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_interrupt); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_asynchronous); +} + +static void bce_vhci_handle_system_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg); +static void bce_vhci_handle_usb_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg); + +static int bce_vhci_create_event_queues(struct bce_vhci *vhci) +{ + vhci->ev_cq = t2bce_core_create_cq(vhci->client, 0x100); + if (!vhci->ev_cq) + return -EINVAL; +#define CREATE_EVENT_QUEUE(field, name, cb) bce_vhci_event_queue_create(vhci, &vhci->field, name, cb) + if (__bce_vhci_event_queue_create(vhci, &vhci->ev_commands, "VHC1FirmwareCommands", + bce_vhci_firmware_event_completion) || + CREATE_EVENT_QUEUE(ev_system, "VHC1FirmwareSystemEvents", bce_vhci_handle_system_event) || + CREATE_EVENT_QUEUE(ev_isochronous, "VHC1FirmwareIsochronousEvents", bce_vhci_handle_usb_event) || + CREATE_EVENT_QUEUE(ev_interrupt, "VHC1FirmwareInterruptEvents", bce_vhci_handle_usb_event) || + CREATE_EVENT_QUEUE(ev_asynchronous, "VHC1FirmwareAsynchronousEvents", bce_vhci_handle_usb_event)) { + bce_vhci_destroy_event_queues(vhci); + return -EINVAL; + } +#undef CREATE_EVENT_QUEUE + return 0; +} + +static void bce_vhci_destroy_event_queues(struct bce_vhci *vhci) +{ + bce_vhci_event_queue_destroy(vhci, &vhci->ev_commands); + bce_vhci_event_queue_destroy(vhci, &vhci->ev_system); + bce_vhci_event_queue_destroy(vhci, &vhci->ev_isochronous); + bce_vhci_event_queue_destroy(vhci, &vhci->ev_interrupt); + bce_vhci_event_queue_destroy(vhci, &vhci->ev_asynchronous); + if (vhci->ev_cq) + t2bce_core_destroy_cq(vhci->client, vhci->ev_cq); +} + +static void bce_vhci_send_fw_event_response(struct bce_vhci *vhci, struct bce_vhci_message *req, u16 status) +{ + unsigned long timeout = 1000; + struct bce_vhci_message r = *req; + r.cmd = (u16) (req->cmd | 0x8000u); + r.status = status; + r.param1 = req->param1; + r.param2 = 0; + + if (t2bce_core_reserve_submission(vhci->msg_system.sq, &timeout)) { + pr_err("t2bce_vhci: Cannot reserve submision for FW event reply\n"); + return; + } + bce_vhci_message_queue_write(&vhci->msg_system, &r); +} + +static int bce_vhci_handle_firmware_event(struct bce_vhci *vhci, struct bce_vhci_message *msg) +{ + unsigned long flags; + bce_vhci_device_t devid; + u8 endp; + struct bce_vhci_device *dev; + struct bce_vhci_transfer_queue *tq; + if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE || msg->cmd == BCE_VHCI_CMD_ENDPOINT_SET_STATE) { + devid = (bce_vhci_device_t) (msg->param1 & 0xff); + endp = bce_vhci_endpoint_index((u8) ((msg->param1 >> 8) & 0xff)); + dev = vhci->devices[devid]; + if (!dev || !(dev->tq_mask & BIT(endp))) + return BCE_VHCI_BAD_ARGUMENT; + tq = &dev->tq[endp]; + } + + if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE) { + if (msg->param2 == BCE_VHCI_ENDPOINT_ACTIVE) { + int ret = bce_vhci_transfer_queue_resume(tq, BCE_VHCI_PAUSE_FIRMWARE); + + pr_debug("t2bce_vhci: FW endpoint request ACTIVE dev=%u ep=%02x paused_by=%x state=%x active=%u ret=%d reply=%u\n", + devid, tq->endp_addr, tq->paused_by, tq->state, tq->active, ret, + ret ? BCE_VHCI_ERROR : BCE_VHCI_SUCCESS); + return ret ? BCE_VHCI_ERROR : BCE_VHCI_SUCCESS; + } else if (msg->param2 == BCE_VHCI_ENDPOINT_PAUSED) { + int ret = bce_vhci_transfer_queue_pause(tq, BCE_VHCI_PAUSE_FIRMWARE); + + pr_debug("t2bce_vhci: FW endpoint request PAUSED dev=%u ep=%02x paused_by=%x state=%x active=%u ret=%d reply=%u\n", + devid, tq->endp_addr, tq->paused_by, tq->state, tq->active, ret, + ret ? BCE_VHCI_ERROR : BCE_VHCI_SUCCESS); + return ret ? BCE_VHCI_ERROR : BCE_VHCI_SUCCESS; + } + return BCE_VHCI_BAD_ARGUMENT; + } else if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_SET_STATE) { + if (msg->param2 == BCE_VHCI_ENDPOINT_STALLED) { + tq->state = msg->param2; + spin_lock_irqsave(&tq->urb_lock, flags); + tq->active = false; + tq->stalled = true; + spin_unlock_irqrestore(&tq->urb_lock, flags); + pr_warn("t2bce_vhci: firmware endpoint stalled dev=%u ep=%02x paused_by=%x; %s\n", + devid, tq->endp_addr, tq->paused_by, + tq->endp_addr == 0x00 ? "scheduling EP0 reset" : "leaving stalled"); + if (tq->endp_addr == 0x00) + bce_vhci_transfer_queue_request_reset(tq); + return BCE_VHCI_SUCCESS; + } + return BCE_VHCI_BAD_ARGUMENT; + } + pr_warn("t2bce_vhci: Unhandled firmware event: %x s=%x p1=%x p2=%llx\n", + msg->cmd, msg->status, msg->param1, msg->param2); + return BCE_VHCI_BAD_ARGUMENT; +} + +static void bce_vhci_handle_firmware_events_w(struct work_struct *ws) +{ + size_t cnt = 0; + int result; + struct bce_vhci *vhci = container_of(ws, struct bce_vhci, w_fw_events); + struct t2bce_core_queue_sq *sq = vhci->ev_commands.sq; + struct t2bce_core_sq_completion_data *cq; + struct bce_vhci_message *msg, *msg2 = NULL; + + while (true) { + if (msg2) { + msg = msg2; + msg2 = NULL; + } else if ((cq = t2bce_core_next_completion(sq))) { + if (cq->status == T2BCE_COMPLETION_ABORTED) { + t2bce_core_notify_submission_complete(sq); + continue; + } + msg = &vhci->ev_commands.data[t2bce_core_queue_sq_head(sq)]; + } else { + break; + } + + pr_debug("t2bce_vhci: Got fw event: %x s=%x p1=%x p2=%llx\n", msg->cmd, msg->status, msg->param1, msg->param2); + if ((cq = t2bce_core_next_completion(sq))) { + msg2 = &vhci->ev_commands.data[(t2bce_core_queue_sq_head(sq) + 1) % t2bce_core_queue_sq_capacity(sq)]; + pr_debug("t2bce_vhci: Got second fw event: %x s=%x p1=%x p2=%llx\n", + msg->cmd, msg->status, msg->param1, msg->param2); + if (cq->status != T2BCE_COMPLETION_ABORTED && + msg2->cmd == (msg->cmd | 0x4000) && msg2->param1 == msg->param1) { + /* Take two elements */ + pr_debug("t2bce_vhci: Cancelled\n"); + bce_vhci_send_fw_event_response(vhci, msg, BCE_VHCI_ABORT); + + t2bce_core_notify_submission_complete(sq); + t2bce_core_notify_submission_complete(sq); + msg2 = NULL; + cnt += 2; + continue; + } + + pr_warn("t2bce_vhci: Handle fw event - unexpected cancellation\n"); + } + + result = bce_vhci_handle_firmware_event(vhci, msg); + bce_vhci_send_fw_event_response(vhci, msg, (u16) result); + + + t2bce_core_notify_submission_complete(sq); + ++cnt; + } + bce_vhci_event_queue_submit_pending(&vhci->ev_commands, cnt); + if (t2bce_core_queue_sq_available(sq) == t2bce_core_queue_sq_capacity(sq) - 1) { + pr_debug("t2bce_vhci: complete\n"); + complete(&vhci->ev_commands.queue_empty_completion); + } +} + +static void bce_vhci_firmware_event_completion(struct t2bce_core_queue_sq *sq) +{ + struct bce_vhci_event_queue *q = t2bce_core_queue_sq_userdata(sq); + queue_work(q->vhci->tq_state_wq, &q->vhci->w_fw_events); +} + +static const char *bce_vhci_port_state_change_label(u64 p2) +{ + return (p2 & 0x4) ? "connected" : "disconnected"; +} + +static void bce_vhci_handle_system_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg) +{ + if (msg->cmd & 0x8000) { + bce_vhci_command_queue_deliver_completion(&q->vhci->cq, msg); + } else if (msg->cmd == BCE_VHCI_EV_PORT_STATUS_CHANGE && + msg->param1 > 0 && + msg->param1 < ARRAY_SIZE(q->vhci->port_to_device)) { + u32 normalized_status = (u32)msg->param2 & ~BCE_VHCI_PORT_STATUS_C_CONNECTION; + + if (q->vhci->stateful_resume && + (msg->param2 & BCE_VHCI_PORT_STATUS_CONNECTED) && + normalized_status == 0x285) { + set_bit(msg->param1 - 1, &q->vhci->stateful_suspended_bounce); + pr_debug("t2bce_vhci: marked stateful suspended bounce port=%u raw=%llx normalized=%x\n", + msg->param1, msg->param2, normalized_status); + } + set_bit(msg->param1 - 1, &q->vhci->port_change_pending); + pr_debug("t2bce_vhci: port state change event: port=%u state=%s status=%x p2=%llx\n", + msg->param1, bce_vhci_port_state_change_label(msg->param2), + msg->status, msg->param2); + } else { + pr_warn("t2bce_vhci: Unhandled system event: %x s=%x p1=%x p2=%llx\n", + msg->cmd, msg->status, msg->param1, msg->param2); + } +} + +static void bce_vhci_handle_usb_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg) +{ + bce_vhci_device_t devid; + u8 endp; + struct bce_vhci_device *dev; + if (msg->cmd & 0x8000) { + bce_vhci_command_queue_deliver_completion(&q->vhci->cq, msg); + } else if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST || msg->cmd == BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS) { + devid = (bce_vhci_device_t) (msg->param1 & 0xff); + endp = bce_vhci_endpoint_index((u8) ((msg->param1 >> 8) & 0xff)); + dev = q->vhci->devices[devid]; + if (!dev || (dev->tq_mask & BIT(endp)) == 0) { + pr_err("t2bce_vhci: Didn't find destination for transfer queue event\n"); + return; + } + bce_vhci_transfer_queue_event(&dev->tq[endp], msg); + } else { + pr_warn("t2bce_vhci: Unhandled USB event: %x s=%x p1=%x p2=%llx\n", + msg->cmd, msg->status, msg->param1, msg->param2); + } +} + + + +static const struct hc_driver bce_vhci_driver = { + .description = "t2bce_vhci", + .product_desc = "BCE VHCI Host Controller", + .hcd_priv_size = sizeof(struct bce_vhci *), + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0) + .flags = HCD_USB2, +#else + .flags = HCD_USB2 | HCD_DMA, +#endif + + .start = bce_vhci_start, + .stop = bce_vhci_stop, + .hub_status_data = bce_vhci_hub_status_data, + .hub_control = bce_vhci_hub_control, + .urb_enqueue = bce_vhci_urb_enqueue, + .urb_dequeue = bce_vhci_urb_dequeue, + .enable_device = bce_vhci_enable_device, + .free_dev = bce_vhci_free_device, + .address_device = bce_vhci_address_device, + .add_endpoint = bce_vhci_add_endpoint, + .drop_endpoint = bce_vhci_drop_endpoint, + .endpoint_reset = bce_vhci_endpoint_reset, + .check_bandwidth = bce_vhci_check_bandwidth, + .get_frame_number = bce_vhci_get_frame_number, + .bus_suspend = bce_vhci_bus_suspend, + .bus_resume = bce_vhci_bus_resume +}; + + +int __init bce_vhci_module_init(void) +{ + struct device *parent; + int result; + + if ((result = alloc_chrdev_region(&bce_vhci_chrdev, 0, 1, "t2bce_vhci"))) + goto fail_chrdev; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0) + bce_vhci_class = class_create(THIS_MODULE, "t2bce_vhci"); +#else + bce_vhci_class = class_create("t2bce_vhci"); +#endif + if (IS_ERR(bce_vhci_class)) { + result = PTR_ERR(bce_vhci_class); + goto fail_class; + } + + global_vhci = kzalloc(sizeof(*global_vhci), GFP_KERNEL); + if (!global_vhci) { + result = -ENOMEM; + goto fail_alloc; + } + + parent = t2bce_core_device_get(); + if (IS_ERR(parent)) { + result = PTR_ERR(parent); + goto fail_create; + } + + result = bce_vhci_create(parent, global_vhci); + t2bce_core_device_put(parent); + if (result) + goto fail_create; + + pr_info("t2bce_vhci: module initialized\n"); + return 0; + +fail_create: + kfree(global_vhci); + global_vhci = NULL; +fail_alloc: + class_destroy(bce_vhci_class); +fail_class: + unregister_chrdev_region(bce_vhci_chrdev, 1); +fail_chrdev: + if (!result) + result = -EINVAL; + pr_info("t2bce_vhci: module init failed status=%d\n", result); + return result; +} +void __exit bce_vhci_module_exit(void) +{ + if (global_vhci) { + bce_vhci_destroy(global_vhci); + kfree(global_vhci); + global_vhci = NULL; + } + class_destroy(bce_vhci_class); + unregister_chrdev_region(bce_vhci_chrdev, 1); + pr_info("t2bce_vhci: module exited\n"); +} + +module_param_named(vhci_port_mask, bce_vhci_port_mask, ushort, 0444); +MODULE_PARM_DESC(vhci_port_mask, "Specifies which VHCI ports are enabled"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("André Eikmeyer "); +MODULE_DESCRIPTION("Apple T2 BCE VHCI Driver"); +MODULE_VERSION("0.02"); +MODULE_SOFTDEP("pre: t2bce_core"); +module_init(bce_vhci_module_init); +module_exit(bce_vhci_module_exit); diff -ruN --no-dereference -x .git a/drivers/staging/t2bce/t2bce_vhci/vhci.h b/drivers/staging/t2bce/t2bce_vhci/vhci.h --- a/drivers/staging/t2bce/t2bce_vhci/vhci.h 1970-01-01 01:00:00.000000000 +0100 +++ b/drivers/staging/t2bce/t2bce_vhci/vhci.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,69 @@ +#ifndef BCE_VHCI_H +#define BCE_VHCI_H + +#include "queue.h" +#include "transfer.h" + +struct device; +struct usb_hcd; +struct t2bce_core_client; + +struct bce_vhci_device { + struct bce_vhci_transfer_queue tq[32]; + u32 tq_mask; +}; +struct bce_vhci { + struct t2bce_core_client *client; + dev_t vdevt; + struct device *vdev; + struct usb_hcd *hcd; + struct spinlock hcd_spinlock; + struct bce_vhci_message_queue msg_commands; + struct bce_vhci_message_queue msg_system; + struct bce_vhci_message_queue msg_isochronous; + struct bce_vhci_message_queue msg_interrupt; + struct bce_vhci_message_queue msg_asynchronous; + struct spinlock msg_asynchronous_lock; + struct bce_vhci_command_queue cq; + struct t2bce_core_queue_cq *ev_cq; + struct bce_vhci_event_queue ev_commands; + struct bce_vhci_event_queue ev_system; + struct bce_vhci_event_queue ev_isochronous; + struct bce_vhci_event_queue ev_interrupt; + struct bce_vhci_event_queue ev_asynchronous; + u16 port_mask; + u8 port_count; + u16 port_power_mask; + bce_vhci_device_t port_to_device[17]; + struct bce_vhci_device *devices[17]; + struct workqueue_struct *tq_state_wq; + struct work_struct w_fw_events; + struct work_struct w_add_hcd; + unsigned long port_change_pending; + unsigned long stateful_suspended_bounce; + bool stateful_resume; + bool no_state_resume; + bool hcd_registered; + bool system_suspending; +}; + +int __init bce_vhci_module_init(void); +void __exit bce_vhci_module_exit(void); + +int bce_vhci_create(struct device *parent, struct bce_vhci *vhci); +void bce_vhci_destroy(struct bce_vhci *vhci); +int bce_vhci_start(struct usb_hcd *hcd); +void bce_vhci_stop(struct usb_hcd *hcd); +int bce_vhci_add_hcd(struct bce_vhci *vhci); +void bce_vhci_remove_hcd(struct bce_vhci *vhci); +void bce_vhci_shutdown(struct bce_vhci *vhci); +void bce_vhci_pm_reset(struct bce_vhci *vhci); +void bce_vhci_pm_prepare(struct bce_vhci *vhci); +void bce_vhci_pm_prepare_no_state(struct bce_vhci *vhci); +void bce_vhci_pm_mark_no_state_resume(struct bce_vhci *vhci); +bool bce_vhci_pm_is_no_state_resume(struct bce_vhci *vhci); +void bce_vhci_pm_complete(struct bce_vhci *vhci); + +struct bce_vhci *bce_vhci_from_hcd(struct usb_hcd *hcd); + +#endif //BCE_VHCI_H diff -ruN --no-dereference -x .git a/drivers/usb/core/message.c b/drivers/usb/core/message.c --- a/drivers/usb/core/message.c 2026-10-11 15:27:50.000000000 +0200 +++ b/drivers/usb/core/message.c 2026-10-13 14:33:30.000000000 +0200 @@ -1052,6 +1052,25 @@ UTF16_LITTLE_ENDIAN, buf, size); buf[err] = 0; + /* + * Some devices report string descriptors with a declared length + * greater than the actual serial, leaving uninitialized firmware + * memory (often including C0 control characters) appended to the + * returned string. Truncate at the first control character so + * callers get a clean, well-formed string. + */ + { + int i; + for (i = 0; i < err; i++) { + unsigned char c = buf[i]; + if (c < 0x20 || c == 0x7f) { + buf[i] = 0; + err = i; + break; + } + } + } + if (tbuf[1] != USB_DT_STRING) dev_dbg(&dev->dev, "wrong descriptor type %02x for string %d (\"%s\")\n", diff -ruN --no-dereference -x .git a/fs/readdir.c b/fs/readdir.c --- a/fs/readdir.c 2026-10-11 15:27:50.000000000 +0200 +++ b/fs/readdir.c 2026-10-13 14:33:30.000000000 +0200 @@ -149,7 +149,7 @@ */ static int verify_dirent_name(const char *name, int len) { - if (len <= 0 || len >= PATH_MAX) + if (unlikely(len <= 0 || len >= PATH_MAX)) return -EIO; if (memchr(name, '/', len)) return -EIO; diff -ruN --no-dereference -x .git a/include/drm/drm_connector.h b/include/drm/drm_connector.h --- a/include/drm/drm_connector.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/drm/drm_connector.h 2026-10-13 14:33:30.000000000 +0200 @@ -255,6 +255,44 @@ }; /** + * struct drm_hdmi_vrr_cap - Information about VRR capabilities of a HDMI sink + * + * Describes the VRR support provided by HDMI 2.1 sink. The information is + * fetched fom additional HFVSDB blocks defined for HDMI 2.1. + */ +struct drm_hdmi_vrr_cap { + /** @fva: flag for Fast VActive (Quick Frame Transport) support */ + bool fva; + + /** @mcnmvrr: flag for Negative M VRR support */ + bool cnmvrr; + + /** @mcinema_vrr: flag for Cinema VRR support */ + bool cinema_vrr; + + /** @mdelta: flag for limited frame-to-frame compensation support */ + bool mdelta; + + /** + * @vrr_min : minimum supported variable refresh rate in Hz. + * Valid values only inide 1 - 48 range + */ + u16 vrr_min; + + /** + * @vrr_max : maximum supported variable refresh rate in Hz (optional). + * Valid values are either 0 (max based on video mode) or >= 100 + */ + u16 vrr_max; + + /** + * @supported: flag for vrr support based on checking for VRRmin and + * VRRmax values having correct values. + */ + bool supported; +}; + +/** * struct drm_hdmi_dsc_cap - DSC capabilities of HDMI sink * * Describes the DSC support provided by HDMI 2.1 sink. @@ -330,6 +368,15 @@ /** @max_lanes: supported by sink */ u8 max_lanes; + /** @fapa_start_location: flag for the FAPA in blanking support */ + bool fapa_start_location; + + /** @allm: flag for Auto Low Latency Mode support by sink */ + bool allm; + + /** @vrr_cap: VRR capabilities of the sink */ + struct drm_hdmi_vrr_cap vrr_cap; + /** @dsc_cap: DSC capabilities of the sink */ struct drm_hdmi_dsc_cap dsc_cap; }; @@ -893,6 +940,12 @@ u32 max_dsc_bpp; /** + * @dp_dsc_bpp: DP Display-Stream-Compression (DSC) timing's target + * DSC bits per pixel in 6.4 fixed point format. 0 means undefined. + */ + u16 dp_dsc_bpp_x16; + + /** * @vics: Array of vics_len VICs. Internal to EDID parsing. */ u8 *vics; diff -ruN --no-dereference -x .git a/include/drm/drm_modes.h b/include/drm/drm_modes.h --- a/include/drm/drm_modes.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/drm/drm_modes.h 2026-10-13 14:33:30.000000000 +0200 @@ -417,6 +417,16 @@ */ enum hdmi_picture_aspect picture_aspect_ratio; + /** + * @dsc_passthrough_timing_support: + * + * Indicates whether this mode timing descriptor is supported + * with specific target DSC bits per pixel only. + * + * VESA vendor-specific data block shall exist with the relevant + * DSC bits per pixel declaration when this flag is set to true. + */ + bool dsc_passthrough_timings_support; }; /** diff -ruN --no-dereference -x .git a/include/drm/ttm/ttm_resource.h b/include/drm/ttm/ttm_resource.h --- a/include/drm/ttm/ttm_resource.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/drm/ttm/ttm_resource.h 2026-10-13 14:33:30.000000000 +0200 @@ -458,10 +458,14 @@ void ttm_resource_fini(struct ttm_resource_manager *man, struct ttm_resource *res); +int ttm_resource_try_charge(struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct dmem_cgroup_pool_state **ret_pool, + struct dmem_cgroup_pool_state **ret_limit_pool); int ttm_resource_alloc(struct ttm_buffer_object *bo, const struct ttm_place *place, struct ttm_resource **res, - struct dmem_cgroup_pool_state **ret_limit_pool); + struct dmem_cgroup_pool_state *charge_pool); void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res); bool ttm_resource_intersects(struct ttm_device *bdev, struct ttm_resource *res, diff -ruN --no-dereference -x .git a/include/linux/cachy-t2.h b/include/linux/cachy-t2.h --- a/include/linux/cachy-t2.h 1970-01-01 01:00:00.000000000 +0100 +++ b/include/linux/cachy-t2.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,35 @@ +#ifndef CACHYOS_T2_H +#define CACHYOS_T2_H + +#include + + +#ifndef T2_MAC +#define T2_MAC(vendor, product) \ + .matches = { \ + DMI_MATCH(DMI_BOARD_VENDOR, vendor), \ + DMI_MATCH(DMI_PRODUCT_NAME, product), \ + }, +#endif + +static const struct dmi_system_id t2_mac_tbl[] = { + { T2_MAC("Apple Inc.", "MacBookPro15,1") }, + { T2_MAC("Apple Inc.", "MacBookPro15,2") }, + { T2_MAC("Apple Inc.", "MacBookPro15,3") }, + { T2_MAC("Apple Inc.", "MacBookPro15,4") }, + { T2_MAC("Apple Inc.", "MacBookPro16,1") }, + { T2_MAC("Apple Inc.", "MacBookPro16,2") }, + { T2_MAC("Apple Inc.", "MacBookPro16,3") }, + { T2_MAC("Apple Inc.", "MacBookPro16,4") }, + { T2_MAC("Apple Inc.", "MacBookAir8,1") }, + { T2_MAC("Apple Inc.", "MacBookAir8,2") }, + { T2_MAC("Apple Inc.", "MacBookAir9,1") }, + { T2_MAC("Apple Inc.", "Macmini8,1") }, + { T2_MAC("Apple Inc.", "MacPro7,1") }, + { T2_MAC("Apple Inc.", "iMac20,1") }, + { T2_MAC("Apple Inc.", "iMac20,2") }, + { T2_MAC("Apple Inc.", "iMacPro1,1") }, + { } +}; + +#endif diff -ruN --no-dereference -x .git a/include/linux/cgroup.h b/include/linux/cgroup.h --- a/include/linux/cgroup.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/linux/cgroup.h 2026-10-13 14:33:30.000000000 +0200 @@ -624,6 +624,27 @@ } /** + * cgroup_common_ancestor - find common ancestor of two cgroups + * @a: first cgroup to find common ancestor of + * @b: second cgroup to find common ancestor of + * + * Find the first cgroup that is an ancestor of both @a and @b, if it exists + * and return a pointer to it. If such a cgroup doesn't exist, return NULL. + * + * This function is safe to call as long as both @a and @b are accessible. + */ +static inline struct cgroup *cgroup_common_ancestor(struct cgroup *a, + struct cgroup *b) +{ + int level; + + for (level = min(a->level, b->level); level >= 0; level--) + if (a->ancestors[level] == b->ancestors[level]) + return a->ancestors[level]; + return NULL; +} + +/** * task_under_cgroup_hierarchy - test task's membership of cgroup ancestry * @task: the task to be tested * @ancestor: possible ancestor of @task's cgroup diff -ruN --no-dereference -x .git a/include/linux/cgroup_dmem.h b/include/linux/cgroup_dmem.h --- a/include/linux/cgroup_dmem.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/linux/cgroup_dmem.h 2026-10-13 14:33:30.000000000 +0200 @@ -24,6 +24,12 @@ bool dmem_cgroup_state_evict_valuable(struct dmem_cgroup_pool_state *limit_pool, struct dmem_cgroup_pool_state *test_pool, bool ignore_low, bool *ret_hit_low); +bool dmem_cgroup_below_min(struct dmem_cgroup_pool_state *root, + struct dmem_cgroup_pool_state *test); +bool dmem_cgroup_below_low(struct dmem_cgroup_pool_state *root, + struct dmem_cgroup_pool_state *test); +struct dmem_cgroup_pool_state *dmem_cgroup_get_common_ancestor(struct dmem_cgroup_pool_state *a, + struct dmem_cgroup_pool_state *b); void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool); #else @@ -59,6 +65,25 @@ return true; } +static inline bool dmem_cgroup_below_min(struct dmem_cgroup_pool_state *root, + struct dmem_cgroup_pool_state *test) +{ + return false; +} + +static inline bool dmem_cgroup_below_low(struct dmem_cgroup_pool_state *root, + struct dmem_cgroup_pool_state *test) +{ + return false; +} + +static inline +struct dmem_cgroup_pool_state *dmem_cgroup_get_common_ancestor(struct dmem_cgroup_pool_state *a, + struct dmem_cgroup_pool_state *b) +{ + return NULL; +} + static inline void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool) { } diff -ruN --no-dereference -x .git a/include/linux/cpuset.h b/include/linux/cpuset.h --- a/include/linux/cpuset.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/linux/cpuset.h 2026-10-13 14:33:30.000000000 +0200 @@ -80,6 +80,7 @@ extern void cpuset_cpus_allowed_locked(struct task_struct *p, struct cpumask *mask); extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask); extern bool cpuset_cpus_allowed_fallback(struct task_struct *p); +extern int cpuset_num_cpus(struct cgroup *cgroup); extern nodemask_t cpuset_mems_allowed(struct task_struct *p); #define cpuset_current_mems_allowed (current->mems_allowed) void cpuset_init_current_mems_allowed(void); @@ -216,6 +217,11 @@ return false; } +static inline int cpuset_num_cpus(struct cgroup *cgroup) +{ + return num_online_cpus(); +} + static inline nodemask_t cpuset_mems_allowed(struct task_struct *p) { return node_possible_map; diff -ruN --no-dereference -x .git a/include/linux/hid.h b/include/linux/hid.h --- a/include/linux/hid.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/linux/hid.h 2026-10-13 14:33:30.000000000 +0200 @@ -625,7 +625,9 @@ enum hid_type { HID_TYPE_OTHER = 0, HID_TYPE_USBMOUSE, - HID_TYPE_USBNONE + HID_TYPE_USBNONE, + HID_TYPE_SPI_KEYBOARD, + HID_TYPE_SPI_MOUSE, }; enum hid_battery_status { @@ -821,6 +823,8 @@ .bus = BUS_BLUETOOTH, .vendor = (ven), .product = (prod) #define HID_I2C_DEVICE(ven, prod) \ .bus = BUS_I2C, .vendor = (ven), .product = (prod) +#define HID_SPI_DEVICE(ven, prod) \ + .bus = BUS_SPI, .vendor = (ven), .product = (prod) #define HID_REPORT_ID(rep) \ .report_type = (rep) diff -ruN --no-dereference -x .git a/include/linux/pagemap.h b/include/linux/pagemap.h --- a/include/linux/pagemap.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/linux/pagemap.h 2026-10-13 14:33:30.000000000 +0200 @@ -1363,7 +1363,7 @@ ._index = i, \ } -#define VM_READAHEAD_PAGES (SZ_128K / PAGE_SIZE) +#define VM_READAHEAD_PAGES (SZ_256K / PAGE_SIZE) void page_cache_ra_unbounded(struct readahead_control *, unsigned long nr_to_read, unsigned long lookahead_count); diff -ruN --no-dereference -x .git a/include/linux/perf_event.h b/include/linux/perf_event.h --- a/include/linux/perf_event.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/linux/perf_event.h 2026-10-13 14:33:30.000000000 +0200 @@ -1616,7 +1616,7 @@ task->sched_migrated = 1; } -static inline void perf_event_task_sched_in(struct task_struct *prev, +static __always_inline void perf_event_task_sched_in(struct task_struct *prev, struct task_struct *task) { if (static_branch_unlikely(&perf_sched_events)) diff -ruN --no-dereference -x .git a/include/linux/rmap.h b/include/linux/rmap.h --- a/include/linux/rmap.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/linux/rmap.h 2026-10-13 14:33:30.000000000 +0200 @@ -843,7 +843,7 @@ * Called from mm/vmscan.c to handle paging out */ int folio_referenced(struct folio *, int is_locked, - struct mem_cgroup *memcg, vm_flags_t *vm_flags); + struct mem_cgroup *memcg, vma_flags_t *vma_flags); void try_to_migrate(struct folio *folio, enum ttu_flags flags); void try_to_unmap(struct folio *, enum ttu_flags flags); @@ -975,10 +975,9 @@ #define anon_vma_prepare(vma) (0) static inline int folio_referenced(struct folio *folio, int is_locked, - struct mem_cgroup *memcg, - vm_flags_t *vm_flags) + struct mem_cgroup *memcg, vma_flags_t *vma_flags) { - *vm_flags = 0; + vma_flags_clear_all(vma_flags); return 0; } diff -ruN --no-dereference -x .git a/include/linux/sched/mm.h b/include/linux/sched/mm.h --- a/include/linux/sched/mm.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/linux/sched/mm.h 2026-10-13 14:33:30.000000000 +0200 @@ -32,7 +32,7 @@ * See also for an in-depth explanation * of &mm_struct.mm_count vs &mm_struct.mm_users. */ -static inline void mmgrab(struct mm_struct *mm) +static __always_inline void mmgrab(struct mm_struct *mm) { atomic_inc(&mm->mm_count); } @@ -44,7 +44,7 @@ extern void __mmdrop(struct mm_struct *mm); -static inline void mmdrop(struct mm_struct *mm) +static __always_inline void mmdrop(struct mm_struct *mm) { /* * The implicit full barrier implied by atomic_dec_and_test() is @@ -71,27 +71,27 @@ * Invoked from finish_task_switch(). Delegates the heavy lifting on RT * kernels via RCU. */ -static inline void mmdrop_sched(struct mm_struct *mm) +static __always_inline void mmdrop_sched(struct mm_struct *mm) { /* Provides a full memory barrier. See mmdrop() */ if (atomic_dec_and_test(&mm->mm_count)) call_rcu(&mm->delayed_drop, __mmdrop_delayed); } #else -static inline void mmdrop_sched(struct mm_struct *mm) +static __always_inline void mmdrop_sched(struct mm_struct *mm) { mmdrop(mm); } #endif /* Helpers for lazy TLB mm refcounting */ -static inline void mmgrab_lazy_tlb(struct mm_struct *mm) +static __always_inline void mmgrab_lazy_tlb(struct mm_struct *mm) { if (IS_ENABLED(CONFIG_MMU_LAZY_TLB_REFCOUNT)) mmgrab(mm); } -static inline void mmdrop_lazy_tlb(struct mm_struct *mm) +static __always_inline void mmdrop_lazy_tlb(struct mm_struct *mm) { if (IS_ENABLED(CONFIG_MMU_LAZY_TLB_REFCOUNT)) { mmdrop(mm); @@ -104,7 +104,7 @@ } } -static inline void mmdrop_lazy_tlb_sched(struct mm_struct *mm) +static __always_inline void mmdrop_lazy_tlb_sched(struct mm_struct *mm) { if (IS_ENABLED(CONFIG_MMU_LAZY_TLB_REFCOUNT)) mmdrop_sched(mm); @@ -128,12 +128,12 @@ * See also for an in-depth explanation * of &mm_struct.mm_count vs &mm_struct.mm_users. */ -static inline void mmget(struct mm_struct *mm) +static __always_inline void mmget(struct mm_struct *mm) { atomic_inc(&mm->mm_users); } -static inline bool mmget_not_zero(struct mm_struct *mm) +static __always_inline bool mmget_not_zero(struct mm_struct *mm) { return atomic_inc_not_zero(&mm->mm_users); } @@ -532,7 +532,7 @@ #include #endif -static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) +static __always_inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) { /* * The atomic_read() below prevents CSE. The following should diff -ruN --no-dereference -x .git a/include/linux/sched/topology.h b/include/linux/sched/topology.h --- a/include/linux/sched/topology.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/linux/sched/topology.h 2026-10-13 14:33:30.000000000 +0200 @@ -86,6 +86,52 @@ unsigned long util_avg; unsigned long capacity; #endif +#ifdef CONFIG_SCHED_POC_SELECTOR + /* + * POC Selector: per-LLC idle CPU tracking + */ + u64 poc_llc_members; /* bitmask of valid CPUs (relative to base) */ + int poc_cpu_base; /* smallest CPU ID in this LLC */ + u8 poc_affinity_shift; /* bit shift for cpumask alignment */ + bool poc_fast_eligible; /* true when LLC CPU count <= 64 */ + bool poc_cluster_valid; /* true when cluster mask is usable */ +#ifdef CONFIG_SCHED_SMT + u8 poc_smt_shift; /* bit distance between SMT siblings */ + u64 poc_primary_mask; /* bitmask of core representative CPUs */ +#endif + + /* + * Hot write path: idle state flag arrays (lock-free mode). + * Each array = exactly 1 cache line (64B). + * Writers: WRITE_ONCE (plain MOV, no LOCK prefix). + * Readers: snapshot to stack, then multiply-and-shift aggregation. + * Active only when sched_poc_atomic_bitmap=0. + */ + u8 poc_idle_cpus[64] ____cacheline_aligned; +#ifdef CONFIG_SCHED_SMT + u8 poc_idle_cores[64] ____cacheline_aligned; +#endif /* CONFIG_SCHED_SMT */ + + /* + * Hot read/write path: idle state bitmaps (bitmap mode, default). + * Readers: single atomic64_read (MOV on x86). + * Writers: atomic64_or / atomic64_andnot (LOCK'd on x86). + * Active only when sched_poc_atomic_bitmap=1. + */ + atomic64_t poc_idle_cpus_mask ____cacheline_aligned; +#ifdef CONFIG_SCHED_SMT + atomic64_t poc_idle_cores_mask ____cacheline_aligned; +#endif /* CONFIG_SCHED_SMT */ + + /* + * Read-only lookup tables (written once at init). + * Cacheline-aligned for exact prefetch targeting. + */ + u64 poc_cluster_mask[64] ____cacheline_aligned; +#ifdef CONFIG_SCHED_SMT + u64 poc_smt_mask[64] ____cacheline_aligned; +#endif /* CONFIG_SCHED_SMT */ +#endif /* CONFIG_SCHED_POC_SELECTOR */ }; struct sched_domain { diff -ruN --no-dereference -x .git a/include/linux/sched.h b/include/linux/sched.h --- a/include/linux/sched.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/linux/sched.h 2026-10-13 14:33:30.000000000 +0200 @@ -575,6 +575,7 @@ struct sched_entity { /* For load-balancing: */ struct load_weight load; + struct load_weight h_load; struct rb_node run_node; u64 deadline; u64 min_vruntime; @@ -823,6 +824,17 @@ #endif }; +#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPTION) +struct task_ipi_mask { + union { + cpumask_t *ipi_mask_ptr; + unsigned long ipi_mask_val; + }; +}; +#else +struct task_ipi_mask { }; +#endif + struct task_struct { #ifdef CONFIG_THREAD_INFO_IN_TASK /* @@ -1359,6 +1371,7 @@ struct list_head perf_event_list; struct perf_ctx_data __rcu *perf_ctx_data; #endif + struct task_ipi_mask __private ipi_mask; #ifdef CONFIG_DEBUG_PREEMPT unsigned long preempt_disable_ip; #endif diff -ruN --no-dereference -x .git a/include/linux/smp.h b/include/linux/smp.h --- a/include/linux/smp.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/linux/smp.h 2026-10-13 14:33:30.000000000 +0200 @@ -47,8 +47,7 @@ /* total number of cpus in this system (may exceed NR_CPUS) */ extern unsigned int total_cpus; -int smp_call_function_single(int cpuid, smp_call_func_t func, void *info, - int wait); +int smp_call_function_single(int cpuid, smp_call_func_t func, void *info, bool wait); void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func, void *info, bool wait, const struct cpumask *mask); @@ -239,6 +238,18 @@ #endif /* !SMP */ +#if defined(CONFIG_PREEMPTION) && defined(CONFIG_SMP) +int smp_task_ipi_mask_alloc(struct task_struct *task); +void smp_task_ipi_mask_free(struct task_struct *task); +#else +static inline int smp_task_ipi_mask_alloc(struct task_struct *task) +{ + return 0; +} + +static inline void smp_task_ipi_mask_free(struct task_struct *task) { } +#endif + /* * raw_smp_processor_id() - get the current (unstable) CPU id * diff -ruN --no-dereference -x .git a/include/linux/soc/apple/dockchannel.h b/include/linux/soc/apple/dockchannel.h --- a/include/linux/soc/apple/dockchannel.h 1970-01-01 01:00:00.000000000 +0100 +++ b/include/linux/soc/apple/dockchannel.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Apple Dockchannel devices + * Copyright (C) The Asahi Linux Contributors + */ +#ifndef _LINUX_APPLE_DOCKCHANNEL_H_ +#define _LINUX_APPLE_DOCKCHANNEL_H_ + +#include +#include +#include + +#if IS_ENABLED(CONFIG_APPLE_DOCKCHANNEL) + +struct dockchannel; + +struct dockchannel *dockchannel_init(struct platform_device *pdev); + +int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t count); +int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count); +int dockchannel_await(struct dockchannel *dockchannel, + void (*callback)(void *cookie, size_t avail), + void *cookie, size_t count); + +#endif +#endif diff -ruN --no-dereference -x .git a/include/linux/tcp.h b/include/linux/tcp.h --- a/include/linux/tcp.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/linux/tcp.h 2026-10-13 14:33:30.000000000 +0200 @@ -232,7 +232,8 @@ tcp_usec_ts : 1, /* TSval values in usec */ is_sack_reneg:1, /* in recovery from loss with SACK reneg? */ is_cwnd_limited:1,/* forward progress limited by snd_cwnd? */ - recvmsg_inq : 1;/* Indicate # of bytes in queue upon recvmsg */ + recvmsg_inq : 1,/* Indicate # of bytes in queue upon recvmsg */ + fast_ack_mode:1;/* ack ASAP if >1 rcv_mss received? */ __cacheline_group_end(tcp_sock_read_txrx); /* RX read-mostly hotpath cache lines */ @@ -298,7 +299,8 @@ * 0x5?10 << 16 + snd_wnd in net byte order */ u8 nonagle : 4,/* Disable Nagle algorithm? */ - rate_app_limited:1; /* rate_{delivered,interval_us} limited? */ + rate_app_limited:1, /* rate_{delivered,interval_us} limited? */ + tlp_orig_data_app_limited:1; /* app-limited before TLP rtx? */ u8 received_ce_pending:4, /* Not yet transmit cnt of received_ce */ accecn_opt_sent_w_dsack:1,/* Sent ACCECN opt in previous ACK w/ D-SACK */ unused2:3; diff -ruN --no-dereference -x .git a/include/linux/tick.h b/include/linux/tick.h --- a/include/linux/tick.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/linux/tick.h 2026-10-13 14:33:30.000000000 +0200 @@ -175,7 +175,7 @@ #ifdef CONFIG_NO_HZ_FULL extern bool tick_nohz_full_running; -static inline bool tick_nohz_full_enabled(void) +static __always_inline bool tick_nohz_full_enabled(void) { if (!context_tracking_enabled()) return false; @@ -299,7 +299,7 @@ static inline void tick_nohz_full_setup(cpumask_var_t cpumask) { } #endif -static inline void tick_nohz_task_switch(void) +static __always_inline void tick_nohz_task_switch(void) { if (tick_nohz_full_enabled()) __tick_nohz_task_switch(); diff -ruN --no-dereference -x .git a/include/linux/vtime.h b/include/linux/vtime.h --- a/include/linux/vtime.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/linux/vtime.h 2026-10-13 14:33:30.000000000 +0200 @@ -89,24 +89,24 @@ * For now vtime state is tied to context tracking. We might want to decouple * those later if necessary. */ -static inline bool vtime_accounting_enabled(void) +static __always_inline bool vtime_accounting_enabled(void) { return context_tracking_enabled(); } -static inline bool vtime_accounting_enabled_cpu(int cpu) +static __always_inline bool vtime_accounting_enabled_cpu(int cpu) { return vtime_generic_enabled_cpu(cpu); } -static inline bool vtime_accounting_enabled_this_cpu(void) +static __always_inline bool vtime_accounting_enabled_this_cpu(void) { return vtime_generic_enabled_this_cpu(); } extern void vtime_task_switch_generic(struct task_struct *prev); -static inline void vtime_task_switch(struct task_struct *prev) +static __always_inline void vtime_task_switch(struct task_struct *prev) { if (vtime_accounting_enabled_this_cpu()) vtime_task_switch_generic(prev); diff -ruN --no-dereference -x .git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h --- a/include/net/inet_connection_sock.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/net/inet_connection_sock.h 2026-10-13 14:33:30.000000000 +0200 @@ -138,8 +138,8 @@ u32 icsk_probes_tstamp; u32 icsk_user_timeout; - u64 icsk_ca_priv[104 / sizeof(u64)]; -#define ICSK_CA_PRIV_SIZE sizeof_field(struct inet_connection_sock, icsk_ca_priv) +#define ICSK_CA_PRIV_SIZE (144) + u64 icsk_ca_priv[ICSK_CA_PRIV_SIZE / sizeof(u64)]; }; #define ICSK_TIME_RETRANS 1 /* Retransmit timer */ diff -ruN --no-dereference -x .git a/include/net/tcp.h b/include/net/tcp.h --- a/include/net/tcp.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/net/tcp.h 2026-10-13 14:33:30.000000000 +0200 @@ -428,6 +428,8 @@ #define TCP_ECN_DEMAND_CWR BIT(2) #define TCP_ECN_SEEN BIT(3) #define TCP_ECN_MODE_ACCECN BIT(4) +#define TCP_ECN_LOW BIT(5) +#define TCP_ECN_ECT_PERMANENT BIT(6) #define TCP_ECN_DISABLED 0 #define TCP_ECN_MODE_PENDING (TCP_ECN_MODE_RFC3168 | TCP_ECN_MODE_ACCECN) @@ -891,6 +893,15 @@ u32 tcp_delack_max(const struct sock *sk); +static inline void tcp_set_ecn_low_from_dst(struct sock *sk, + const struct dst_entry *dst) +{ + struct tcp_sock *tp = tcp_sk(sk); + + if (dst_feature(dst, RTAX_FEATURE_ECN_LOW)) + tp->ecn_flags |= TCP_ECN_LOW; +} + /* Compute the actual rto_min value */ static inline u32 tcp_rto_min(const struct sock *sk) { @@ -1028,6 +1039,11 @@ return max_t(s64, t1 - t0, 0); } +static inline u32 tcp_stamp32_us_delta(u32 t1, u32 t0) +{ + return max_t(s32, t1 - t0, 0); +} + /* provide the departure time in us unit */ static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb) { @@ -1141,9 +1157,14 @@ /* pkts S/ACKed so far upon tx of skb, incl retrans: */ __u32 delivered; /* start of send pipeline phase */ - u64 first_tx_mstamp; + u32 first_tx_mstamp; /* when we reached the "delivered" count */ - u64 delivered_mstamp; + u32 delivered_mstamp; +#define TCPCB_IN_FLIGHT_BITS 20 +#define TCPCB_IN_FLIGHT_MAX ((1U << TCPCB_IN_FLIGHT_BITS) - 1) + u32 in_flight:20, /* packets in flight at transmit */ + unused2:12; + u32 lost; /* packets lost so far upon tx of skb */ } tx; /* only used for outgoing skbs */ union { struct inet_skb_parm h4; @@ -1254,6 +1275,7 @@ CA_EVENT_LOSS, /* loss timeout */ CA_EVENT_ECN_NO_CE, /* ECT set, but not CE marked */ CA_EVENT_ECN_IS_CE, /* received CE marked IP packet */ + CA_EVENT_TLP_RECOVERY, /* a lost segment was repaired by TLP probe */ }; /* Information about inbound ACK, passed to cong_ops->in_ack_event() */ @@ -1282,9 +1304,11 @@ #define TCP_CONG_ECT_1_NEGOTIATION BIT(3) /* Cannot fallback to RFC3168 during AccECN negotiation */ #define TCP_CONG_NO_FALLBACK_RFC3168 BIT(4) +/* Wants notification of CE events (CA_EVENT_ECN_IS_CE, CA_EVENT_ECN_NO_CE). */ +#define TCP_CONG_WANTS_CE_EVENTS BIT(5) #define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN | \ TCP_CONG_NEEDS_ACCECN | TCP_CONG_ECT_1_NEGOTIATION | \ - TCP_CONG_NO_FALLBACK_RFC3168) + TCP_CONG_NO_FALLBACK_RFC3168 | TCP_CONG_WANTS_CE_EVENTS) union tcp_cc_info; @@ -1304,10 +1328,13 @@ */ struct rate_sample { u64 prior_mstamp; /* starting timestamp for interval */ + u32 prior_lost; /* tp->lost at "prior_mstamp" */ u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */ + u32 tx_in_flight; /* packets in flight at starting timestamp */ + s32 lost; /* number of packets lost over interval */ s32 delivered; /* number of packets delivered over interval */ - s32 delivered_ce; /* number of packets delivered w/ CE marks*/ + s32 delivered_ce; /* packets delivered w/ CE mark over interval */ long interval_us; /* time for tp->delivered to incr "delivered" */ u32 snd_interval_us; /* snd interval for delivered packets */ u32 rcv_interval_us; /* rcv interval for delivered packets */ @@ -1318,7 +1345,9 @@ u32 last_end_seq; /* end_seq of most recently ACKed packet */ bool is_app_limited; /* is sample from packet with bubble in pipe? */ bool is_retrans; /* is sample from retransmission? */ + bool is_acking_tlp_retrans_seq; /* ACKed a TLP retransmit sequence? */ bool is_ack_delayed; /* is this (likely) a delayed ACK? */ + bool is_ece; /* did this ACK have ECN marked? */ }; struct tcp_congestion_ops { @@ -1360,8 +1389,11 @@ /* hook for packet ack accounting (optional) */ void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); - /* override sysctl_tcp_min_tso_segs (optional) */ - u32 (*min_tso_segs)(struct sock *sk); + /* pick target number of segments per TSO/GSO skb (optional): */ + u32 (*tso_segs)(struct sock *sk, unsigned int mss_now); + + /* react to a specific lost skb (optional) */ + void (*skb_marked_lost)(struct sock *sk, const struct sk_buff *skb); /* new value of cwnd after loss (required) */ u32 (*undo_cwnd)(struct sock *sk); @@ -1421,6 +1453,14 @@ } #endif +static inline bool tcp_ca_wants_ce_events(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & (TCP_CONG_NEEDS_ECN | + TCP_CONG_WANTS_CE_EVENTS); +} + static inline bool tcp_ca_needs_ecn(const struct sock *sk) { const struct inet_connection_sock *icsk = inet_csk(sk); @@ -1471,6 +1511,21 @@ return t1 > t2 || (t1 == t2 && after(seq1, seq2)); } +/* If a retransmit failed due to local qdisc congestion or other local issues, + * then we may have called tcp_set_skb_tso_segs() to increase the number of + * segments in the skb without increasing the tx.in_flight. In all other cases, + * the tx.in_flight should be at least as big as the pcount of the sk_buff. We + * do not have the state to know whether a retransmit failed due to local qdisc + * congestion or other local issues, so to avoid spurious warnings we consider + * that any skb marked lost may have suffered that fate. + */ +static inline bool tcp_skb_tx_in_flight_is_suspicious(u32 skb_pcount, + u32 skb_sacked_flags, + u32 tx_in_flight) +{ + return (skb_pcount > tx_in_flight) && !(skb_sacked_flags & TCPCB_LOST); +} + /* These functions determine how the current flow behaves in respect of SACK * handling. SACK is negotiated with the peer, and therefore it can vary * between different flows. @@ -2662,7 +2717,7 @@ u8 consec_cong_rounds:5, /* consecutive congested rounds */ unused:3; u32 pause_until; /* jiffies32 when PLB can resume rerouting */ -}; +} __attribute__ ((__packed__)); static inline void tcp_plb_init(const struct sock *sk, struct tcp_plb_state *plb) diff -ruN --no-dereference -x .git a/include/net/tcp_ecn.h b/include/net/tcp_ecn.h --- a/include/net/tcp_ecn.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/net/tcp_ecn.h 2026-10-13 14:33:30.000000000 +0200 @@ -606,10 +606,9 @@ use_ecn = tcp_ecn == TCP_ECN_IN_ECN_OUT_ECN || tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ECN || tcp_ca_needs_ecn(sk) || bpf_needs_ecn || use_accecn; + const struct dst_entry *dst = __sk_dst_get(sk); if (!use_ecn) { - const struct dst_entry *dst = __sk_dst_get(sk); - if (dst && dst_feature(dst, RTAX_FEATURE_ECN)) use_ecn = true; } @@ -627,6 +626,9 @@ tp->syn_ect_snt = inet_sk(sk)->tos & INET_ECN_MASK; } else { tcp_ecn_mode_set(tp, TCP_ECN_MODE_RFC3168); + + if (dst) + tcp_set_ecn_low_from_dst(sk, dst); } } } diff -ruN --no-dereference -x .git a/include/sound/aw88399.h b/include/sound/aw88399.h --- a/include/sound/aw88399.h 1970-01-01 01:00:00.000000000 +0100 +++ b/include/sound/aw88399.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,621 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * + * linux/sound/aw88399.h -- Platform data for AW88399 + * + * Copyright (c) 2023 AWINIC Technology CO., LTD + * + * Author: Weidong Wang + */ + +#ifndef __SOUND_AW88399_H +#define __SOUND_AW88399_H + +#include +#include + +/* registers list */ +#define AW88399_ID_REG (0x00) +#define AW88399_SYSST_REG (0x01) +#define AW88399_SYSINT_REG (0x02) +#define AW88399_SYSINTM_REG (0x03) +#define AW88399_SYSCTRL_REG (0x04) +#define AW88399_SYSCTRL2_REG (0x05) +#define AW88399_I2SCTRL1_REG (0x06) +#define AW88399_I2SCTRL2_REG (0x07) +#define AW88399_I2SCTRL3_REG (0x08) +#define AW88399_DACCFG1_REG (0x09) +#define AW88399_DACCFG2_REG (0x0A) +#define AW88399_DACCFG3_REG (0x0B) +#define AW88399_DACCFG4_REG (0x0C) +#define AW88399_DACCFG5_REG (0x0D) +#define AW88399_DACCFG6_REG (0x0E) +#define AW88399_DACCFG7_REG (0x0F) +#define AW88399_MPDCFG1_REG (0x10) +#define AW88399_MPDCFG2_REG (0x11) +#define AW88399_MPDCFG3_REG (0x12) +#define AW88399_MPDCFG4_REG (0x13) +#define AW88399_PWMCTRL1_REG (0x14) +#define AW88399_PWMCTRL2_REG (0x15) +#define AW88399_PWMCTRL3_REG (0x16) +#define AW88399_I2SCFG1_REG (0x17) +#define AW88399_DBGCTRL_REG (0x18) +#define AW88399_HAGCST_REG (0x20) +#define AW88399_VBAT_REG (0x21) +#define AW88399_TEMP_REG (0x22) +#define AW88399_PVDD_REG (0x23) +#define AW88399_ISNDAT_REG (0x24) +#define AW88399_VSNDAT_REG (0x25) +#define AW88399_I2SINT_REG (0x26) +#define AW88399_I2SCAPCNT_REG (0x27) +#define AW88399_ANASTA1_REG (0x28) +#define AW88399_ANASTA2_REG (0x29) +#define AW88399_ANASTA3_REG (0x2A) +#define AW88399_TESTDET_REG (0x2B) +#define AW88399_DSMCFG1_REG (0x30) +#define AW88399_DSMCFG2_REG (0x31) +#define AW88399_DSMCFG3_REG (0x32) +#define AW88399_DSMCFG4_REG (0x33) +#define AW88399_DSMCFG5_REG (0x34) +#define AW88399_DSMCFG6_REG (0x35) +#define AW88399_DSMCFG7_REG (0x36) +#define AW88399_DSMCFG8_REG (0x37) +#define AW88399_TESTIN_REG (0x38) +#define AW88399_TESTOUT_REG (0x39) +#define AW88399_MEMTEST_REG (0x3A) +#define AW88399_VSNCTRL1_REG (0x3B) +#define AW88399_ISNCTRL1_REG (0x3C) +#define AW88399_ISNCTRL2_REG (0x3D) +#define AW88399_DSPMADD_REG (0x40) +#define AW88399_DSPMDAT_REG (0x41) +#define AW88399_WDT_REG (0x42) +#define AW88399_ACR1_REG (0x43) +#define AW88399_ACR2_REG (0x44) +#define AW88399_ASR1_REG (0x45) +#define AW88399_ASR2_REG (0x46) +#define AW88399_DSPCFG_REG (0x47) +#define AW88399_ASR3_REG (0x48) +#define AW88399_ASR4_REG (0x49) +#define AW88399_DSPVCALB_REG (0x4A) +#define AW88399_CRCCTRL_REG (0x4B) +#define AW88399_DSPDBG1_REG (0x4C) +#define AW88399_DSPDBG2_REG (0x4D) +#define AW88399_DSPDBG3_REG (0x4E) +#define AW88399_PLLCTRL1_REG (0x50) +#define AW88399_PLLCTRL2_REG (0x51) +#define AW88399_PLLCTRL3_REG (0x52) +#define AW88399_CDACTRL1_REG (0x53) +#define AW88399_CDACTRL2_REG (0x54) +#define AW88399_CDACTRL3_REG (0x55) +#define AW88399_SADCCTRL1_REG (0x56) +#define AW88399_SADCCTRL2_REG (0x57) +#define AW88399_BOPCTRL1_REG (0x58) +#define AW88399_BOPCTRL2_REG (0x5A) +#define AW88399_BOPCTRL3_REG (0x5B) +#define AW88399_BOPCTRL4_REG (0x5C) +#define AW88399_BOPCTRL5_REG (0x5D) +#define AW88399_BOPCTRL6_REG (0x5E) +#define AW88399_BOPCTRL7_REG (0x5F) +#define AW88399_BSTCTRL1_REG (0x60) +#define AW88399_BSTCTRL2_REG (0x61) +#define AW88399_BSTCTRL3_REG (0x62) +#define AW88399_BSTCTRL4_REG (0x63) +#define AW88399_BSTCTRL5_REG (0x64) +#define AW88399_BSTCTRL6_REG (0x65) +#define AW88399_BSTCTRL7_REG (0x66) +#define AW88399_BSTCTRL8_REG (0x67) +#define AW88399_BSTCTRL9_REG (0x68) +#define AW88399_BSTCTRL10_REG (0x69) +#define AW88399_CPCTRL_REG (0x6A) +#define AW88399_EFWH_REG (0x6C) +#define AW88399_EFWM2_REG (0x6D) +#define AW88399_EFWM1_REG (0x6E) +#define AW88399_EFWL_REG (0x6F) +#define AW88399_TESTCTRL1_REG (0x70) +#define AW88399_TESTCTRL2_REG (0x71) +#define AW88399_EFCTRL1_REG (0x72) +#define AW88399_EFCTRL2_REG (0x73) +#define AW88399_EFRH4_REG (0x74) +#define AW88399_EFRH3_REG (0x75) +#define AW88399_EFRH2_REG (0x76) +#define AW88399_EFRH1_REG (0x77) +#define AW88399_EFRL4_REG (0x78) +#define AW88399_EFRL3_REG (0x79) +#define AW88399_EFRL2_REG (0x7A) +#define AW88399_EFRL1_REG (0x7B) +#define AW88399_TM_REG (0x7C) +#define AW88399_TM2_REG (0x7D) + +#define AW88399_REG_MAX (0x7E) +#define AW88399_MUTE_VOL (1023) + +#define AW88399_DSP_CFG_ADDR (0x9B00) +#define AW88399_DSP_REG_CFG_ADPZ_RA (0x9B68) +#define AW88399_DSP_FW_ADDR (0x8980) +#define AW88399_DSP_ROM_CHECK_ADDR (0x1F40) +#define AW88399_DSP_ROM_CHECK_DATA (0x4638) + +#define AW88399_CALI_RE_HBITS_MASK (~(0xFFFF0000)) +#define AW88399_CALI_RE_HBITS_SHIFT (16) + +#define AW88399_CALI_RE_LBITS_MASK (~(0xFFFF)) +#define AW88399_CALI_RE_LBITS_SHIFT (0) + +#define AW88399_I2STXEN_START_BIT (9) +#define AW88399_I2STXEN_BITS_LEN (1) +#define AW88399_I2STXEN_MASK \ + (~(((1<> (shift)) +#define AW88399_SHOW_RE_TO_DSP_RE(re, shift) (((re) << shift) / (1000)) +#define AW88399_CRC_CHECK_PASS_VAL (0x4) + +#define AW88399_CRC_CFG_BASE_ADDR (0xD80) +#define AW88399_CRC_FW_BASE_ADDR (0x4C0) +#define AW88399_ACF_FILE "aw88399_acf.bin" +#define AW88399_DEV_SYSST_CHECK_MAX (10) +#define AW88399_CHIP_ID 0x2183 + +#define AW88399_START_RETRIES (5) +#define AW88399_START_WORK_DELAY_MS (0) + +enum { + AW_EF_AND_CHECK = 0, + AW_EF_OR_CHECK, +}; + +enum { + AW88399_DEV_VDSEL_DAC = 0, + AW88399_DEV_VDSEL_VSENSE = 32, +}; + +enum { + AW88399_DSP_CRC_NA = 0, + AW88399_DSP_CRC_OK = 1, +}; + +enum { + AW88399_DSP_FW_UPDATE_OFF = 0, + AW88399_DSP_FW_UPDATE_ON = 1, +}; + +enum { + AW88399_FORCE_UPDATE_OFF = 0, + AW88399_FORCE_UPDATE_ON = 1, +}; + +enum { + AW88399_1000_US = 1000, + AW88399_2000_US = 2000, + AW88399_3000_US = 3000, + AW88399_4000_US = 4000, +}; + +enum AW88399_DEV_STATUS { + AW88399_DEV_PW_OFF = 0, + AW88399_DEV_PW_ON, +}; + +enum AW88399_DEV_FW_STATUS { + AW88399_DEV_FW_FAILED = 0, + AW88399_DEV_FW_OK, +}; + +enum AW88399_DEV_MEMCLK { + AW88399_DEV_MEMCLK_OSC = 0, + AW88399_DEV_MEMCLK_PLL = 1, +}; + +enum AW88399_DEV_DSP_CFG { + AW88399_DEV_DSP_WORK = 0, + AW88399_DEV_DSP_BYPASS = 1, +}; + +enum { + AW88399_NOT_RCV_MODE = 0, + AW88399_RCV_MODE = 1, +}; + +enum { + AW88399_SYNC_START = 0, + AW88399_ASYNC_START, +}; + +struct aw_device; +struct aw_container; +struct aw_cali_desc; +struct gpio_desc; +struct i2c_client; +struct regmap; + +struct aw88399 { + struct aw_device *aw_pa; + struct mutex lock; + struct gpio_desc *reset_gpio; + struct delayed_work start_work; + struct regmap *regmap; + struct aw_container *aw_cfg; + + unsigned int check_val; + unsigned int crc_init_val; + unsigned int vcalb_init_val; + unsigned int dither_st; + bool bsts_unreliable; + bool fw_needs_reload; +}; + +int aw_dev_check_syspll(struct aw_device *aw_dev); +void aw_dev_dsp_enable(struct aw_device *aw_dev, bool is_enable); +int aw_dev_get_dsp_status(struct aw_device *aw_dev); +int aw_dev_set_volume(struct aw_device *aw_dev, unsigned int value); +int aw_dev_update_cali_re(struct aw_cali_desc *cali_desc); +int aw88399_dev_get_prof_name(struct aw_device *aw_dev, int index, char **prof_name); +void aw88399_dev_mute(struct aw_device *aw_dev, bool is_mute); +void aw88399_dev_set_channel(struct aw88399 *aw88399, int channel); +void aw88399_hw_reset(struct aw88399 *aw88399); +int aw88399_init(struct aw88399 *aw88399, struct i2c_client *i2c, struct regmap *regmap); +extern const struct regmap_config aw88399_remap_config; +int aw88399_request_firmware_file(struct aw88399 *aw88399); +void aw88399_start(struct aw88399 *aw88399, bool sync_start); +void aw88399_startup_work(struct work_struct *work); +int aw88399_stop(struct aw_device *aw_dev); + +#endif /* __SOUND_AW88399_H */ diff -ruN --no-dereference -x .git a/include/uapi/linux/inet_diag.h b/include/uapi/linux/inet_diag.h --- a/include/uapi/linux/inet_diag.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/uapi/linux/inet_diag.h 2026-10-13 14:33:30.000000000 +0200 @@ -238,6 +238,29 @@ __u32 bbr_min_rtt; /* min-filtered RTT in uSec */ __u32 bbr_pacing_gain; /* pacing gain shifted left 8 bits */ __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */ + __u32 bbr_bw_hi_lsb; /* lower 32 bits of bw_hi */ + __u32 bbr_bw_hi_msb; /* upper 32 bits of bw_hi */ + __u32 bbr_bw_lo_lsb; /* lower 32 bits of bw_lo */ + __u32 bbr_bw_lo_msb; /* upper 32 bits of bw_lo */ + __u8 bbr_mode; /* current bbr_mode in state machine */ + __u8 bbr_phase; /* current state machine phase */ + __u8 unused1; /* alignment padding; not used yet */ + __u8 bbr_version; /* BBR algorithm version */ + __u32 bbr_inflight_lo; /* lower short-term data volume bound */ + __u32 bbr_inflight_hi; /* higher long-term data volume bound */ + __u32 bbr_extra_acked; /* max excess packets ACKed in epoch */ +}; + +/* TCP BBR congestion control bbr_phase as reported in netlink/ss stats. */ +enum tcp_bbr_phase { + BBR_PHASE_INVALID = 0, + BBR_PHASE_STARTUP = 1, + BBR_PHASE_DRAIN = 2, + BBR_PHASE_PROBE_RTT = 3, + BBR_PHASE_PROBE_BW_UP = 4, + BBR_PHASE_PROBE_BW_DOWN = 5, + BBR_PHASE_PROBE_BW_CRUISE = 6, + BBR_PHASE_PROBE_BW_REFILL = 7, }; union tcp_cc_info { diff -ruN --no-dereference -x .git a/include/uapi/linux/rtnetlink.h b/include/uapi/linux/rtnetlink.h --- a/include/uapi/linux/rtnetlink.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/uapi/linux/rtnetlink.h 2026-10-13 14:33:30.000000000 +0200 @@ -517,12 +517,14 @@ #define RTAX_FEATURE_TIMESTAMP (1 << 2) /* unused */ #define RTAX_FEATURE_ALLFRAG (1 << 3) /* unused */ #define RTAX_FEATURE_TCP_USEC_TS (1 << 4) +#define RTAX_FEATURE_ECN_LOW (1 << 5) #define RTAX_FEATURE_MASK (RTAX_FEATURE_ECN | \ RTAX_FEATURE_SACK | \ RTAX_FEATURE_TIMESTAMP | \ RTAX_FEATURE_ALLFRAG | \ - RTAX_FEATURE_TCP_USEC_TS) + RTAX_FEATURE_TCP_USEC_TS | \ + RTAX_FEATURE_ECN_LOW) struct rta_session { __u8 proto; diff -ruN --no-dereference -x .git a/include/uapi/linux/tcp.h b/include/uapi/linux/tcp.h --- a/include/uapi/linux/tcp.h 2026-10-11 15:27:50.000000000 +0200 +++ b/include/uapi/linux/tcp.h 2026-10-13 14:33:30.000000000 +0200 @@ -185,6 +185,7 @@ #define TCPI_OPT_SYN_DATA 32 /* SYN-ACK acked data in SYN sent or rcvd */ #define TCPI_OPT_USEC_TS 64 /* usec timestamps */ #define TCPI_OPT_TFO_CHILD 128 /* child from a Fast Open option on SYN */ +#define TCPI_OPT_ECN_LOW 256 /* Low-latency ECN enabled at conn init */ /* * Sender's congestion state indicating normal or abnormal situations diff -ruN --no-dereference -x .git a/init/Kconfig b/init/Kconfig --- a/init/Kconfig 2026-10-11 15:27:50.000000000 +0200 +++ b/init/Kconfig 2026-10-13 14:33:30.000000000 +0200 @@ -223,6 +223,10 @@ menu "General setup" +config CACHY + bool "Kernel tweaks by CachyOS" + default y + config BROKEN bool help @@ -1497,6 +1501,19 @@ desktop applications. Task group autogeneration is currently based upon task session. +config SCHED_POC_SELECTOR + bool "Piece-Of-Cake Fast Idle CPU Selector" + depends on SMP + default y + help + Idle CPU selector using cached bitmasks inspired by the scx_cake BPF + scheduler. Reduces select_idle_cpu overhead by using bitmap scanning. + + This optimization does not affect scheduler fairness - it only + speeds up the process of finding an idle CPU for task wakeup. + + If unsure, say Y. + config RELAY bool "Kernel->user space relay support (formerly relayfs)" select IRQ_WORK @@ -1621,6 +1638,12 @@ with the "-O2" compiler flag for best performance and most helpful compile-time warnings. +config CC_OPTIMIZE_FOR_PERFORMANCE_O3 + bool "Optimize more for performance (-O3)" + help + Choosing this option will pass "-O3" to your compiler to optimize + the kernel yet more for performance. + config CC_OPTIMIZE_FOR_SIZE bool "Optimize for size (-Os)" help diff -ruN --no-dereference -x .git a/kernel/Kconfig.hz b/kernel/Kconfig.hz --- a/kernel/Kconfig.hz 2026-10-11 15:27:50.000000000 +0200 +++ b/kernel/Kconfig.hz 2026-10-13 14:33:30.000000000 +0200 @@ -40,6 +40,27 @@ on SMP and NUMA systems and exactly dividing by both PAL and NTSC frame rates for video and multimedia work. + config HZ_500 + bool "500 HZ" + help + 500 Hz is a balanced timer frequency. Provides fast interactivity + on desktops with good smoothness without increasing CPU power + consumption and sacrificing the battery life on laptops. + + config HZ_600 + bool "600 HZ" + help + 600 Hz is a balanced timer frequency. Provides fast interactivity + on desktops with good smoothness without increasing CPU power + consumption and sacrificing the battery life on laptops. + + config HZ_750 + bool "750 HZ" + help + 750 Hz is a balanced timer frequency. Provides fast interactivity + on desktops with good smoothness without increasing CPU power + consumption and sacrificing the battery life on laptops. + config HZ_1000 bool "1000 HZ" help @@ -53,6 +74,9 @@ default 100 if HZ_100 default 250 if HZ_250 default 300 if HZ_300 + default 500 if HZ_500 + default 600 if HZ_600 + default 750 if HZ_750 default 1000 if HZ_1000 config SCHED_HRTICK diff -ruN --no-dereference -x .git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt --- a/kernel/Kconfig.preempt 2026-10-11 15:27:50.000000000 +0200 +++ b/kernel/Kconfig.preempt 2026-10-13 14:33:30.000000000 +0200 @@ -91,7 +91,7 @@ config PREEMPT_RT bool "Fully Preemptible Kernel (Real-Time)" - depends on EXPERT && ARCH_SUPPORTS_RT && !COMPILE_TEST + depends on ARCH_SUPPORTS_RT && !COMPILE_TEST select PREEMPTION help This option turns the kernel into a real-time kernel by replacing diff -ruN --no-dereference -x .git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c --- a/kernel/cgroup/cpuset.c 2026-10-11 15:27:50.000000000 +0200 +++ b/kernel/cgroup/cpuset.c 2026-10-13 14:33:30.000000000 +0200 @@ -4158,6 +4158,28 @@ return changed; } +/* + * Returns the number of CPUs available for this cgroup. + * + * This only really works for cgroup-v2 where all the controllers are mounted + * in the same hierarchy. If not cgroup-v2 or no cpuset controller is + * configured it reverts to num_online_cpus(). + */ +int cpuset_num_cpus(struct cgroup *cgrp) +{ + int nr = num_online_cpus(); + struct cpuset *cs; + + if (is_in_v2_mode()) { + guard(rcu)(); + cs = css_cs(cgroup_e_css(cgrp, &cpuset_cgrp_subsys)); + if (cs) + nr = cpumask_weight(cs->effective_cpus); + } + + return nr; +} + void __init cpuset_init_current_mems_allowed(void) { nodes_setall(current->mems_allowed); diff -ruN --no-dereference -x .git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c --- a/kernel/cgroup/dmem.c 2026-10-11 15:27:50.000000000 +0200 +++ b/kernel/cgroup/dmem.c 2026-10-13 14:33:30.000000000 +0200 @@ -695,6 +695,110 @@ } EXPORT_SYMBOL_GPL(dmem_cgroup_try_charge); +/** + * dmem_cgroup_below_min() - Tests whether current usage is within min limit. + * + * @root: Root of the subtree to calculate protection for, or NULL to calculate global protection. + * @test: The pool to test the usage/min limit of. + * + * Return: true if usage is below min and the cgroup is protected, false otherwise. + */ +bool dmem_cgroup_below_min(struct dmem_cgroup_pool_state *root, + struct dmem_cgroup_pool_state *test) +{ + if (root == test || !pool_parent(test)) + return false; + + if (!root) { + for (root = test; pool_parent(root); root = pool_parent(root)) + {} + } + + /* + * In mem_cgroup_below_min(), the memcg pendant, this call is missing. + * mem_cgroup_below_min() gets called during traversal of the cgroup tree, where + * protection is already calculated as part of the traversal. dmem cgroup eviction + * does not traverse the cgroup tree, so we need to recalculate effective protection + * here. + */ + dmem_cgroup_calculate_protection(root, test); + return page_counter_read(&test->cnt) <= READ_ONCE(test->cnt.emin); +} +EXPORT_SYMBOL_GPL(dmem_cgroup_below_min); + +/** + * dmem_cgroup_below_low() - Tests whether current usage is within low limit. + * + * @root: Root of the subtree to calculate protection for, or NULL to calculate global protection. + * @test: The pool to test the usage/low limit of. + * + * Return: true if usage is below low and the cgroup is protected, false otherwise. + */ +bool dmem_cgroup_below_low(struct dmem_cgroup_pool_state *root, + struct dmem_cgroup_pool_state *test) +{ + if (root == test || !pool_parent(test)) + return false; + + if (!root) { + for (root = test; pool_parent(root); root = pool_parent(root)) + {} + } + + /* + * In mem_cgroup_below_low(), the memcg pendant, this call is missing. + * mem_cgroup_below_low() gets called during traversal of the cgroup tree, where + * protection is already calculated as part of the traversal. dmem cgroup eviction + * does not traverse the cgroup tree, so we need to recalculate effective protection + * here. + */ + dmem_cgroup_calculate_protection(root, test); + return page_counter_read(&test->cnt) <= READ_ONCE(test->cnt.elow); +} +EXPORT_SYMBOL_GPL(dmem_cgroup_below_low); + +/** + * dmem_cgroup_get_common_ancestor(): Find the first common ancestor of two pools. + * @a: First pool to find the common ancestor of. + * @b: First pool to find the common ancestor of. + * + * Return: The first pool that is a parent of both @a and @b, or NULL if either @a or @b are NULL, + * or if such a pool does not exist. A reference to the returned pool is grabbed and must be + * released by the caller when it is done using the pool. + */ +struct dmem_cgroup_pool_state *dmem_cgroup_get_common_ancestor(struct dmem_cgroup_pool_state *a, + struct dmem_cgroup_pool_state *b) +{ + struct cgroup *ancestor_cgroup; + struct cgroup_subsys_state *ancestor_css; + struct dmemcg_state *ancestor_dmemcs = NULL; + struct dmem_cgroup_pool_state *pool = NULL; + + if (!a || !b) + return NULL; + + ancestor_cgroup = cgroup_common_ancestor(a->cs->css.cgroup, b->cs->css.cgroup); + if (!ancestor_cgroup) + return NULL; + + rcu_read_lock(); + ancestor_css = cgroup_e_css(ancestor_cgroup, &dmem_cgrp_subsys); + if (css_tryget(ancestor_css)) + ancestor_dmemcs = css_to_dmemcs(ancestor_css); + rcu_read_unlock(); + + if (ancestor_dmemcs) { + pool = get_cg_pool_unlocked(css_to_dmemcs(ancestor_css), + a->region); + if (WARN_ON(IS_ERR(pool))) { + pool = NULL; + css_put(ancestor_css); + } + } + return pool; +} +EXPORT_SYMBOL_GPL(dmem_cgroup_get_common_ancestor); + static int dmem_cgroup_region_capacity_show(struct seq_file *sf, void *v) { struct dmem_cgroup_region *region; diff -ruN --no-dereference -x .git a/kernel/fork.c b/kernel/fork.c --- a/kernel/fork.c 2026-10-11 15:27:50.000000000 +0200 +++ b/kernel/fork.c 2026-10-13 14:33:30.000000000 +0200 @@ -127,6 +127,12 @@ #include +#ifdef CONFIG_USER_NS +static int unprivileged_userns_clone = 1; +#else +#define unprivileged_userns_clone 1 +#endif + /* * Minimum number of threads to boot the kernel */ @@ -537,6 +543,7 @@ #endif release_user_cpus_ptr(tsk); scs_release(tsk); + smp_task_ipi_mask_free(tsk); #ifndef CONFIG_THREAD_INFO_IN_TASK /* @@ -935,10 +942,14 @@ #endif account_kernel_stack(tsk, 1); - err = scs_prepare(tsk, node); + err = smp_task_ipi_mask_alloc(tsk); if (err) goto free_stack; + err = scs_prepare(tsk, node); + if (err) + goto free_ipi_mask; + #ifdef CONFIG_SECCOMP /* * We must handle setting up seccomp filters once we're under @@ -1016,6 +1027,8 @@ #endif return tsk; +free_ipi_mask: + smp_task_ipi_mask_free(tsk); free_stack: exit_task_stack_account(tsk); free_thread_stack(tsk); @@ -2093,6 +2106,11 @@ return ERR_PTR(-EPERM); } + if ((clone_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) { + if (!capable(CAP_SYS_ADMIN)) + return ERR_PTR(-EPERM); + } + /* * Force any signals received before this point to be delivered * before the fork happens. Collect up signals sent to multiple @@ -3167,6 +3185,10 @@ if (!current_is_single_threaded()) return -EINVAL; } + if ((unshare_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) { + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + } return 0; } @@ -3402,6 +3424,15 @@ .mode = 0644, .proc_handler = sysctl_max_threads, }, +#ifdef CONFIG_USER_NS + { + .procname = "unprivileged_userns_clone", + .data = &unprivileged_userns_clone, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif }; static int __init init_fork_sysctl(void) diff -ruN --no-dereference -x .git a/kernel/scftorture.c b/kernel/scftorture.c --- a/kernel/scftorture.c 2026-10-11 15:27:50.000000000 +0200 +++ b/kernel/scftorture.c 2026-10-13 14:33:30.000000000 +0200 @@ -348,6 +348,8 @@ int ret = 0; struct scf_check *scfcp = NULL; struct scf_selector *scfsp = scf_sel_rand(trsp); + bool is_single = (scfsp->scfs_prim == SCF_PRIM_SINGLE || + scfsp->scfs_prim == SCF_PRIM_SINGLE_RPC); if (scfsp->scfs_prim == SCF_PRIM_SINGLE || scfsp->scfs_wait) { scfcp = kmalloc_obj(*scfcp, GFP_ATOMIC); @@ -364,8 +366,6 @@ } if (use_cpus_read_lock) cpus_read_lock(); - else - preempt_disable(); switch (scfsp->scfs_prim) { case SCF_PRIM_RESCHED: if (IS_BUILTIN(CONFIG_SCF_TORTURE_TEST)) { @@ -411,13 +411,10 @@ if (!ret) { if (use_cpus_read_lock) cpus_read_unlock(); - else - preempt_enable(); + wait_for_completion(&scfcp->scfc_completion); if (use_cpus_read_lock) cpus_read_lock(); - else - preempt_disable(); } else { scfp->n_single_rpc_ofl++; scf_add_to_free_list(scfcp); @@ -452,7 +449,7 @@ scfcp->scfc_out = true; } if (scfcp && scfsp->scfs_wait) { - if (WARN_ON_ONCE((num_online_cpus() > 1 || scfsp->scfs_prim == SCF_PRIM_SINGLE) && + if (WARN_ON_ONCE(((use_cpus_read_lock && num_online_cpus() > 1) || is_single) && !scfcp->scfc_out)) { pr_warn("%s: Memory-ordering failure, scfs_prim: %d.\n", __func__, scfsp->scfs_prim); atomic_inc(&n_mb_out_errs); // Leak rather than trash! @@ -463,8 +460,6 @@ } if (use_cpus_read_lock) cpus_read_unlock(); - else - preempt_enable(); if (allocfail) schedule_timeout_idle((1 + longwait) * HZ); // Let no-wait handlers complete. else if (!(torture_random(trsp) & 0xfff)) diff -ruN --no-dereference -x .git a/kernel/sched/core.c b/kernel/sched/core.c --- a/kernel/sched/core.c 2026-10-11 15:27:50.000000000 +0200 +++ b/kernel/sched/core.c 2026-10-13 14:33:30.000000000 +0200 @@ -674,7 +674,7 @@ /* Matches synchronize_rcu() in __sched_core_enable() */ preempt_disable(); - if (sched_core_disabled()) { + if (likely(sched_core_disabled())) { raw_spin_lock_nested(&rq->__lock, subclass); /* preempt_count *MUST* be > 1 */ preempt_enable_no_resched(); @@ -886,7 +886,7 @@ scx_rq_clock_update(rq, clock); delta = clock - rq->clock; - if (delta < 0) + if (unlikely(delta < 0)) return; rq->clock += delta; @@ -5098,7 +5098,7 @@ WRITE_ONCE(next->on_cpu, 1); } -static inline void finish_task(struct task_struct *prev) +static __always_inline void finish_task(struct task_struct *prev) { /* * This must be the very last reference to @prev from this CPU. After @@ -5142,7 +5142,7 @@ rq->balance_callback = found ? &balance_push_callback : NULL; } -static void do_balance_callbacks(struct rq *rq, struct balance_callback *head) +static __always_inline void do_balance_callbacks(struct rq *rq, struct balance_callback *head) { void (*func)(struct rq *rq); struct balance_callback *next; @@ -5177,7 +5177,7 @@ .func = balance_push, }; -static inline struct balance_callback * +static __always_inline struct balance_callback * __splice_balance_callbacks(struct rq *rq, bool split) { struct balance_callback *head = rq->balance_callback; @@ -5251,7 +5251,7 @@ __acquire(__rq_lockp(this_rq())); } -static inline void finish_lock_switch(struct rq *rq) +static __always_inline void finish_lock_switch(struct rq *rq) __releases(__rq_lockp(rq)) { /* @@ -5285,7 +5285,7 @@ #endif } -static inline void kmap_local_sched_in(void) +static __always_inline void kmap_local_sched_in(void) { #ifdef CONFIG_KMAP_LOCAL if (unlikely(current->kmap_ctrl.idx)) @@ -5339,7 +5339,7 @@ * past. 'prev == current' is still correct but we need to recalculate this_rq * because prev may have moved to another CPU. */ -static struct rq *finish_task_switch(struct task_struct *prev) +static __always_inline struct rq *finish_task_switch(struct task_struct *prev) __releases(__rq_lockp(this_rq())) { struct rq *rq = this_rq(); @@ -5681,11 +5681,8 @@ */ static inline void prefetch_curr_exec_start(struct task_struct *p) { -#ifdef CONFIG_FAIR_GROUP_SCHED - struct sched_entity *curr = p->se.cfs_rq->curr; -#else struct sched_entity *curr = task_rq(p)->cfs.curr; -#endif + prefetch(curr); prefetch(&curr->exec_start); } @@ -6250,7 +6247,7 @@ struct rq *rq_i; bool need_sync = false; - if (!sched_core_enabled(rq)) + if (likely(!sched_core_enabled(rq))) return __pick_next_task(rq, rf); cpu = cpu_of(rq); @@ -7798,7 +7795,7 @@ #if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) int __sched __cond_resched(void) { - if (should_resched(0) && !irqs_disabled()) { + if (unlikely(should_resched(0) && !irqs_disabled())) { preempt_schedule_common(); return 1; } diff -ruN --no-dereference -x .git a/kernel/sched/debug.c b/kernel/sched/debug.c --- a/kernel/sched/debug.c 2026-10-11 15:27:50.000000000 +0200 +++ b/kernel/sched/debug.c 2026-10-13 14:33:30.000000000 +0200 @@ -633,6 +633,82 @@ } } +#ifdef CONFIG_FAIR_GROUP_SCHED +static int cgroup_mode = 2; + +/* See __sched_cgroup_mode_update(). */ +static const char *cgroup_mode_str[] = { + "up", + "smp", + "concur", + "max", + "tasks", +}; + +static int sched_cgroup_mode(const char *str) +{ + for (int i = 0; i < ARRAY_SIZE(cgroup_mode_str); i++) { + if (!strcmp(str, cgroup_mode_str[i])) + return i; + } + return -EINVAL; +} + +static ssize_t sched_cgroup_write(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *ppos) +{ + char buf[16]; + int mode; + + if (cnt > 15) + cnt = 15; + + if (copy_from_user(buf, ubuf, cnt)) + return -EFAULT; + + buf[cnt] = 0; + mode = sched_cgroup_mode(strstrip(buf)); + if (mode < 0) + return mode; + + __sched_cgroup_mode_update(mode); + WRITE_ONCE(cgroup_mode, mode); + + *ppos += cnt; + return cnt; +} + +static int sched_cgroup_show(struct seq_file *m, void *v) +{ + int mode = READ_ONCE(cgroup_mode); + + for (int i = 0; i < ARRAY_SIZE(cgroup_mode_str); i++) { + if (mode == i) + seq_puts(m, "("); + seq_puts(m, cgroup_mode_str[i]); + if (mode == i) + seq_puts(m, ")"); + + seq_puts(m, " "); + } + seq_puts(m, "\n"); + return 0; +} + +static int sched_cgroup_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, sched_cgroup_show, NULL); +} + +static const struct file_operations sched_cgroup_fops = { + .open = sched_cgroup_open, + .write = sched_cgroup_write, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +#endif + static __init int sched_init_debug(void) { struct dentry __maybe_unused *numa, *llc; @@ -686,6 +762,10 @@ debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); +#ifdef CONFIG_FAIR_GROUP_SCHED + debugfs_create_file("cgroup_mode", 0644, debugfs_sched, NULL, &sched_cgroup_fops); +#endif + debugfs_fair_server_init(); #ifdef CONFIG_SCHED_CLASS_EXT debugfs_ext_server_init(); @@ -895,10 +975,11 @@ else SEQ_printf(m, " %c", task_state_to_char(p)); - SEQ_printf(m, " %15s %5d %9Ld.%06ld %c %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld %5d ", + SEQ_printf(m, " %15s %5d %10ld %9Ld.%06ld %c %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld %5d ", p->comm, task_pid_nr(p), + p->se.h_load.weight, SPLIT_NS(p->se.vruntime), - entity_eligible(cfs_rq_of(&p->se), &p->se) ? 'E' : 'N', + entity_eligible(&rq->cfs, &p->se) ? 'E' : 'N', SPLIT_NS(p->se.deadline), p->se.custom_slice ? 'S' : ' ', SPLIT_NS(p->se.slice), @@ -927,7 +1008,7 @@ SEQ_printf(m, "\n"); SEQ_printf(m, "runnable tasks:\n"); - SEQ_printf(m, " S task PID vruntime eligible " + SEQ_printf(m, " S task PID weight vruntime eligible " "deadline slice sum-exec switches " "prio wait-time sum-sleep sum-block" #ifdef CONFIG_NUMA_BALANCING @@ -1035,6 +1116,8 @@ cfs_rq->tg_load_avg_contrib); SEQ_printf(m, " .%-30s: %ld\n", "tg_load_avg", atomic_long_read(&cfs_rq->tg->load_avg)); + SEQ_printf(m, " .%-30s: %lu\n", "h_load", + cfs_rq->h_load); #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_CFS_BANDWIDTH SEQ_printf(m, " .%-30s: %d\n", "throttled", diff -ruN --no-dereference -x .git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c --- a/kernel/sched/ext/ext.c 2026-10-11 15:27:50.000000000 +0200 +++ b/kernel/sched/ext/ext.c 2026-10-13 14:33:30.000000000 +0200 @@ -6390,6 +6390,9 @@ mutex_unlock(&scx_enable_mutex); WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); +#ifdef CONFIG_SCHED_POC_SELECTOR + poc_notify_scx(false); +#endif done: scx_bypass(sch, false); } @@ -7606,6 +7609,10 @@ } } +#ifdef CONFIG_SCHED_POC_SELECTOR + poc_notify_scx(true); +#endif + pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n", sch->ops.name, scx_switched_all() ? "" : " (partial)"); kobject_uevent(&sch->kobj, KOBJ_ADD); diff -ruN --no-dereference -x .git a/kernel/sched/fair.c b/kernel/sched/fair.c --- a/kernel/sched/fair.c 2026-10-11 15:27:50.000000000 +0200 +++ b/kernel/sched/fair.c 2026-10-13 14:33:30.000000000 +0200 @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -49,6 +50,7 @@ #include #include #include +#include #include @@ -76,10 +78,19 @@ * * (default: 0.70 msec * (1 + ilog(ncpus)), units: nanoseconds) */ +#ifdef CONFIG_CACHY +unsigned int sysctl_sched_base_slice = 400000ULL; +static unsigned int normalized_sysctl_sched_base_slice = 400000ULL; +#else unsigned int sysctl_sched_base_slice = 700000ULL; static unsigned int normalized_sysctl_sched_base_slice = 700000ULL; +#endif /* CONFIG_CACHY */ +#ifdef CONFIG_CACHY +__read_mostly unsigned int sysctl_sched_migration_cost = 400000UL; +#else __read_mostly unsigned int sysctl_sched_migration_cost = 500000UL; +#endif static int __init setup_sched_thermal_decay_shift(char *str) { @@ -296,8 +307,8 @@ */ static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) { - if (unlikely(se->load.weight != NICE_0_LOAD)) - delta = __calc_delta(delta, NICE_0_LOAD, &se->load); + if (se->h_load.weight != NICE_0_LOAD) + delta = __calc_delta(delta, NICE_0_LOAD, &se->h_load); return delta; } @@ -427,38 +438,6 @@ return se->parent; } -static void -find_matching_se(struct sched_entity **se, struct sched_entity **pse) -{ - int se_depth, pse_depth; - - /* - * preemption test can be made between sibling entities who are in the - * same cfs_rq i.e who have a common parent. Walk up the hierarchy of - * both tasks until we find their ancestors who are siblings of common - * parent. - */ - - /* First walk up until both entities are at same depth */ - se_depth = (*se)->depth; - pse_depth = (*pse)->depth; - - while (se_depth > pse_depth) { - se_depth--; - *se = parent_entity(*se); - } - - while (pse_depth > se_depth) { - pse_depth--; - *pse = parent_entity(*pse); - } - - while (!is_same_group(*se, *pse)) { - *se = parent_entity(*se); - *pse = parent_entity(*pse); - } -} - static int tg_is_idle(struct task_group *tg) { return tg->idle > 0; @@ -502,11 +481,6 @@ return NULL; } -static inline void -find_matching_se(struct sched_entity **se, struct sched_entity **pse) -{ -} - static inline int tg_is_idle(struct task_group *tg) { return 0; @@ -685,7 +659,7 @@ static inline void __sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) { - unsigned long weight = avg_vruntime_weight(cfs_rq, se->load.weight); + unsigned long weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); s64 w_vruntime, key = entity_key(cfs_rq, se); w_vruntime = key * weight; @@ -702,7 +676,7 @@ s64 key, tmp; again: - weight = avg_vruntime_weight(cfs_rq, se->load.weight); + weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); key = entity_key(cfs_rq, se); if (check_mul_overflow(key, weight, &key)) @@ -748,7 +722,7 @@ static void sum_w_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) { - unsigned long weight = avg_vruntime_weight(cfs_rq, se->load.weight); + unsigned long weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); s64 key = entity_key(cfs_rq, se); cfs_rq->sum_w_vruntime -= key * weight; @@ -790,7 +764,7 @@ s64 runtime = cfs_rq->sum_w_vruntime; if (curr) { - unsigned long w = avg_vruntime_weight(cfs_rq, curr->load.weight); + unsigned long w = avg_vruntime_weight(cfs_rq, curr->h_load.weight); runtime += entity_key(cfs_rq, curr) * w; weight += w; @@ -861,8 +835,6 @@ u64 avruntime = avg_vruntime(cfs_rq); s64 vlag = entity_lag(cfs_rq, se, avruntime); - WARN_ON_ONCE(!se->on_rq); - if (se->sched_delayed) { /* previous vlag < 0 otherwise se would not be delayed */ vlag = max(vlag, se->vlag); @@ -898,7 +870,7 @@ long load = cfs_rq->sum_weight; if (curr && curr->on_rq) { - unsigned long weight = avg_vruntime_weight(cfs_rq, curr->load.weight); + unsigned long weight = avg_vruntime_weight(cfs_rq, curr->h_load.weight); avg += entity_key(cfs_rq, curr) * weight; load += weight; @@ -1047,6 +1019,9 @@ */ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { + WARN_ON_ONCE(&rq_of(cfs_rq)->cfs != cfs_rq); + WARN_ON_ONCE(!entity_is_task(se)); + sum_w_vruntime_add(cfs_rq, se); se->min_vruntime = se->vruntime; se->min_slice = se->slice; @@ -1058,6 +1033,9 @@ static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { + WARN_ON_ONCE(&rq_of(cfs_rq)->cfs != cfs_rq); + WARN_ON_ONCE(!entity_is_task(se)); + rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, &min_vruntime_cb); sum_w_vruntime_sub(cfs_rq, se); @@ -1154,7 +1132,7 @@ * We can safely skip eligibility check if there is only one entity * in this cfs_rq, saving some cycles. */ - if (cfs_rq->nr_queued == 1) + if (cfs_rq->h_nr_queued == 1) return curr && curr->on_rq ? curr : se; /* @@ -1272,7 +1250,7 @@ #include "pelt.h" -static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu); +static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu, int sync); static unsigned long task_h_load(struct task_struct *p); static unsigned long capacity_of(int cpu); @@ -1402,8 +1380,6 @@ return delta_exec; } -static void set_next_buddy(struct sched_entity *se); - #ifdef CONFIG_SCHED_CACHE /* @@ -2189,7 +2165,7 @@ * not necessarily be the actual task running * (rq->curr.se). This is easy to confuse! */ - struct sched_entity *curr = cfs_rq->curr; + struct sched_entity *curr = cfs_rq->h_curr; struct rq *rq = rq_of(cfs_rq); s64 delta_exec; bool resched; @@ -2201,26 +2177,29 @@ if (unlikely(delta_exec <= 0)) return; + account_cfs_rq_runtime(cfs_rq, delta_exec); + + if (!entity_is_task(curr)) + return; + + cfs_rq = &rq->cfs; + curr->vruntime += calc_delta_fair(delta_exec, curr); resched = update_deadline(cfs_rq, curr); - if (entity_is_task(curr)) { - /* - * If the fair_server is active, we need to account for the - * fair_server time whether or not the task is running on - * behalf of fair_server or not: - * - If the task is running on behalf of fair_server, we need - * to limit its time based on the assigned runtime. - * - Fair task that runs outside of fair_server should account - * against fair_server such that it can account for this time - * and possibly avoid running this period. - */ - dl_server_update(&rq->fair_server, delta_exec); - } - - account_cfs_rq_runtime(cfs_rq, delta_exec); + /* + * If the fair_server is active, we need to account for the + * fair_server time whether or not the task is running on + * behalf of fair_server or not: + * - If the task is running on behalf of fair_server, we need + * to limit its time based on the assigned runtime. + * - Fair task that runs outside of fair_server should account + * against fair_server such that it can account for this time + * and possibly avoid running this period. + */ + dl_server_update(&rq->fair_server, delta_exec); - if (cfs_rq->nr_queued == 1) + if (cfs_rq->h_nr_queued == 1) return; if (resched || !protect_slice(curr)) { @@ -2231,7 +2210,10 @@ static void update_curr_fair(struct rq *rq) { - update_curr(cfs_rq_of(&rq->donor->se)); + struct sched_entity *se = &rq->donor->se; + + for_each_sched_entity(se) + update_curr(cfs_rq_of(se)); } static inline void @@ -2307,7 +2289,7 @@ * Are we enqueueing a waiting task? (for current tasks * a dequeue/enqueue event is a NOP) */ - if (se != cfs_rq->curr) + if (se != cfs_rq->h_curr) update_stats_wait_start_fair(cfs_rq, se); if (flags & ENQUEUE_WAKEUP) @@ -2325,7 +2307,7 @@ * Mark the end of the wait period if dequeueing a * waiting task: */ - if (se != cfs_rq->curr) + if (se != cfs_rq->h_curr) update_stats_wait_end_fair(cfs_rq, se); if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { @@ -4672,6 +4654,7 @@ static void account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) { + WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); update_load_add(&cfs_rq->load, se->load.weight); if (entity_is_task(se)) { struct task_struct *p = task_of(se); @@ -4687,6 +4670,7 @@ static void account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) { + WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); update_load_sub(&cfs_rq->load, se->load.weight); if (entity_is_task(se)) { struct task_struct *p = task_of(se); @@ -4768,7 +4752,7 @@ static void rescale_entity(struct sched_entity *se, unsigned long weight, bool rel_vprot) { - unsigned long old_weight = se->load.weight; + long old_weight = se->h_load.weight; /* * VRUNTIME @@ -4868,16 +4852,17 @@ se->vprot = div64_long(se->vprot * old_weight, weight); } -static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, - unsigned long weight) +static void reweight_eevdf(struct cfs_rq *cfs_rq, struct sched_entity *se, + unsigned long weight, bool on_rq) { bool curr = cfs_rq->curr == se; bool rel_vprot = false; u64 avruntime = 0; - if (se->on_rq) { - /* commit outstanding execution time */ - update_curr(cfs_rq); + if (se->h_load.weight == weight) + return; + + if (on_rq) { avruntime = avg_vruntime(cfs_rq); se->vlag = entity_lag(cfs_rq, se, avruntime); se->deadline -= avruntime; @@ -4887,46 +4872,90 @@ rel_vprot = true; } - cfs_rq->nr_queued--; + cfs_rq->h_nr_queued--; if (!curr) __dequeue_entity(cfs_rq, se); - update_load_sub(&cfs_rq->load, se->load.weight); } - dequeue_load_avg(cfs_rq, se); rescale_entity(se, weight, rel_vprot); - update_load_set(&se->load, weight); + update_load_set(&se->h_load, weight); - do { - u32 divider = get_pelt_divider(&se->avg); - se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); - } while (0); - - enqueue_load_avg(cfs_rq, se); - if (se->on_rq) { + if (on_rq) { if (rel_vprot) se->vprot += avruntime; se->deadline += avruntime; se->rel_deadline = 0; se->vruntime = avruntime - se->vlag; - update_load_add(&cfs_rq->load, se->load.weight); if (!curr) __enqueue_entity(cfs_rq, se); - cfs_rq->nr_queued++; + cfs_rq->h_nr_queued++; } } +static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, + unsigned long weight) +{ + if (se->load.weight == weight) + return; + + if (se->on_rq) { + WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); + update_load_sub(&cfs_rq->load, se->load.weight); + } + dequeue_load_avg(cfs_rq, se); + + update_load_set(&se->load, weight); + + do { + u32 divider = get_pelt_divider(&se->avg); + se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); + } while (0); + + enqueue_load_avg(cfs_rq, se); + + if (se->on_rq) + update_load_add(&cfs_rq->load, se->load.weight); +} + +/* + * weight = NICE_0_LOAD; + * for_each_entity_se(se) + * weight = __calc_prop_weight(cfs_rq_of(se), se, weight); + */ +static __always_inline +unsigned long __calc_prop_weight(struct cfs_rq *cfs_rq, struct sched_entity *se, + unsigned long weight) +{ + weight *= se->load.weight; + if (parent_entity(se)) + weight /= cfs_rq->load.weight; + else + weight /= NICE_0_LOAD; + + return max(weight, MIN_SHARES); +} + static void reweight_task_fair(struct rq *rq, struct task_struct *p, const struct load_weight *lw) { struct sched_entity *se = &p->se; - struct cfs_rq *cfs_rq = cfs_rq_of(se); - struct load_weight *load = &se->load; + unsigned long weight = NICE_0_LOAD; + + if (se->on_rq) + update_curr_fair(rq); + + reweight_entity(cfs_rq_of(se), se, lw->weight); + se->load.inv_weight = lw->inv_weight; + + if (!se->on_rq) + return; - reweight_entity(cfs_rq, se, lw->weight); - load->inv_weight = lw->inv_weight; + for_each_sched_entity(se) + weight = __calc_prop_weight(cfs_rq_of(se), se, weight); + + reweight_eevdf(&rq->cfs, &p->se, weight, p->se.on_rq); } static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); @@ -5005,12 +5034,10 @@ * * hence icky! */ -static long calc_group_shares(struct cfs_rq *cfs_rq) +static long __calc_smp_shares(struct cfs_rq *cfs_rq, long tg_shares, long shares_max) { - long tg_weight, tg_shares, load, shares; struct task_group *tg = cfs_rq->tg; - - tg_shares = READ_ONCE(tg->shares); + long tg_weight, load, shares; load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); @@ -5036,7 +5063,120 @@ * case no task is runnable on a CPU MIN_SHARES=2 should be returned * instead of 0. */ - return clamp_t(long, shares, MIN_SHARES, tg_shares); + return clamp_t(long, shares, MIN_SHARES, shares_max); +} + +static int tg_cpus(struct task_group *tg) +{ + int nr = num_online_cpus(); + + if (cpusets_enabled()) { + struct cgroup *cgrp = tg->css.cgroup; + if (cgrp) + nr = cpuset_num_cpus(cgrp); + } + + /* + * An empty cpuset would propagate a 0 shares_max into + * __calc_smp_shares(), where clamp() yields hi when hi < lo and so + * defeats the MIN_SHARES floor. Match tg_tasks(), which floors at 1. + */ + return max(nr, 1); +} + +static inline int tg_tasks(struct task_group *tg) +{ + return max(1, atomic_long_read(&tg->runnable_avg) >> SCHED_CAPACITY_SHIFT); +} + +/* + * Func: fraction(nr_tasks * tg->shares) + * + * Scale tg->shares by the number of tasks. + */ +static long calc_tasks_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + int nr = tg_tasks(tg); + long tg_shares = READ_ONCE(tg->shares); + return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares); +} + +/* + * Func: min(fraction(nr_cpus * tg->shares), nice -20) + * + * Scale tg->shares by the maximal number of CPUs; but clip the max shares at + * nice -20, otherwise a single spinner on a 512 CPU machine would result in + * 512*NICE_0_LOAD, which is also crazy. + */ +static long calc_max_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + int nr = tg_cpus(tg); + long tg_shares = READ_ONCE(tg->shares); + long max_shares = scale_load(sched_prio_to_weight[0]); + return __calc_smp_shares(cfs_rq, tg_shares * nr, max_shares); +} + +/* + * Func: fraction(nr * tg->shares); nr = min(nr_tasks, nr_cpus) + * + * Scales between "smp" and "max" in a natural way. No longer needs clipping + * since there are no unnatural inflations like with "max". + */ +static long calc_concur_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + int nr = min(tg_tasks(tg), tg_cpus(tg)); + long tg_shares = READ_ONCE(tg->shares); + return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares); +} + +/* + * Func: fraction(tg->shares) + * + * This infamously results in tiny shares when you have many CPUs. + */ +static long calc_smp_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + long tg_shares = READ_ONCE(tg->shares); + return __calc_smp_shares(cfs_rq, tg_shares, tg_shares); +} + +/* + * Ignore this pesky SMP stuff, use (4). + */ +static long calc_up_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + return READ_ONCE(tg->shares); +} + +DEFINE_STATIC_CALL(calc_group_shares, calc_concur_shares); + +void __sched_cgroup_mode_update(int mode) +{ + long (*func)(struct cfs_rq *); + switch (mode) { + case 0: + func = &calc_up_shares; + break; + case 1: + func = &calc_smp_shares; + break; + case 2: + default: + func = &calc_concur_shares; + break; + case 3: + func = &calc_max_shares; + break; + case 4: + func = &calc_tasks_shares; + break; + } + static_call_update(calc_group_shares, func); } /* @@ -5055,9 +5195,8 @@ if (!gcfs_rq || !gcfs_rq->load.weight) return; - shares = calc_group_shares(gcfs_rq); - if (unlikely(se->load.weight != shares)) - reweight_entity(cfs_rq_of(se), se, shares); + shares = static_call(calc_group_shares)(gcfs_rq); + reweight_entity(cfs_rq_of(se), se, shares); } #else /* !CONFIG_FAIR_GROUP_SCHED: */ @@ -5175,11 +5314,11 @@ * differential update where we store the last value we propagated. This in * turn allows skipping updates if the differential is 'small'. * - * Updating tg's load_avg is necessary before update_cfs_share(). + * Updating tg's load_avg is necessary before update_cfs_group(). */ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) { - long delta; + long dl, dr; u64 now; /* @@ -5200,17 +5339,21 @@ if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC) return; - delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; - if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { - atomic_long_add(delta, &cfs_rq->tg->load_avg); + dl = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; + dr = cfs_rq->avg.runnable_avg - cfs_rq->tg_runnable_avg_contrib; + if (abs(dl) > cfs_rq->tg_load_avg_contrib / 64 || + abs(dr) > cfs_rq->tg_runnable_avg_contrib / 64) { + atomic_long_add(dl, &cfs_rq->tg->load_avg); + atomic_long_add(dr, &cfs_rq->tg->runnable_avg); cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; + cfs_rq->tg_runnable_avg_contrib = cfs_rq->avg.runnable_avg; cfs_rq->last_update_tg_load_avg = now; } } static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) { - long delta; + long dl, dr; u64 now; /* @@ -5220,9 +5363,12 @@ return; now = rq_clock(rq_of(cfs_rq)); - delta = 0 - cfs_rq->tg_load_avg_contrib; - atomic_long_add(delta, &cfs_rq->tg->load_avg); + dl = 0 - cfs_rq->tg_load_avg_contrib; + dr = 0 - cfs_rq->tg_runnable_avg_contrib; + atomic_long_add(dl, &cfs_rq->tg->load_avg); + atomic_long_add(dr, &cfs_rq->tg->runnable_avg); cfs_rq->tg_load_avg_contrib = 0; + cfs_rq->tg_runnable_avg_contrib = 0; cfs_rq->last_update_tg_load_avg = now; } @@ -5636,7 +5782,7 @@ * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) * avg. The immediate corollary is that all (fair) tasks must be attached. * - * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example. + * cfs_rq->avg is used for task_h_load() and update_cfs_group() for example. * * Return: true if the load decayed or we removed load. * @@ -6174,6 +6320,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { u64 vslice, vruntime = avg_vruntime(cfs_rq); + unsigned int nr_queued = cfs_rq->h_nr_queued; bool update_zero = false; s64 lag = 0; @@ -6181,6 +6328,9 @@ se->slice = sysctl_sched_base_slice; vslice = calc_delta_fair(se->slice, se); + if (flags & ENQUEUE_QUEUED) + nr_queued -= 1; + /* * Due to how V is constructed as the weighted average of entities, * adding tasks with positive lag, or removing tasks with negative lag @@ -6189,7 +6339,7 @@ * * EEVDF: placement strategy #1 / #2 */ - if (sched_feat(PLACE_LAG) && cfs_rq->nr_queued && se->vlag) { + if (sched_feat(PLACE_LAG) && nr_queued && se->vlag) { struct sched_entity *curr = cfs_rq->curr; long load, weight; @@ -6249,9 +6399,9 @@ */ load = cfs_rq->sum_weight; if (curr && curr->on_rq) - load += avg_vruntime_weight(cfs_rq, curr->load.weight); + load += avg_vruntime_weight(cfs_rq, curr->h_load.weight); - weight = avg_vruntime_weight(cfs_rq, se->load.weight); + weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); lag *= load + weight; if (WARN_ON_ONCE(!load)) load = 1; @@ -6310,22 +6460,8 @@ static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq); static void -requeue_delayed_entity(struct sched_entity *se); - -static void enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { - bool curr = cfs_rq->curr == se; - - /* - * If we're the current task, we must renormalise before calling - * update_curr(). - */ - if (curr) - place_entity(cfs_rq, se, flags); - - update_curr(cfs_rq); - /* * When enqueuing a sched_entity, we must: * - Update loads to have both entity and cfs_rq synced with now. @@ -6344,13 +6480,6 @@ */ update_cfs_group(se); - /* - * XXX now that the entity has been re-weighted, and it's lag adjusted, - * we can place the entity. - */ - if (!curr) - place_entity(cfs_rq, se, flags); - account_entity_enqueue(cfs_rq, se); /* Entity has migrated, no longer consider this task hot */ @@ -6359,8 +6488,6 @@ check_schedstat_required(); update_stats_enqueue_fair(cfs_rq, se, flags); - if (!curr) - __enqueue_entity(cfs_rq, se); se->on_rq = 1; if (cfs_rq->nr_queued == 1) { @@ -6378,21 +6505,19 @@ } } -static void __clear_buddies_next(struct sched_entity *se) +static void set_next_buddy(struct cfs_rq *cfs_rq, struct sched_entity *se) { - for_each_sched_entity(se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (cfs_rq->next != se) - break; - - cfs_rq->next = NULL; - } + if (WARN_ON_ONCE(!se->on_rq || se->sched_delayed)) + return; + if (se_is_idle(se)) + return; + cfs_rq->next = se; } static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) { if (cfs_rq->next == se) - __clear_buddies_next(se); + cfs_rq->next = NULL; } static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); @@ -6401,7 +6526,7 @@ { /* * Delayed se of cfs_rq have no tasks queued on them. - * Do not adjust h_nr_runnable since dequeue_entities() + * Do not adjust h_nr_runnable since __dequeue_task() * will account it for blocked tasks. * * This check can be removed because when flat pick @@ -6455,45 +6580,16 @@ } } -static bool +static void dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { - bool sleep = flags & DEQUEUE_SLEEP; - int action = 0; - - update_curr(cfs_rq); - clear_buddies(cfs_rq, se); + int action = UPDATE_TG; - if (flags & DEQUEUE_DELAYED) { - WARN_ON_ONCE(!se->sched_delayed); - } else { - bool delay = sleep; - /* - * DELAY_DEQUEUE relies on spurious wakeups, special task - * states must not suffer spurious wakeups, excempt them. - */ - if (flags & (DEQUEUE_SPECIAL | DEQUEUE_THROTTLE)) - delay = false; - - WARN_ON_ONCE(delay && se->sched_delayed); - - if (sched_feat(DELAY_DEQUEUE) && delay && - !entity_eligible(cfs_rq, se)) { - if (entity_is_task(se)) - action |= UPDATE_UTIL_EST; - update_load_avg(cfs_rq, se, action); - update_entity_lag(cfs_rq, se); - set_delayed(se); - return false; - } - } - - action = UPDATE_TG; if (entity_is_task(se)) { if (task_on_rq_migrating(task_of(se))) action |= DO_DETACH; - if (sleep && !(flags & DEQUEUE_DELAYED)) + if ((flags & DEQUEUE_SLEEP) && !(flags & DEQUEUE_DELAYED)) action |= UPDATE_UTIL_EST; } @@ -6511,14 +6607,6 @@ update_stats_dequeue_fair(cfs_rq, se, flags); - update_entity_lag(cfs_rq, se); - if (sched_feat(PLACE_REL_DEADLINE) && !sleep) { - se->deadline -= se->vruntime; - se->rel_deadline = 1; - } - - if (se != cfs_rq->curr) - __dequeue_entity(cfs_rq, se); se->on_rq = 0; account_entity_dequeue(cfs_rq, se); @@ -6527,9 +6615,6 @@ update_cfs_group(se); - if (flags & DEQUEUE_DELAYED) - clear_delayed(se); - if (cfs_rq->nr_queued == 0) { update_idle_cfs_rq_clock_pelt(cfs_rq); #ifdef CONFIG_CFS_BANDWIDTH @@ -6542,15 +6627,11 @@ } #endif } - - return true; } static void -set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first) +set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - clear_buddies(cfs_rq, se); - /* 'current' is not kept within the tree. */ if (se->on_rq) { /* @@ -6559,16 +6640,12 @@ * runqueue. */ update_stats_wait_end_fair(cfs_rq, se); - __dequeue_entity(cfs_rq, se); update_load_avg(cfs_rq, se, UPDATE_TG); - - if (first) - set_protect_slice(cfs_rq, se); } update_stats_curr_start(cfs_rq, se); - WARN_ON_ONCE(cfs_rq->curr); - cfs_rq->curr = se; + WARN_ON_ONCE(cfs_rq->h_curr); + cfs_rq->h_curr = se; /* * Track our maximum slice length, if the CPU's load is at @@ -6588,23 +6665,17 @@ se->prev_sum_exec_runtime = se->sum_exec_runtime; } -static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags); +static bool __dequeue_task(struct rq *rq, struct task_struct *p, int flags); -/* - * Pick the next process, keeping these things in mind, in this order: - * 1) keep things fair between processes/task groups - * 2) pick the "next" process, since someone really wants that to run - * 3) pick the "last" process, for cache locality - * 4) do not run the "skip" process, if something else is available - */ static struct sched_entity * -pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq, bool protect) +pick_next_entity(struct rq *rq, bool protect) { + struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *se; se = pick_eevdf(cfs_rq, protect); if (se->sched_delayed) { - dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); + __dequeue_task(rq, task_of(se), DEQUEUE_SLEEP | DEQUEUE_DELAYED); /* * Must not reference @se again, see __block_task(). */ @@ -6624,13 +6695,11 @@ if (prev->on_rq) { update_stats_wait_start_fair(cfs_rq, prev); - /* Put 'current' back into the tree. */ - __enqueue_entity(cfs_rq, prev); /* in !on_rq case, update occurred at dequeue */ update_load_avg(cfs_rq, prev, 0); } - WARN_ON_ONCE(cfs_rq->curr != prev); - cfs_rq->curr = NULL; + WARN_ON_ONCE(cfs_rq->h_curr != prev); + cfs_rq->h_curr = NULL; } static void @@ -7185,7 +7254,7 @@ assert_list_leaf_cfs_rq(rq); /* Determine whether we need to wake up potentially idle CPU: */ - if (rq->curr == rq->idle && rq->cfs.nr_queued) + if (rq->curr == rq->idle && rq->cfs.h_nr_queued) resched_curr(rq); } @@ -7520,7 +7589,7 @@ return; /* an active group must be handled by the update_curr() path */ - if (!cfs_rq->runtime_enabled || cfs_rq->curr) + if (!cfs_rq->runtime_enabled || cfs_rq->h_curr) return; /* ensure the group is not already throttled */ @@ -7892,7 +7961,7 @@ resched_curr(rq); return; } - delta = (se->load.weight * vdelta) / NICE_0_LOAD; + delta = (se->h_load.weight * vdelta) / NICE_0_LOAD; /* * Correct for instantaneous load of other classes. @@ -7992,10 +8061,8 @@ } static void -requeue_delayed_entity(struct sched_entity *se) +requeue_delayed_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - /* * se->sched_delayed should imply: se->on_rq == 1. * Because a delayed entity is one that is still on @@ -8007,19 +8074,58 @@ update_curr(cfs_rq); if (update_entity_lag(cfs_rq, se)) { - cfs_rq->nr_queued--; + cfs_rq->h_nr_queued--; if (se != cfs_rq->curr) __dequeue_entity(cfs_rq, se); place_entity(cfs_rq, se, 0); if (se != cfs_rq->curr) __enqueue_entity(cfs_rq, se); - cfs_rq->nr_queued++; + cfs_rq->h_nr_queued++; } update_load_avg(cfs_rq, se, 0); clear_delayed(se); } +static unsigned long enqueue_hierarchy(struct task_struct *p, int flags) +{ + unsigned long weight = NICE_0_LOAD; + int task_new = !(flags & ENQUEUE_WAKEUP); + struct sched_entity *se = &p->se; + int h_nr_idle = task_has_idle_policy(p); + int h_nr_runnable = 1; + + if (task_new && se->sched_delayed) + h_nr_runnable = 0; + + for_each_sched_entity(se) { + struct cfs_rq *cfs_rq = cfs_rq_of(se); + + update_curr(cfs_rq); + + if (!se->on_rq) { + enqueue_entity(cfs_rq, se, flags); + } else { + update_load_avg(cfs_rq, se, UPDATE_TG); + se_update_runnable(se); + update_cfs_group(se); + } + + cfs_rq->h_nr_runnable += h_nr_runnable; + cfs_rq->h_nr_queued++; + cfs_rq->h_nr_idle += h_nr_idle; + + if (cfs_rq_is_idle(cfs_rq)) + h_nr_idle = 1; + + weight = __calc_prop_weight(cfs_rq, se, weight); + + flags = ENQUEUE_WAKEUP; + } + + return weight; +} + /* * The enqueue_task method is called before nr_running is * increased. Here we update the fair scheduling stats and @@ -8028,13 +8134,12 @@ static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) { - struct cfs_rq *cfs_rq; - struct sched_entity *se = &p->se; - int h_nr_idle = task_has_idle_policy(p); - int h_nr_runnable = 1; - int task_new = !(flags & ENQUEUE_WAKEUP); int rq_h_nr_queued = rq->cfs.h_nr_queued; - u64 slice = 0; + int task_new = !(flags & ENQUEUE_WAKEUP); + struct sched_entity *se = &p->se; + struct cfs_rq *cfs_rq = &rq->cfs; + unsigned long weight; + bool curr; if (task_is_throttled(p) && enqueue_throttled_task(p)) return; @@ -8046,10 +8151,10 @@ * estimated utilization, before we update schedutil. */ if (!p->se.sched_delayed || (flags & ENQUEUE_DELAYED)) - util_est_enqueue(&rq->cfs, p); + util_est_enqueue(cfs_rq, p); if (flags & ENQUEUE_DELAYED) { - requeue_delayed_entity(se); + requeue_delayed_entity(cfs_rq, se); return; } @@ -8061,57 +8166,22 @@ if (p->in_iowait) cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); - if (task_new && se->sched_delayed) - h_nr_runnable = 0; - - for_each_sched_entity(se) { - if (se->on_rq) { - if (se->sched_delayed) - requeue_delayed_entity(se); - break; - } - cfs_rq = cfs_rq_of(se); - - /* - * Basically set the slice of group entries to the min_slice of - * their respective cfs_rq. This ensures the group can service - * its entities in the desired time-frame. - */ - if (slice) { - se->slice = slice; - se->custom_slice = 1; - } - enqueue_entity(cfs_rq, se, flags); - slice = cfs_rq_min_slice(cfs_rq); - - cfs_rq->h_nr_runnable += h_nr_runnable; - cfs_rq->h_nr_queued++; - cfs_rq->h_nr_idle += h_nr_idle; - - if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = 1; - - flags = ENQUEUE_WAKEUP; - } - - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - - update_load_avg(cfs_rq, se, UPDATE_TG); - se_update_runnable(se); - update_cfs_group(se); + /* + * XXX comment on the curr thing + */ + curr = (cfs_rq->curr == se); + if (curr) + place_entity(cfs_rq, se, flags); - se->slice = slice; - if (se != cfs_rq->curr) - min_vruntime_cb_propagate(&se->run_node, NULL); - slice = cfs_rq_min_slice(cfs_rq); + if (se->on_rq && se->sched_delayed) + requeue_delayed_entity(cfs_rq, se); - cfs_rq->h_nr_runnable += h_nr_runnable; - cfs_rq->h_nr_queued++; - cfs_rq->h_nr_idle += h_nr_idle; + weight = enqueue_hierarchy(p, flags); - if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = 1; + if (!curr) { + reweight_eevdf(cfs_rq, se, weight, false); + place_entity(cfs_rq, se, flags | ENQUEUE_QUEUED); + __enqueue_entity(cfs_rq, se); } if (!rq_h_nr_queued && rq->cfs.h_nr_queued) @@ -8142,105 +8212,109 @@ hrtick_update(rq); } -/* - * Basically dequeue_task_fair(), except it can deal with dequeue_entity() - * failing half-way through and resume the dequeue later. - * - * Returns: - * -1 - dequeue delayed - * 0 - dequeue throttled - * 1 - dequeue complete - */ -static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) +static void dequeue_hierarchy(struct task_struct *p, int flags) { - bool was_sched_idle = sched_idle_rq(rq); + struct sched_entity *se = &p->se; bool task_sleep = flags & DEQUEUE_SLEEP; bool task_delayed = flags & DEQUEUE_DELAYED; bool task_throttled = flags & DEQUEUE_THROTTLE; - struct task_struct *p = NULL; - int h_nr_idle = 0; - int h_nr_queued = 0; int h_nr_runnable = 0; - struct cfs_rq *cfs_rq; - u64 slice = 0; + int h_nr_idle = task_has_idle_policy(p); + bool dequeue = true; - if (entity_is_task(se)) { - p = task_of(se); - h_nr_queued = 1; - h_nr_idle = task_has_idle_policy(p); - if (task_sleep || task_delayed || !se->sched_delayed) - h_nr_runnable = 1; - } + if (task_sleep || task_delayed || !se->sched_delayed) + h_nr_runnable = 1; for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); + struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (!dequeue_entity(cfs_rq, se, flags)) { - if (p && &p->se == se) - return -1; + update_curr(cfs_rq); - slice = cfs_rq_min_slice(cfs_rq); - break; + if (dequeue) { + dequeue_entity(cfs_rq, se, flags); + /* Don't dequeue parent if it has other entities besides us */ + if (cfs_rq->load.weight) + dequeue = false; + } else { + update_load_avg(cfs_rq, se, UPDATE_TG); + se_update_runnable(se); + update_cfs_group(se); } cfs_rq->h_nr_runnable -= h_nr_runnable; - cfs_rq->h_nr_queued -= h_nr_queued; + cfs_rq->h_nr_queued--; cfs_rq->h_nr_idle -= h_nr_idle; if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = h_nr_queued; + h_nr_idle = 1; if (throttled_hierarchy(cfs_rq) && task_throttled) record_throttle_clock(cfs_rq); - /* Don't dequeue parent if it has other entities besides us */ - if (cfs_rq->load.weight) { - slice = cfs_rq_min_slice(cfs_rq); - - /* Avoid re-evaluating load for this entity: */ - se = parent_entity(se); - /* - * Bias pick_next to pick a task from this cfs_rq, as - * p is sleeping when it is within its sched_slice. - */ - if (task_sleep && se) - set_next_buddy(se); - break; - } flags |= DEQUEUE_SLEEP; flags &= ~(DEQUEUE_DELAYED | DEQUEUE_SPECIAL); } +} - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); +/* + * The part of dequeue_task_fair() that is needed to dequeue delayed tasks. + * + * Returns: + * true - dequeued + * false - delayed + */ +static bool __dequeue_task(struct rq *rq, struct task_struct *p, int flags) +{ + struct sched_entity *se = &p->se; + struct cfs_rq *cfs_rq = &rq->cfs; + bool was_sched_idle = sched_idle_rq(rq); + bool task_sleep = flags & DEQUEUE_SLEEP; + bool task_delayed = flags & DEQUEUE_DELAYED; - update_load_avg(cfs_rq, se, UPDATE_TG); - se_update_runnable(se); - update_cfs_group(se); + clear_buddies(cfs_rq, se); - se->slice = slice; - if (se != cfs_rq->curr) - min_vruntime_cb_propagate(&se->run_node, NULL); - slice = cfs_rq_min_slice(cfs_rq); + update_curr(cfs_rq_of(se)); + update_entity_lag(cfs_rq, se); - cfs_rq->h_nr_runnable -= h_nr_runnable; - cfs_rq->h_nr_queued -= h_nr_queued; - cfs_rq->h_nr_idle -= h_nr_idle; + if (flags & DEQUEUE_DELAYED) { + WARN_ON_ONCE(!se->sched_delayed); + } else { + bool delay = task_sleep; + /* + * DELAY_DEQUEUE relies on spurious wakeups, special task + * states must not suffer spurious wakeups, excempt them. + */ + if (flags & (DEQUEUE_SPECIAL | DEQUEUE_THROTTLE)) + delay = false; - if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = h_nr_queued; + WARN_ON_ONCE(delay && se->sched_delayed); - if (throttled_hierarchy(cfs_rq) && task_throttled) - record_throttle_clock(cfs_rq); + if (sched_feat(DELAY_DEQUEUE) && delay && + !entity_eligible(cfs_rq, se)) { + update_load_avg(cfs_rq_of(se), se, UPDATE_UTIL_EST); + set_delayed(se); + return false; + } } - sub_nr_running(rq, h_nr_queued); + dequeue_hierarchy(p, flags); + + if (sched_feat(PLACE_REL_DEADLINE) && !task_sleep) { + se->deadline -= se->vruntime; + se->rel_deadline = 1; + } + if (se != cfs_rq->curr) + __dequeue_entity(cfs_rq, se); + + sub_nr_running(rq, 1); /* balance early to pull high priority tasks */ if (unlikely(!was_sched_idle && sched_idle_rq(rq))) rq->next_balance = jiffies; - if (p && task_delayed) { + if (task_delayed) { + clear_delayed(se); + WARN_ON_ONCE(!task_sleep); WARN_ON_ONCE(p->on_rq != 1); @@ -8252,7 +8326,7 @@ __block_task(rq, p); } - return 1; + return true; } /* @@ -8270,11 +8344,11 @@ if (!p->se.sched_delayed) util_est_dequeue(&rq->cfs, p); - if (dequeue_entities(rq, &p->se, flags) < 0) + if (!__dequeue_task(rq, p, flags)) return false; /* - * Must not reference @p after dequeue_entities(DEQUEUE_DELAYED). + * Must not reference @p after __dequeue_task(DEQUEUE_DELAYED). */ return true; } @@ -8695,6 +8769,20 @@ } /* + * Check if the entire core (all SMT siblings) containing @cpu is idle. + */ +static inline bool is_idle_core(int cpu) +{ + int sibling; + + for_each_cpu(sibling, cpu_smt_mask(cpu)) { + if (!available_idle_cpu(sibling)) + return false; + } + return true; +} + +/* * Scan the entire LLC domain for idle cores; this dynamically switches off if * there are no idle cores left in the system; tracked through * sd_balance_shared->has_idle_cores and enabled through update_idle_core() @@ -9021,16 +9109,38 @@ return true; } +#ifdef CONFIG_SCHED_POC_SELECTOR +#include "poc_selector.c" +#endif /* * Try and locate an idle core/thread in the LLC cache domain. */ -static int select_idle_sibling(struct task_struct *p, int prev, int target) +static int select_idle_sibling(struct task_struct *p, int prev, int target, int sync) { bool has_idle_core = false; struct sched_domain *sd; unsigned long task_util, util_min, util_max; int i, recent_used_cpu, prev_aff = -1; + /* Check a recently used CPU as a potential idle candidate: */ + recent_used_cpu = p->recent_used_cpu; + p->recent_used_cpu = prev; + if (recent_used_cpu != prev && + recent_used_cpu != target && + cpus_share_cache(recent_used_cpu, target) && + choose_idle_cpu(recent_used_cpu, p) && + cpumask_test_cpu(recent_used_cpu, p->cpus_ptr)) { +#ifdef CONFIG_SCHED_POC_SELECTOR + if (!static_branch_likely(&poc_selector_active) || + static_branch_unlikely(&sched_poc_early_select)) +#endif + if ((unsigned int)recent_used_cpu < nr_cpumask_bits && + is_idle_core(recent_used_cpu)) + return recent_used_cpu; + } else { + recent_used_cpu = -1; + } + /* * On asymmetric system, update task utilization because we will check * that the task fits with CPU's capacity. @@ -9047,23 +9157,13 @@ */ lockdep_assert_irqs_disabled(); - if (choose_idle_cpu(target, p) && +#ifdef CONFIG_SCHED_POC_SELECTOR + if (static_branch_likely(&poc_selector_active) && + static_branch_unlikely(&sched_poc_early_select) && + is_idle_core(target) && asym_fits_cpu(task_util, util_min, util_max, target)) return target; - - /* - * If the previous CPU is cache affine and idle, don't be stupid: - */ - if (prev != target && cpus_share_cache(prev, target) && - choose_idle_cpu(prev, p) && - asym_fits_cpu(task_util, util_min, util_max, prev)) { - - if (!static_branch_unlikely(&sched_cluster_active) || - cpus_share_resources(prev, target)) - return prev; - - prev_aff = prev; - } +#endif /* * Allow a per-cpu kthread to stack with the wakee if the @@ -9081,24 +9181,6 @@ return prev; } - /* Check a recently used CPU as a potential idle candidate: */ - recent_used_cpu = p->recent_used_cpu; - p->recent_used_cpu = prev; - if (recent_used_cpu != prev && - recent_used_cpu != target && - cpus_share_cache(recent_used_cpu, target) && - choose_idle_cpu(recent_used_cpu, p) && - cpumask_test_cpu(recent_used_cpu, p->cpus_ptr) && - asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) { - - if (!static_branch_unlikely(&sched_cluster_active) || - cpus_share_resources(recent_used_cpu, target)) - return recent_used_cpu; - - } else { - recent_used_cpu = -1; - } - /* * For asymmetric CPU capacity systems, our domain of interest is * sd_asym_cpucapacity rather than sd_llc. @@ -9123,6 +9205,74 @@ if (!sd) return target; +#ifdef CONFIG_SCHED_POC_SELECTOR + { + struct sched_domain_shared *sd_share = + rcu_dereference(per_cpu(sd_llc_shared, target)); + if (static_branch_likely(&poc_selector_active) + && !sched_asym_cpucap_active() + && sd_share && likely(sd_share->poc_fast_eligible)) { + int poc_cpu = select_idle_cpu_poc(target, prev, + recent_used_cpu, sync, + sd_share, p->cpus_ptr); + if (poc_cpu >= 0) { + return poc_cpu; + } + /* + * POC returns -2 when the SIS_UTIL overload gate fires + * (smt_fallback=0 only). POC has already checked + * prev's SMT sibling (Level 4) and decided broader + * search is not worthwhile. CFS would reach the same + * conclusion, so skip select_idle_smt/select_idle_cpu. + * + * POC returns -1 for Level 0 saturation (no idle CPUs + * in bitmap), but CFS may still find sched_idle CPUs, + * so we must NOT skip CFS in that case. + */ + if (poc_cpu == -2) + goto give_up; + } else { + /* + * poc_selector_active is off — POC is either disabled + * by sysctl or suppressed while scx is running. + * If an scx scheduler called us, flip poc_selector_skip + * and schedule a workqueue item to re-enable POC with + * bitmap resync. + */ + poc_check_skip_fallback(); + } + } + poc_count(POC_FALLBACK); +#endif /* CONFIG_SCHED_POC_SELECTOR */ + + if ((unsigned int)recent_used_cpu < nr_cpumask_bits) { + if (choose_idle_cpu(recent_used_cpu, p)) { + if (is_idle_core(recent_used_cpu)) + return recent_used_cpu; + /* idle CPU but not idle core → preserve for give_up */ + } else { + recent_used_cpu = -1; /* not idle → discard */ + } + } + + if (sync && is_idle_core(target) && + asym_fits_cpu(task_util, util_min, util_max, target)) + return target; + + /* + * If the previous CPU is cache affine and idle, don't be stupid: + */ + if (prev != target && cpus_share_cache(prev, target) && + choose_idle_cpu(prev, p) && + asym_fits_cpu(task_util, util_min, util_max, prev)) { + + if (!static_branch_unlikely(&sched_cluster_active) || + cpus_share_resources(prev, target)) + return prev; + + prev_aff = prev; + } + if (sched_smt_active()) { has_idle_core = test_idle_cores(target); @@ -9137,6 +9287,9 @@ if ((unsigned)i < nr_cpumask_bits) return i; +#ifdef CONFIG_SCHED_POC_SELECTOR +give_up: +#endif /* * For cluster machines which have lower sharing cache like L2 or * LLC Tag, we tend to find an idle CPU in the target's cluster @@ -9148,6 +9301,13 @@ if ((unsigned int)recent_used_cpu < nr_cpumask_bits) return recent_used_cpu; +#ifdef CONFIG_SCHED_POC_SELECTOR + /* Last resort: avoid enqueuing behind RT/DL tasks on target */ + if (static_branch_likely(&poc_selector_active) && + rt_task(cpu_rq(target)->curr) && + prev != target && !rt_task(cpu_rq(prev)->curr)) + return prev; +#endif return target; } @@ -9830,7 +9990,7 @@ /* Fast path */ if (wake_flags & WF_TTWU) - return select_idle_sibling(p, prev_cpu, new_cpu); + return select_idle_sibling(p, prev_cpu, new_cpu, sync); return new_cpu; } @@ -9869,19 +10029,6 @@ static void task_dead_fair(struct task_struct *p) { struct sched_entity *se = &p->se; - - if (se->sched_delayed) { - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); - if (se->sched_delayed) { - update_rq_clock(rq); - dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); - } - task_rq_unlock(rq, p, &rf); - } - remove_entity_load_avg(se); } @@ -9915,21 +10062,10 @@ set_task_max_allowed_capacity(p); } -static void set_next_buddy(struct sched_entity *se) -{ - for_each_sched_entity(se) { - if (WARN_ON_ONCE(!se->on_rq)) - return; - if (se_is_idle(se)) - return; - cfs_rq_of(se)->next = se; - } -} - enum preempt_wakeup_action { PREEMPT_WAKEUP_NONE, /* No preemption. */ PREEMPT_WAKEUP_SHORT, /* Ignore slice protection. */ - PREEMPT_WAKEUP_PICK, /* Let __pick_eevdf() decide. */ + PREEMPT_WAKEUP_PICK, /* Let pick_eevdf() decide. */ PREEMPT_WAKEUP_RESCHED, /* Force reschedule. */ }; @@ -9946,7 +10082,7 @@ if (cfs_rq->next && entity_before(cfs_rq->next, pse)) return false; - set_next_buddy(pse); + set_next_buddy(cfs_rq, pse); return true; } @@ -9999,7 +10135,7 @@ enum preempt_wakeup_action preempt_action = PREEMPT_WAKEUP_PICK; struct task_struct *donor = rq->donor; struct sched_entity *nse, *se = &donor->se, *pse = &p->se; - struct cfs_rq *cfs_rq = task_cfs_rq(donor); + struct cfs_rq *cfs_rq = &rq->cfs; int cse_is_idle, pse_is_idle; /* @@ -10037,7 +10173,6 @@ if (!sched_feat(WAKEUP_PREEMPTION)) return; - find_matching_se(&se, &pse); WARN_ON_ONCE(!pse); cse_is_idle = se_is_idle(se); @@ -10065,8 +10200,7 @@ if (unlikely(!normal_policy(p->policy))) return; - cfs_rq = cfs_rq_of(se); - update_curr(cfs_rq); + update_curr_fair(rq); /* * If @p has a shorter slice than current and @p is eligible, override * current's slice protection in order to allow preemption. @@ -10110,18 +10244,15 @@ } pick: - nse = pick_next_entity(rq, cfs_rq, preempt_action != PREEMPT_WAKEUP_SHORT); - /* If @p has become the most eligible task, force preemption */ - if (nse == pse) - goto preempt; - - /* - * Because p is enqueued, nse being null can only mean that we - * dequeued a delayed task. If there are still entities queued in - * cfs, check if the next one will be p. - */ - if (!nse && cfs_rq->nr_queued) - goto pick; + if (cfs_rq->h_nr_queued) { + nse = pick_next_entity(rq, preempt_action != PREEMPT_WAKEUP_SHORT); + if (unlikely(!nse)) + goto pick; + + /* If @p has become the most eligible task, force preemption */ + if (nse == pse) + goto preempt; + } if (sched_feat(RUN_TO_PARITY)) update_protect_slice(cfs_rq, se); @@ -10140,33 +10271,24 @@ struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf) __must_hold(__rq_lockp(rq)) { + struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *se; - struct cfs_rq *cfs_rq; struct task_struct *p; - bool throttled; int new_tasks; again: - cfs_rq = &rq->cfs; - if (!cfs_rq->nr_queued) + if (!cfs_rq->h_nr_queued) goto idle; - throttled = false; - - do { - /* Might not have done put_prev_entity() */ - if (cfs_rq->curr && cfs_rq->curr->on_rq) - update_curr(cfs_rq); + /* Might not have done put_prev_entity() */ + if (cfs_rq->curr && cfs_rq->curr->on_rq) + update_curr(cfs_rq); - se = pick_next_entity(rq, cfs_rq, true); - if (!se) - goto again; - cfs_rq = group_cfs_rq(se); - } while (cfs_rq); + se = pick_next_entity(rq, true); + if (!se) + goto again; p = task_of(se); - if (unlikely(throttled)) - task_throttle_setup_work(p); return p; idle: @@ -10203,7 +10325,7 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct task_struct *next) { struct sched_entity *se = &prev->se; - struct cfs_rq *cfs_rq; + struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *nse = NULL; #ifdef CONFIG_FAIR_GROUP_SCHED @@ -10213,7 +10335,7 @@ while (se) { cfs_rq = cfs_rq_of(se); - if (!nse || cfs_rq->curr) + if (!nse || cfs_rq->h_curr) put_prev_entity(cfs_rq, se); #ifdef CONFIG_FAIR_GROUP_SCHED if (nse) { @@ -10232,6 +10354,14 @@ #endif se = parent_entity(se); } + + /* Put 'current' back into the tree. */ + cfs_rq = &rq->cfs; + se = &prev->se; + WARN_ON_ONCE(cfs_rq->curr != se); + cfs_rq->curr = NULL; + if (se->on_rq) + __enqueue_entity(cfs_rq, se); } /* @@ -10240,8 +10370,8 @@ static void yield_task_fair(struct rq *rq) { struct task_struct *curr = rq->donor; - struct cfs_rq *cfs_rq = task_cfs_rq(curr); struct sched_entity *se = &curr->se; + struct cfs_rq *cfs_rq = &rq->cfs; /* * Are we the only task in the tree? @@ -10282,11 +10412,11 @@ struct sched_entity *se = &p->se; /* !se->on_rq also covers throttled task */ - if (!se->on_rq) + if (!se->on_rq || se->sched_delayed) return false; /* Tell the scheduler that we'd really like se to run next. */ - set_next_buddy(se); + set_next_buddy(&task_rq(p)->cfs, se); yield_task_fair(rq); @@ -10626,15 +10756,10 @@ */ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_cpu) { - struct cfs_rq *dst_cfs_rq; + struct cfs_rq *dst_cfs_rq = &cpu_rq(dest_cpu)->cfs; -#ifdef CONFIG_FAIR_GROUP_SCHED - dst_cfs_rq = tg_cfs_rq(task_group(p), dest_cpu); -#else - dst_cfs_rq = &cpu_rq(dest_cpu)->cfs; -#endif - if (sched_feat(PLACE_LAG) && dst_cfs_rq->nr_queued && - !entity_eligible(task_cfs_rq(p), &p->se)) + if (sched_feat(PLACE_LAG) && dst_cfs_rq->h_nr_queued && + !entity_eligible(&task_rq(p)->cfs, &p->se)) return 1; return 0; @@ -11468,7 +11593,7 @@ while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) { load = cfs_rq->h_load; load = div64_ul(load * se->avg.load_avg, - cfs_rq_load_avg(cfs_rq) + 1); + cfs_rq_load_avg(cfs_rq) + 1); cfs_rq = group_cfs_rq(se); cfs_rq->h_load = load; cfs_rq->last_h_load_update = now; @@ -13641,12 +13766,15 @@ * still unbalanced. ld_moved simply stays zero, so it is * correctly treated as an imbalance. */ - env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); - more_balance: rq_lock_irqsave(busiest, &rf); update_rq_clock(busiest); + if (!env.loop_max) + env.loop_max = min(sysctl_sched_nr_migrate, busiest->cfs.h_nr_queued); + else + env.loop_max = min(env.loop_max, busiest->cfs.h_nr_queued); + /* * cur_ld_moved - load moved in current iteration * ld_moved - cumulative load moved across iterations @@ -14895,7 +15023,7 @@ * MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check * if we need to give up the CPU. */ - if (rq->core->core_forceidle_count && rq->cfs.nr_queued == 1 && + if (rq->core->core_forceidle_count && rq->cfs.h_nr_queued == 1 && __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE)) resched_curr(rq); } @@ -15104,30 +15232,8 @@ WARN_ON_ONCE(task_rq(b)->core != rq->core); -#ifdef CONFIG_FAIR_GROUP_SCHED - /* - * Find an se in the hierarchy for tasks a and b, such that the se's - * are immediate siblings. - */ - while (sea->cfs_rq->tg != seb->cfs_rq->tg) { - int sea_depth = sea->depth; - int seb_depth = seb->depth; - - if (sea_depth >= seb_depth) - sea = parent_entity(sea); - if (sea_depth <= seb_depth) - seb = parent_entity(seb); - } - - se_fi_update(sea, rq->core->core_forceidle_seq, in_fi); - se_fi_update(seb, rq->core->core_forceidle_seq, in_fi); - - cfs_rqa = sea->cfs_rq; - cfs_rqb = seb->cfs_rq; -#else /* !CONFIG_FAIR_GROUP_SCHED: */ cfs_rqa = &task_rq(a)->cfs; cfs_rqb = &task_rq(b)->cfs; -#endif /* !CONFIG_FAIR_GROUP_SCHED */ /* * Find delta after normalizing se's vruntime with its cfs_rq's @@ -15166,11 +15272,20 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) { struct sched_entity *se = &curr->se; - struct cfs_rq *cfs_rq; - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - entity_tick(cfs_rq, se, queued); + if (se->on_rq) { + unsigned long weight = NICE_0_LOAD; + struct cfs_rq *cfs_rq; + + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + entity_tick(cfs_rq, se, queued); + + weight = __calc_prop_weight(cfs_rq, se, weight); + } + + se = &curr->se; + reweight_eevdf(cfs_rq, se, weight, se->on_rq); } if (queued) @@ -15210,7 +15325,7 @@ if (p->prio == oldprio) return; - if (rq->cfs.nr_queued == 1) + if (rq->cfs.h_nr_queued == 1) return; /* @@ -15339,33 +15454,44 @@ } } -/* - * Account for a task changing its policy or group. - * - * This routine is mostly called to set cfs_rq->curr field when a task - * migrates between groups/classes. - */ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) { struct sched_entity *se = &p->se; bool throttled = false; + struct cfs_rq *cfs_rq = &rq->cfs; + unsigned long weight = NICE_0_LOAD; + bool on_rq = se->on_rq; + + clear_buddies(cfs_rq, se); + + if (on_rq) + __dequeue_entity(cfs_rq, se); for_each_sched_entity(se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); + cfs_rq = cfs_rq_of(se); - if (IS_ENABLED(CONFIG_FAIR_GROUP_SCHED) && - first && cfs_rq->curr) - break; + if (!IS_ENABLED(CONFIG_FAIR_GROUP_SCHED) || + !first || !cfs_rq->h_curr) + set_next_entity(cfs_rq, se); - set_next_entity(cfs_rq, se, first); /* ensure bandwidth has been allocated on our new cfs_rq */ throttled |= account_cfs_rq_runtime(cfs_rq, 0); + + if (on_rq) + weight = __calc_prop_weight(cfs_rq, se, weight); } if (throttled) task_throttle_setup_work(p); se = &p->se; + cfs_rq->curr = se; + + if (on_rq) { + reweight_eevdf(cfs_rq, se, weight, se->on_rq); + if (first) + set_protect_slice(cfs_rq, se); + } if (task_on_rq_queued(p)) { /* @@ -15478,17 +15604,8 @@ struct sched_entity *se = tg_se(tg, cpu); struct rq *rq = cpu_rq(cpu); - if (se) { - if (se->sched_delayed) { - guard(rq_lock_irqsave)(rq); - if (se->sched_delayed) { - update_rq_clock(rq); - dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); - } - list_del_leaf_cfs_rq(cfs_rq); - } + if (se) remove_entity_load_avg(se); - } /* * Only empty task groups can be destroyed; so we can speculatively diff -ruN --no-dereference -x .git a/kernel/sched/idle.c b/kernel/sched/idle.c --- a/kernel/sched/idle.c 2026-10-11 15:27:50.000000000 +0200 +++ b/kernel/sched/idle.c 2026-10-13 14:33:30.000000000 +0200 @@ -305,6 +305,11 @@ __current_set_polling(); tick_nohz_idle_enter(); +#ifdef CONFIG_SCHED_POC_SELECTOR + /* POC Selector: mark CPU as idle */ + set_cpu_idle_state_poc(cpu, 1); +#endif /* CONFIG_SCHED_POC_SELECTOR */ + while (!need_resched()) { /* @@ -358,6 +363,11 @@ arch_cpu_idle_exit(); } +#ifdef CONFIG_SCHED_POC_SELECTOR + /* POC Selector: mark CPU as busy */ + set_cpu_idle_state_poc(cpu, 0); +#endif /* CONFIG_SCHED_POC_SELECTOR */ + /* * Since we fell out of the loop above, we know TIF_NEED_RESCHED must * be set, propagate it into PREEMPT_NEED_RESCHED. diff -ruN --no-dereference -x .git a/kernel/sched/pelt.c b/kernel/sched/pelt.c --- a/kernel/sched/pelt.c 2026-10-11 15:27:50.000000000 +0200 +++ b/kernel/sched/pelt.c 2026-10-13 14:33:30.000000000 +0200 @@ -206,7 +206,7 @@ /* * running is a subset of runnable (weight) so running can't be set if * runnable is clear. But there are some corner cases where the current - * se has been already dequeued but cfs_rq->curr still points to it. + * se has been already dequeued but cfs_rq->h_curr still points to it. * This means that weight will be 0 but not running for a sched_entity * but also for a cfs_rq if the latter becomes idle. As an example, * this happens during sched_balance_newidle() which calls @@ -307,7 +307,7 @@ int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se) { if (___update_load_sum(now, &se->avg, !!se->on_rq, se_runnable(se), - cfs_rq->curr == se)) { + cfs_rq->h_curr == se)) { ___update_load_avg(&se->avg, se_weight(se)); cfs_se_util_change(&se->avg); @@ -323,7 +323,7 @@ if (___update_load_sum(now, &cfs_rq->avg, scale_load_down(cfs_rq->load.weight), cfs_rq->h_nr_runnable, - cfs_rq->curr != NULL)) { + cfs_rq->h_curr != NULL)) { ___update_load_avg(&cfs_rq->avg, 1); trace_pelt_cfs_tp(cfs_rq); diff -ruN --no-dereference -x .git a/kernel/sched/poc_selector.c b/kernel/sched/poc_selector.c --- a/kernel/sched/poc_selector.c 1970-01-01 01:00:00.000000000 +0100 +++ b/kernel/sched/poc_selector.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,2038 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Piece-Of-Cake (POC) CPU Selector + * + * Fast idle CPU selector inspired by RitzDaCat's scx_cake scheduler + * "Piece of Cake" - making idle CPU search a piece of cake! + * + * Tracks idle state in per-LLC atomic64_t bitmaps with lock-free + * atomic64_read/or/andnot for O(1) idle CPU lookup. + * Supports up to 64 CPUs per LLC (single 64-bit word). + * Includes affinity-aware filtering via cpumask intersection. + * + * When the fast path is not eligible (LLC exceeds 64 CPUs), + * returns -1 to let CFS standard select_idle_cpu handle it. + * + * Copyright (C) 2026 Masahito Suzuki + * + * Acknowledgements: + * This work is heavily inspired by RitzDaCat's scx_cake scheduler. + * + * The select_idle_sibling() restructuring on which the POC fast + * path depends (is_idle_core() helper, @sync parameter, hoisting + * the recent_used_cpu check to the function top, switching the + * target fast-return to is_idle_core()) is adapted from: + * + * Andrea Righi, Mario Roy, and Eric Naim + * ("sched/fair: Prefer the previous cpu for wakeup") + * + * - Andrea Righi: is_idle_core() helper and idle-core + * target selection in select_idle_sibling() + * - Mario Roy: prev-cpu preference and recent_used_cpu + * hoisting with idle-core check + * + * Special thanks to the algorithm inventors whose research enabled + * the O(1) techniques used in this implementation: + * + * - Prashant Pandey, Michael A. Bender, Rob Johnson + * ("A Fast x86 Implementation of Select") + * + * - Daniel Lemire + * ("Fast Random Integer Generation in an Interval") + */ + +#ifdef CONFIG_SCHED_POC_SELECTOR + +/************************************************************** + * Version Information: + */ + +#define SCHED_POC_SELECTOR_AUTHOR "Masahito Suzuki" +#define SCHED_POC_SELECTOR_PROGNAME "Piece-Of-Cake (POC) CPU Selector" + +#define SCHED_POC_SELECTOR_VERSION "2.6.3" + +/************************************************************** + * Static keys: + */ + +/* + * Runtime control: poc_selector_active (static key) + * Derived from: sched_poc_selector && !poc_selector_skip + * + * sched_poc_selector: user-visible sysctl (kernel.sched_poc_selector), + * plain bool, default true. + * poc_selector_skip: set true while sched_ext is active to avoid + * idle-bitmap overhead in do_idle. + * poc_selector_active: the actual static key gating all POC hot paths. + * Enabled only when sched_poc_selector && !poc_selector_skip. + * On enable transition, poc_resync_idle_state() is called. + */ +DEFINE_STATIC_KEY_TRUE(poc_selector_active); +static bool sched_poc_selector = true; +static bool poc_selector_skip; + +/* + * SMT fallback control: sched_poc_smt_fallback + * (sysctl kernel.sched_poc_smt_fallback) + * + * When enabled, POC bails out to CFS when no idle + * cores exist (has_idle_cores == false). CFS then handles + * SMT sibling selection via select_idle_smt(prev) and + * nr_idle_scan-limited select_idle_cpu(). + * + * When disabled (default), POC handles SMT sibling selection + * itself, trying prev's SMT sibling for cache locality, then + * LLC-wide RR search for remaining idle CPUs. Level 5/6 is + * gated by nr_idle_scan (SIS_UTIL): when LLC utilization + * exceeds ~85%, broader SMT search is skipped. + */ +DEFINE_STATIC_KEY_FALSE(sched_poc_smt_fallback); + +/* + * SMT consecutive layout: sched_poc_smt_consecutive + * + * When true (default), SMT siblings occupy consecutive LLC-relative + * positions (e.g., CPU 0,1 / 2,3 / ...). The idle core mask is + * derived from the idle CPU mask via bit-parallel operations: + * core_mask = cpu_mask & (cpu_mask >> 1) & 0x5555555555555555ULL + * + * Disabled at boot if non-consecutive 2-way SMT or >2-way SMT + * is detected on any LLC. + */ +DEFINE_STATIC_KEY_TRUE(sched_poc_smt_consecutive); + +/* + * SMT uniform 2-way layout: sched_poc_smt_uniform + * + * When true (default), all cores in every LLC have uniform 2-way SMT + * with a constant stride between siblings. The idle core mask is + * derived at read time via: + * core_mask = cpu_mask & (cpu_mask >> poc_smt_shift) & poc_primary_mask + * + * This covers both consecutive (stride=1) and stride-N (e.g., Intel + * Xeon) layouts without write-path overhead. + * + * When false (>2-way SMT or non-uniform topology), falls back to + * write-time maintenance of poc_idle_cores_mask atomic64_t. + * + * Disabled at boot if any LLC contains non-2-way or non-uniform SMT. + */ +DEFINE_STATIC_KEY_TRUE(sched_poc_smt_uniform); + +/* + * Target CPU sticky: sched_poc_target_sticky + * (sysctl kernel.sched_poc_target_sticky) + * + * When enabled, if the target CPU is idle in the bitmap, return it + * immediately — regardless of whether its core is fully idle. + * This provides L1 cache affinity: the waking task reuses the CPU + * it ran on last, keeping warm TLB/L1/L2 state. + * + * Checked after Level 0 (saturation) and before core_mask derivation. + * Default: disabled. + */ +DEFINE_STATIC_KEY_FALSE(sched_poc_target_sticky); + +/* + * Early select: sched_poc_early_select + * (sysctl kernel.sched_poc_early_select) + * + * When enabled, select_idle_sibling performs idle-core checks + * for recent_used_cpu and target BEFORE entering POC search: + * - recent_used_cpu with fully idle core → return immediately + * (matches upstream CFS Gate 4 behavior) + * - target with fully idle core → return immediately + * (avoids POC overhead: RCU deref, bitmap read, mask ops) + * + * These two checks must be toggled together to preserve POC's + * internal priority order (Level 1r before 1t). Enabling only + * one would let the pre-POC path return a lower-priority result + * before POC can evaluate the higher-priority candidate. + * + * Default: enabled. + */ +DEFINE_STATIC_KEY_TRUE(sched_poc_early_select); + +/* + * Greedy search: sched_poc_greedy_search + * (sysctl kernel.sched_poc_greedy_search) + * + * When enabled, POC always attempts Level 5/6 (LLC-wide SMT sibling + * search) regardless of utilization, ignoring the SIS_UTIL overload + * gate (nr_idle_scan == 0). This may benefit latency-sensitive + * workloads that want to find any idle CPU at all costs. + * + * When disabled, POC skips Level 5/6 under overload, + * returning -2 to also skip CFS fallback search. + * + * Default: enabled. + */ +DEFINE_STATIC_KEY_TRUE(sched_poc_greedy_search); + +/* + * sched_poc_aligned: true when all LLCs have poc_cpu_base aligned to 64 + * + * When true, cpumask-to-POC conversion is a simple word load (zero shift). + * When false (e.g., Threadripper CCDs at CPU 8, 16, ...), bit shifting + * is needed to align cpumask bits with POC's LLC-relative positions. + * Defaults to true; disabled at boot if any LLC has non-aligned base. + */ +DEFINE_STATIC_KEY_TRUE(sched_poc_aligned); + +/* + * Packed priority search: sched_poc_packed + * + * When true (default), per-LLC CPU count is ≤ 32, enabling packed + * priority search. Cluster candidates (Level 2) and LLC-wide + * candidates (Level 3) are packed into a single 64-bit word: + * + * bits [31:0]: cluster idle candidates (high priority) + * bits [63:32]: all LLC idle candidates (low priority) + * + * A single TZCNT resolves both levels simultaneously. + * ror32-based rotation distributes selections across idle CPUs. + * + * When false (LLC > 32 CPUs), falls back to separate cluster + * search + PTSELECT-based RR. + * + * Disabled at boot if any LLC has > 32 CPUs. + */ +DEFINE_STATIC_KEY_TRUE(sched_poc_packed); + +/* + * Improved RR strategy: sched_poc_rr_improved + * (sysctl kernel.sched_poc_rr_improved) + * + * When enabled (default), idle CPU selection in poc_select_rr, + * poc_cluster_search, and the packed priority search uses an + * improved RR strategy combining two techniques: + * 1. total size case-split (1/2/>=3): direct / interleave / full + * 2. golden-ratio scrambling (Lemire fastrange) + * + * When disabled, the current strategy is used unchanged: + * - poc_select_rr: poc_rr_step[] table (perfect RR) + * - poc_cluster_search: ctz lowest-bit selection (no RR) + * - packed search: ror32(counter & 31) + * + * The current path is preserved as the A/B-testing baseline; + * once the improved path is validated, the legacy code will + * be removed in a follow-up. + */ +DEFINE_STATIC_KEY_TRUE(sched_poc_rr_improved); + +/* + * Lockless bitmap mode: sched_poc_lockless_bitmap + * (sysctl kernel.sched_poc_lockless_bitmap) + * + * When enabled, idle state is tracked in u8[64] flag arrays. + * Writers use plain WRITE_ONCE (no LOCK prefix); readers snapshot + * the 64-byte cache line to the stack, then use multiply-and-shift + * aggregation to assemble a u64 bitmask. + * + * When disabled (default), idle state is tracked in atomic64_t bitmaps. + * Readers use a single atomic64_read (MOV on x86); writers use + * atomic64_or / atomic64_andnot (LOCK'd on x86). + * + * Only one representation is maintained at a time (single-write). + * Switching via sysctl resyncs the newly-active representation + * before readers can observe it. + * + * Default: disabled. + */ +DEFINE_STATIC_KEY_FALSE(sched_poc_lockless_bitmap); + +/************************************************************** + * Debug counters (sysctl kernel.sched_poc_count): + * + * Per-CPU counters for each selection level hit. + * Guarded by static key — zero overhead when disabled (default). + * Aggregated across all CPUs and exposed via sysfs. + */ +enum poc_level { + POC_LV1S = 0, /* target CPU sticky (L1/TLB affinity) */ + POC_LV1T, /* target core idle */ + POC_LV1P, /* prev core idle */ + POC_LV1R, /* recent core idle */ + POC_LV2, /* idle core in L2 cluster */ + POC_LV3, /* idle core across LLC (RR) */ + POC_LV4S, /* sync + target CPU idle (no idle cores) */ + POC_LV4P, /* prev's SMT sibling (cache locality) */ + POC_LV4R, /* recent's SMT sibling (warm cache) */ + POC_LV4T, /* target's SMT sibling */ + POC_LV5, /* idle CPU in L2 cluster */ + POC_LV6, /* idle CPU across LLC (RR) */ + POC_FALLBACK, /* POC returned -1, CFS fallback */ + POC_NR_LEVELS +}; + +#define POC_SMT_LEVEL_OFFSET (POC_LV5 - POC_LV2) + +DEFINE_STATIC_KEY_FALSE(sched_poc_count_enabled); + +static DEFINE_PER_CPU(unsigned long[POC_NR_LEVELS], poc_debug_cnt); + +static __always_inline void poc_count(enum poc_level lv) +{ + if (static_branch_unlikely(&sched_poc_count_enabled)) + __this_cpu_inc(poc_debug_cnt[lv]); +} + +/************************************************************** + * Per-CPU round-robin counter and division-free mapping: + */ + +/* + * POC_HASH_MULT / POC_SCRAMBLE — Golden-ratio scrambling + * + * Multiplying a 32-bit counter by ⌊2^32 / φ⌋ = 0x9E3779B9 scatters + * consecutive values across the 32-bit output space with good + * avalanche properties (Knuth's multiplicative hash, TAOCP Vol. 3). + * The scrambled value feeds POC_FASTRANGE for uniform [0, range) + * mapping in the improved RR path, or is used directly with a bit + * shift to derive an uncorrelated rotation amount in packed search. + */ +#define POC_HASH_MULT 0x9E3779B9U /* golden ratio * 2^32 */ +#define POC_SCRAMBLE(counter) ((u32)(counter) * POC_HASH_MULT) + +/* + * Per-CPU round-robin counter for idle CPU selection. + * Each CPU starts at a different offset to reduce cross-CPU + * collision probability. Combined with poc_rr_step[] and + * POC_FIXED_MOD16, consecutive calls on the same CPU produce + * perfect round-robin: each call picks a different idle CPU + * until all candidates have been visited. + */ +static DEFINE_PER_CPU(u32, poc_rr_counter); + +/* + * Division-free modulo via 16-bit fixed-point reciprocal multiplication + * + * The multiply-and-shift technique is inspired by: + * D. Lemire, "Fast Random Integer Generation in an Interval", + * ACM Trans. Model. Comput. Simul. 29, 1, Article 3, 2019. + * + * Combined with poc_rr_step[], this replaces modulo with two + * multiplications and a shift: + * phase = (u16)(counter * poc_rr_step[total - 1]) + * pick = POC_FIXED_MOD16(phase, total) + * + * Proof that pick == counter % total (for total ≤ 64): + * Let S = ceil(2^16 / N). For k in [0, N): + * k*S*N / 2^16 ∈ [k, k + kN/2^16) + * Since kN < N² ≤ 64² = 4096 ≪ 2^16, floor(kN/2^16) = 0, + * so floor(k*S*N / 2^16) = k. QED. + */ +#define POC_FIXED_MOD16(phase, range) ((u32)(((u32)(phase) * (u32)(range)) >> 16)) + +/* + * POC_FASTRANGE — Map a 32-bit scrambled value to [0, range) + * + * Implements Lemire's fastrange technique: + * D. Lemire, "Fast Random Integer Generation in an Interval", + * ACM Trans. Model. Comput. Simul. 29, 1, Article 3, 2019. + * + * Computes (seed * range) >> 32, giving a uniform mapping of + * a 32-bit seed into [0, range) using only one 64-bit multiply + * and a shift. Used with golden-ratio hashing for pseudo-random + * RR distribution in the improved RR path. + */ +#define POC_FASTRANGE(seed, range) ((u32)(((u64)(seed) * (u32)(range)) >> 32)) + +/* + * RR step table: poc_rr_step[n-1] = ceil(2^16 / n) for n = 1..64 + * + * Indexed by (total - 1) where total = popcount(idle mask). + * total == 0 is unreachable (caller guarantees mask != 0). + * 64 entries × 2 bytes = 128 bytes = exactly 2 cache lines. + */ +static const u16 poc_rr_step[64] = { + 0, 0x8000, 0x5556, 0x4000, 0x3334, 0x2AAB, 0x2493, 0x2000, /* 1.. 8 */ + 0x1C72, 0x199A, 0x1746, 0x1556, 0x13B2, 0x124A, 0x1112, 0x1000, /* 9..16 */ + 0x0F10, 0x0E39, 0x0D7A, 0x0CCD, 0x0C31, 0x0BA3, 0x0B22, 0x0AAB, /* 17..24 */ + 0x0A3E, 0x09D9, 0x097C, 0x0925, 0x08D4, 0x0889, 0x0843, 0x0800, /* 25..32 */ + 0x07C2, 0x0788, 0x0751, 0x071D, 0x06EC, 0x06BD, 0x0691, 0x0667, /* 33..40 */ + 0x063F, 0x0619, 0x05F5, 0x05D2, 0x05B1, 0x0591, 0x0573, 0x0556, /* 41..48 */ + 0x053A, 0x051F, 0x0506, 0x04ED, 0x04D5, 0x04BE, 0x04A8, 0x0493, /* 49..56 */ + 0x047E, 0x046A, 0x0457, 0x0445, 0x0433, 0x0422, 0x0411, 0x0400, /* 57..64 */ +}; + +/************************************************************** + * Bit manipulation primitives: + */ + +/* + * POC_CTZ64 — Portable Count Trailing Zeros (64-bit) + * + * Three-tier architecture detection: + * + * Tier 1: Native hardware CTZ with well-defined zero semantics + * x86-64 + BMI1 (__BMI__): TZCNT — returns 64 for input 0 + * ARM64: RBIT + CLZ + * RISC-V Zbb: CTZ instruction + * + * Tier 2: x86-64 without BMI1 (Bulldozer, pre-Haswell, etc.) + * BSF is fast (~3 cyc) but UNDEFINED for input 0. + * On AMD Bulldozer: BSF(0) leaves dest register unchanged (stale value). + * On Intel pre-Haswell: BSF(0) is architecturally undefined. + * Wrap with explicit zero check to guarantee returning 64. + * + * Tier 3: De Bruijn fallback (BPF, unknown architectures) + * Software multiply + 64-entry table lookup, branchless O(1). + */ + +/* + * POC_CTZ64 is defined in sched.h for use by load balancer functions. + * Here we only define POC_CTZ64_NAME for sysfs hardware info display. + */ +#if defined(__x86_64__) && defined(__BMI__) +#define POC_CTZ64_NAME "HW (TZCNT)" +#elif defined(__aarch64__) +#define POC_CTZ64_NAME "HW (RBIT+CLZ)" +#elif defined(__riscv) && defined(__riscv_zbb) +#define POC_CTZ64_NAME "HW (ctz)" +#elif defined(__x86_64__) +#define POC_CTZ64_NAME "HW (BSF)" +#else +#define POC_CTZ64_NAME "SW (De Bruijn)" +#endif + +/* + * POC_PTSELECT — Select position of the j-th set bit in a 64-bit word + * + * Based on the algorithm described in: + * P. Pandey, M. A. Bender, R. Johnson, + * "A Fast x86 Implementation of Select", arXiv:1706.00990, 2017. + * + * Returns the bit position (0-indexed) of the j-th set bit in v. + * Undefined behavior if j >= popcount(v). + * + * Tier 1 (x86-64 + BMI2, excluding AMD Zen 1/2 slow microcode PDEP): + * PDEP + TZCNT — 4 instructions total. + * PDEP deposits the j-th source bit at the j-th mask position. + * + * Tier 2 (fallback): Iterative bit-clear — O(j) iterations + * Clears the lowest set bit j times, then CTZ on remainder. + */ + +#if defined(__x86_64__) && defined(__BMI2__) && \ + !defined(__znver1) && !defined(__znver2) +static __always_inline int poc_ptselect(u64 v, int j) +{ + u64 deposited; + + asm("pdep %2, %1, %0" : "=r"(deposited) : "r"(1ULL << j), "rm"(v)); + return POC_CTZ64(deposited); +} +#define POC_PTSELECT(v, j) poc_ptselect(v, j) +#define POC_PTSELECT_NAME "HW (PDEP)" + +/* + * Tier 2 (fallback): Iterative bit-clear — O(j) iterations. + * Clears the lowest set bit j times, then returns its position via CTZ. + */ +#else +static __always_inline int poc_ptselect_sw(u64 v, int j) +{ + int k; + + for (k = 0; k < j; k++) + v &= v - 1; /* clear lowest set bit */ + return POC_CTZ64(v); +} +#define POC_PTSELECT(v, j) poc_ptselect_sw(v, j) +#define POC_PTSELECT_NAME "SW (loop)" + +#endif /* POC_PTSELECT */ + +/************************************************************** + * Flag array to bitmask conversion (lock-free mode): + */ + +/* + * POC_BYTE_EXTRACT / POC_BYTE_PACK - constants for multiply-and-shift trick. + * + * Isolates bit 0 of each byte in a u64 word, then packs the 8 bits + * into the most significant byte via multiply. + */ +#define POC_BYTE_EXTRACT 0x0101010101010101ULL +#define POC_BYTE_PACK 0x0102040810204080ULL + +/* + * POC_BMP8 - Convert one 8-byte slice of the flag array to 8 packed bits. + * + * Tier 1 (x86-64 + BMI2, excluding AMD Zen 1/2 slow microcode PEXT): + * PEXT extracts bit 0 of each byte directly into 8 contiguous bits. + * Single instruction replaces AND + MUL + SHR. + * + * Tier 2 (fallback): Multiply-and-shift trick. + * Isolates bit 0 of each byte (AND), packs via MUL, shifts to position. + */ +#if defined(__x86_64__) && defined(__BMI2__) && \ + !defined(__znver1) && !defined(__znver2) + +static __always_inline u64 poc_bmp8_pext(u64 word, int i) +{ + u64 extracted; + + asm("pext %2, %1, %0" : "=r"(extracted) : "r"(word), "r"(POC_BYTE_EXTRACT)); + return extracted << (i * 8); +} +#define POC_BMP8(w, i) poc_bmp8_pext((w)[i], i) + +#else + +#define POC_BMP8(w, i) \ + ((((w)[i] & POC_BYTE_EXTRACT) * POC_BYTE_PACK >> 56) << ((i) * 8)) + +#endif /* POC_BMP8 */ + +/* + * poc_flags_to_u64 - Convert u8[64] flag array to u64 bitmask + * @flags: pointer to 64-byte flag array (cacheline-aligned) + * + * Phase 1 (memcpy): snapshot the 64-byte cache line to the stack. + * This eliminates the window in which a concurrent MESI invalidation + * could cause a re-fetch mid-computation. All 64 bytes land in one + * or two cache line transfers; subsequent computation is purely local. + * + * Phase 2: pack the stack-local copy into a u64 bitmask via + * multiply-and-shift (or PEXT on BMI2 x86). Always processes all + * 8 chunks — the extra iterations for small LLCs are negligible + * on stack-local data and avoid the poc_chunks_bit* dispatch tree. + * + * Returns: u64 bitmask with bit N set iff flags[N] != 0 + */ +static __always_inline u64 poc_flags_to_u64(const u8 *flags) +{ + u64 w[8]; + + /* Phase 1: snapshot shared cache line to stack */ + memcpy(w, flags, 64); + + /* Phase 2: pack stack-local copy into bitmask */ + return POC_BMP8(w, 0) | POC_BMP8(w, 1) | POC_BMP8(w, 2) | POC_BMP8(w, 3) | + POC_BMP8(w, 4) | POC_BMP8(w, 5) | POC_BMP8(w, 6) | POC_BMP8(w, 7); +} + +/************************************************************** + * Idle mask accessors: + */ + +/* + * poc_idle_cpu_mask - Get idle CPU bitmask filtered by LLC and affinity + * @affinity: task's allowed CPU mask (poc-relative, from poc_cpumask_to_u64) + * @sd_share: per-LLC shared data + * + * Returns a snapshot of idle CPUs within this LLC, masked by + * llc_members (valid CPUs) and @affinity (task placement). + * + * bitmap mode (default): single atomic64_read (MOV on x86). + * flag array mode: stack-snapshot + multiply-and-shift aggregation. + */ +static __always_inline u64 poc_idle_cpu_mask(u64 affinity, + struct sched_domain_shared *sd_share) +{ + u64 cpus; + + if (static_branch_unlikely(&sched_poc_lockless_bitmap)) + cpus = poc_flags_to_u64(sd_share->poc_idle_cpus); + else + cpus = (u64)atomic64_read(&sd_share->poc_idle_cpus_mask); + + return cpus & sd_share->poc_llc_members & affinity; +} + +#ifdef CONFIG_SCHED_SMT +/* + * poc_idle_core_mask - Get idle core bitmask + * @cpu_mask: snapshot of idle CPUs (already masked by llc_members & affinity) + * @sd_share: per-LLC shared data + * + * Returns a bitmask with bits set at core representative positions + * (lowest-numbered sibling) for cores where ALL SMT siblings are idle. + * + * Three-tier derivation: + * + * Tier 1 (consecutive 2-way SMT): 3 register ops with compile-time + * constants — AND, SHR 1, AND 0x5555... No memory loads. + * + * Tier 2 (uniform stride-N 2-way SMT): 3 register ops with + * precomputed per-LLC shift and primary mask — AND, SHR N, AND. + * Two extra loads (poc_smt_shift, poc_primary_mask) from sd_share, + * but no write-path overhead. + * + * Tier 3 (exotic: >2-way SMT or non-uniform topology): reads the + * separately-maintained poc_idle_cores_mask atomic64_t. Write path + * maintains this bitmap on every idle transition. + */ +static __always_inline u64 poc_idle_core_mask(u64 cpu_mask, + struct sched_domain_shared *sd_share) +{ + /* Tier 1: consecutive — constants only, zero loads */ + if (static_branch_likely(&sched_poc_smt_consecutive)) + return cpu_mask & (cpu_mask >> 1) & 0x5555555555555555ULL; + + /* Tier 2: uniform stride-N — precomputed shift + mask */ + if (static_branch_likely(&sched_poc_smt_uniform)) + return cpu_mask & (cpu_mask >> sd_share->poc_smt_shift) + & sd_share->poc_primary_mask; + + /* Tier 3: exotic — bitmap or flag array based on mode */ + if (static_branch_unlikely(&sched_poc_lockless_bitmap)) + return poc_flags_to_u64(sd_share->poc_idle_cores) & cpu_mask; + + return (u64)atomic64_read(&sd_share->poc_idle_cores_mask) & cpu_mask; +} +#endif /* CONFIG_SCHED_SMT */ + +/* + * __set_cpu_idle_state_poc - Update idle state in atomic64_t bitmap + * @cpu: CPU number + * @state: 0=busy, 1=idle + * + * Updates the atomic64_t cpus bitmap via atomic64_or/andnot (LOCK'd on x86). + * + * On uniform 2-way SMT (Tier 1 & 2: consecutive or stride-N), only + * the cpus state is updated; core idle state is derived at read time + * via bit-parallel operations. + * + * On exotic SMT (Tier 3: >2-way or non-uniform), also maintains the + * separate cores state (bitmap or flag array) for O(1) read-time lookup. + * + * Only one representation is maintained at a time (single-write), + * selected by sched_poc_lockless_bitmap. + * + * Caller (inline wrapper in sched.h) ensures poc_selector_active is on + * and sched_asym_cpucap_active() is false before calling here. + */ +void __set_cpu_idle_state_poc(int cpu, int state) +{ + struct rq *rq = cpu_rq(cpu); + if (!static_branch_unlikely(&sched_poc_lockless_bitmap) && + !state && READ_ONCE(rq->poc_idle_committed)) + return; + + guard(rcu)(); + struct sched_domain_shared *sd_share = + rcu_dereference(per_cpu(sd_llc_shared, cpu)); + if (!sd_share || !sd_share->poc_fast_eligible) + return; + + int bit = cpu - sd_share->poc_cpu_base; + u64 bit_mask = 1ULL << bit; + + if (static_branch_unlikely(&sched_poc_lockless_bitmap)) { + WRITE_ONCE(sd_share->poc_idle_cpus[bit], state > 0 ? 1 : 0); + } else if (state > 0) { + /* Entering idle: clear any stale committed flag */ + WRITE_ONCE(rq->poc_idle_committed, 0); + atomic64_or(bit_mask, &sd_share->poc_idle_cpus_mask); + } else { + /* + * Exiting idle: if a waker already committed (cleared the + * bitmap bit), skip the redundant atomic on the shared + * cacheline. The flag lives in rq's first cacheline — + * same line the waker already dirtied via ttwu_pending. + */ + atomic64_andnot(bit_mask, &sd_share->poc_idle_cpus_mask); + WRITE_ONCE(rq->poc_idle_committed, 1); + } + +#ifdef CONFIG_SCHED_SMT + if (sched_smt_active()) { + /* Tier 1 & 2: read-time derivation, no write-path cost */ + if (static_branch_likely(&sched_poc_smt_uniform)) + return; + /* + * Tier 3 (exotic SMT): maintain separate cores state. + * Check whether all SMT siblings are idle. + */ + u64 smt = sd_share->poc_smt_mask[bit]; + u64 core_bitmask = smt & (-smt); /* core representative */ + int core_bit = __builtin_ctzll(core_bitmask); + bool core_idle; + + if (static_branch_unlikely(&sched_poc_lockless_bitmap)) { + /* + * Flag array mode: check siblings via WRITE_ONCE-stored + * flags. smp_wmb() ensures our store to poc_idle_cpus[] + * is visible before we read sibling flags. + * On x86 TSO: compiler barrier only (~0 cyc). + * On ARM64: dmb ishst. + */ + smp_wmb(); + u64 tmp = smt; + + core_idle = state > 0; + while (core_idle && tmp) { + int s = __builtin_ctzll(tmp); + + if (!READ_ONCE(sd_share->poc_idle_cpus[s])) + core_idle = false; + tmp &= tmp - 1; + } + WRITE_ONCE(sd_share->poc_idle_cores[core_bit], + core_idle ? 1 : 0); + } else { + /* + * smp_mb__after_atomic() ensures our atomic store is + * visible before we read sibling bits. On x86 TSO this + * is a compiler barrier (~0 cyc); on ARM64: dmb ish. + */ + smp_mb__after_atomic(); + u64 cpus = (u64)atomic64_read(&sd_share->poc_idle_cpus_mask); + core_idle = (cpus & smt) == smt; + u64 cores = (u64)atomic64_read(&sd_share->poc_idle_cores_mask); + + if (core_idle) { + if (!(cores & core_bitmask)) + atomic64_or(core_bitmask, + &sd_share->poc_idle_cores_mask); + } else { + if (cores & core_bitmask) + atomic64_andnot(core_bitmask, + &sd_share->poc_idle_cores_mask); + } + } + } +#endif /* CONFIG_SCHED_SMT */ +} + +/************************************************************** + * Idle CPU selection helpers: + */ + +/* Test whether a single CPU is idle in a POC bitmap snapshot. + * Assumes cpu_mask is in scope — works in any function with that variable. */ +#define POC_IDLE_CPU(bit) (cpu_mask & (1ULL << (bit))) +/* Scope-free validity checks — usable in any function. */ +#define POC_CPU_VALID(cpu) ((cpu) >= 0) +#define POC_CPU_IN_LLC(bit) ((unsigned int)(bit) < 64) + +/* + * poc_select_rr_improved - Improved round-robin idle CPU selection + * @base: poc_cpu_base (smallest CPU ID in this LLC) + * @mask: idle bitmask (snapshot, caller guarantees non-zero) + * @counter: per-CPU round-robin counter value + * + * Improved RR with two techniques: + * 1. Case-split by total: + * total=1: direct ctz + * total=2: interleave by counter LSB (guarantees non-repeat), + * single CTZ via cmov-selected source mask + * total>=3: golden-ratio scramble + Lemire fastrange + * 2. Golden-ratio scrambling (counter * 0x9E3779B9) mapped via + * Lemire fastrange for pseudo-random uniform distribution. + * + * eager_commit (unconditional) already prevents burst wake-ups from + * re-selecting the same CPU by clearing the bitmap bit at selection + * time, so no previous-pick exclusion state is needed here. + * + * Returns: selected CPU number. + */ +static __always_inline int poc_select_rr_improved( + int base, u64 mask, unsigned int counter) +{ + int total = hweight64(mask); + + if (total <= 2) { + /* + * Pick the lower or upper set bit via counter LSB if total == 2. + * Select the mask first (cmov), then one CTZ — halves the + * cost on archs where CTZ64 is a SW fallback (De Bruijn). + */ + if ((total == 2) && (counter & 1)) + mask &= mask - 1; + + return base + POC_CTZ64(mask); + } + + /* total >= 3: golden-ratio scramble + Lemire fastrange */ + { + u32 scrambled = POC_SCRAMBLE(counter); + int pick = POC_FASTRANGE(scrambled, total); + + return base + POC_PTSELECT(mask, pick); + } +} + +/* + * poc_select_rr - Round-robin idle CPU selection from a single-word mask + * @base: poc_cpu_base (smallest CPU ID in this LLC) + * @mask: idle bitmask (snapshot) + * @counter: per-CPU round-robin counter value + * + * Division-free perfect round-robin via FASTRANGE16 + PTSELECT. + * Consecutive calls on the same CPU never repeat an idle CPU + * until all candidates have been visited. + * Caller must ensure at least one bit is set in mask. + * Returns: selected CPU number. + */ +static __always_inline int poc_select_rr(int base, u64 mask, unsigned int counter) +{ + if (static_branch_likely(&sched_poc_rr_improved)) + return poc_select_rr_improved(base, mask, counter); + + /* Current strategy: poc_rr_step[] table (perfect RR), unchanged */ + { + int total = hweight64(mask); + u16 phase = (u16)(counter * (u32)poc_rr_step[total - 1]); + int pick = POC_FIXED_MOD16(phase, total); + + return POC_PTSELECT(mask, pick) + base; + } +} + +/* + * poc_cluster_search - Search for an idle CPU within the target's L2 cluster + * @base: poc_cpu_base (smallest CPU ID in this LLC) + * @tgt_bit: target CPU's POC-relative bit position + * @sd_share: per-LLC shared data containing cluster geometry + * @mask: snapshot of idle bitmask (cores or cpus, caller decides) + * + * Uses pre-computed cluster mask for O(1) lookup via CTZ. + * Returns: idle CPU number if found within cluster, -1 otherwise. + */ +static __always_inline int poc_cluster_search(int base, int tgt_bit, + struct sched_domain_shared *sd_share, u64 mask) +{ + u64 cls_idle = mask & sd_share->poc_cluster_mask[tgt_bit]; + + if (!cls_idle) + return -1; + + if (static_branch_likely(&sched_poc_rr_improved)) { + /* Improved path: inc counter here so LV3 fallback sees fresh value */ + unsigned int counter = __this_cpu_inc_return(poc_rr_counter); + return poc_select_rr_improved(base, cls_idle, counter); + } + + /* Current strategy: ctz lowest-bit (no RR), unchanged */ + return base + POC_CTZ64(cls_idle); +} + +#ifdef CONFIG_SCHED_SMT +/* + * poc_smt_sibling_mask - Get SMT sibling bitmask for a given CPU + * @bit: POC-relative bit position + * @sd_share: per-LLC shared data + * + * Three-tier computation matching poc_idle_core_mask(): + * + * Tier 1 (consecutive): 3ULL << (bit & ~1) — shift only, zero loads. + * + * Tier 2 (uniform stride-N): determine sibling via poc_smt_shift + * and poc_primary_mask. Avoids poc_smt_mask[] array lookup. + * + * Tier 3 (exotic): loads from pre-computed poc_smt_mask[] table. + */ +static __always_inline u64 poc_smt_sibling_mask(int bit, + struct sched_domain_shared *sd_share) +{ + if (static_branch_likely(&sched_poc_smt_consecutive)) + return 3ULL << (bit & ~1); + + if (static_branch_likely(&sched_poc_smt_uniform)) { + u8 shift = sd_share->poc_smt_shift; + int sib = (sd_share->poc_primary_mask & (1ULL << bit)) + ? bit + shift : bit - shift; + if (unlikely(sib < 0 || sib >= 64)) + return 1ULL << bit; + return (1ULL << bit) | (1ULL << sib); + } + + return sd_share->poc_smt_mask[bit]; +} + +/* + * poc_find_idle_smt_sibling - Find an idle CPU among target and its SMT siblings + * @base: poc_cpu_base (smallest CPU ID in this LLC) + * @tgt_bit: target CPU's POC-relative bit position + * @cpu_mask: snapshot of idle CPU bitmask + * @smt_mask: pre-computed SMT sibling mask for target (includes self) + * + * Searches target itself and its SMT siblings for an idle CPU. + * Target is checked first for cache locality. + * Returns: idle CPU number if found, -1 otherwise + */ +static __always_inline int poc_find_idle_smt_sibling( + int base, int tgt_bit, u64 cpu_mask, u64 smt_mask) +{ + /* Check target first for cache locality */ + if (POC_IDLE_CPU(tgt_bit)) + return base + tgt_bit; + + u64 idle_sibs = cpu_mask & smt_mask; + + if (idle_sibs) + return base + POC_CTZ64(idle_sibs); + + return -1; +} +/* + * poc_try_idle_smt - Find an idle CPU among a CPU and its SMT siblings + * @base: poc_cpu_base (smallest CPU ID in this LLC) + * @cpu: the CPU to check (and its SMT siblings) + * @cpu_mask: snapshot of idle CPU bitmask + * @sd_share: per-LLC shared data + * + * Checks if the given CPU or any of its SMT siblings is idle. + * Caller is responsible for poc_count() and poc_commit_selection(). + * Returns: idle CPU number if found, -1 otherwise + */ +static __always_inline int poc_try_idle_smt(int base, int cpu, + u64 cpu_mask, struct sched_domain_shared *sd_share) +{ + int bit = cpu - base; + + if (sd_share->poc_llc_members & (1ULL << bit)) { + int smt_cpu = poc_find_idle_smt_sibling(base, bit, + cpu_mask, poc_smt_sibling_mask(bit, sd_share)); + if (POC_CPU_VALID(smt_cpu)) + return smt_cpu; + } + return -1; +} + +#endif /* CONFIG_SCHED_SMT */ + +/* + * poc_commit_selection - Atomically clear selected CPU from idle bitmap + * @cpu: the CPU number selected by POC + * @sd_share: per-LLC shared data + * + * Clears the selected CPU's bit in poc_idle_cpus_mask at selection + * time to close the race window where multiple waker CPUs read the + * same stale bitmap and select the same idle CPU. The do_idle() + * exit path performs an idempotent clear as a safety net for + * non-POC wakeups; poc_idle_committed gates that path so the atomic + * fires at most once per selection. + */ +static __always_inline void poc_commit_selection(int cpu, + struct sched_domain_shared *sd_share) +{ + if (cpu_rq(cpu)->nr_running <= 2) { + int bit = cpu - sd_share->poc_cpu_base; + + if (static_branch_unlikely(&sched_poc_lockless_bitmap)) { + WRITE_ONCE(sd_share->poc_idle_cpus[bit], 0); + smp_wmb(); + } else { + atomic64_andnot(1ULL << bit, &sd_share->poc_idle_cpus_mask); + smp_mb__after_atomic(); + /* Mark committed so target skips redundant andnot on wakeup */ + WRITE_ONCE(cpu_rq(cpu)->poc_idle_committed, 1); + } + } +} + +/* + * POC_IDLE_CORE - Test whether a CPU's core is fully idle. + * POC_IDLE_SMT - Find an idle CPU among @cpu and its SMT siblings. + * + * POC_RETURN - Record hit counter, clear bitmap, return selected CPU. + * POC_RETURN_IF - Same, but only if @cpu >= 0 (used after POC_IDLE_SMT). + * + * These assume core_mask, base, sd_share are in scope + * (only used inside select_idle_cpu_poc). + */ +#define POC_IDLE_CORE(bit) (core_mask & poc_smt_sibling_mask((bit), sd_share)) +#define POC_IDLE_SMT(cpu) poc_try_idle_smt(base, (cpu), cpu_mask, sd_share) + +#define POC_RETURN(cpu, level) do { \ + poc_count(level); \ + poc_commit_selection(cpu, sd_share); \ + return cpu; \ +} while (0) + +#define POC_RETURN_IF(cpu, level) do { \ + if ((cpu) >= 0) \ + POC_RETURN(cpu, level); \ +} while (0) + +/************************************************************** + * Fast path dispatcher: + */ + +/* + * select_idle_cpu_poc - Fast idle CPU selector (atomic64 bitmap path) + * @target: CPU chosen by wake_affine (Level 1 preferred CPU; + * search origin for L2/L3/L5/L6) + * @prev: task's previous CPU (Level 4 cache locality preference) + * @recent: task's recent_used_cpu (-1 if none; pre-filtered by caller) + * @sync: 1 if synchronous wakeup (Level 4s: waker yields CPU) + * @sd_share: per-LLC shared data (caller provides; never NULL) + * @allowed: task's cpumask (p->cpus_ptr) for affinity filtering + * + * Two operating modes (sysctl kernel.sched_poc_smt_fallback): + * + * smt_fallback=0 (default): POC handles all idle CPU + * selection itself, including SMT siblings. Prioritizes + * prev's SMT sibling for cache locality. Uses CFS's + * nr_idle_scan (SIS_UTIL) to gate Level 5/6 under overload. + * + * smt_fallback=1: Bails out to CFS when has_idle_cores is + * false. CFS handles SMT sibling selection via + * select_idle_smt(prev) and nr_idle_scan-limited + * select_idle_cpu(). + * + * Selection levels: + * + * Level 0: Saturation check -- no idle CPUs → return -1 + * (smt_fallback: also when has_idle_cores == false) + * Level 1r: Recent's core is fully idle → return recent (!early_select) + * Level 1s: Target CPU idle in bitmap → return target (L1/TLB affinity) + * Level 1t: Target CPU's core is fully idle → return target + * Level 1p: Prev's core is fully idle → return prev (prev != target) + * --- core_mask != 0: search idle-core bitmap --- + * Level 2: Idle core in L2 cluster (CTZ) + * Level 3: Idle core across LLC (RR PTSELECT) + * --- core_mask == 0: search idle-CPU bitmap --- + * Level 4s: sync + target CPU idle (waker frees core) + * Level 4p: Prev's SMT sibling (cache locality) + * Level 4t: Target's SMT sibling + * Level 4r: Recent's SMT sibling (warm cache, always) + * [SIS_UTIL gate: nr_idle_scan == 0 → return -2] + * Level 5: Idle CPU in L2 cluster (CTZ) + * Level 6: Idle CPU across LLC (RR PTSELECT) + * + * Non-SMT: Level 1r → 1t → 1p → Level 2 → Level 3 (core = CPU). + * + * Returns: idle CPU number if found, -1 if not found (CFS may retry), + * -2 if SIS_UTIL overload (caller should skip CFS) + */ +static __always_inline int select_idle_cpu_poc(int target, int prev, + int recent, int sync, + struct sched_domain_shared *sd_share, + const struct cpumask *allowed) +{ + int base = sd_share->poc_cpu_base; + int rct_bit = recent - base; + int tgt_bit = target - base; + int prv_bit = prev - base; +#ifdef CONFIG_SCHED_SMT + u64 core_mask __maybe_unused; +#endif + u64 affinity; + u64 cpu_mask; + int level_offset = 0; + +#ifdef CONFIG_SCHED_SMT + /* SMT fallback: bail to CFS for SMT sibling selection */ + if (sched_smt_active() && + static_branch_unlikely(&sched_poc_smt_fallback) && + !READ_ONCE(sd_share->has_idle_cores)) + return -1; +#endif + + if (static_branch_unlikely(&sched_poc_lockless_bitmap)) + prefetch(sd_share->poc_idle_cpus); + else + prefetch(&sd_share->poc_idle_cpus_mask); +#ifdef CONFIG_SCHED_SMT + if (sched_smt_active()) { + if (!static_branch_likely(&sched_poc_smt_uniform)) { + if (static_branch_unlikely(&sched_poc_lockless_bitmap)) + prefetch(sd_share->poc_idle_cores); + else + prefetch(&sd_share->poc_idle_cores_mask); + if (POC_CPU_VALID(recent)) + prefetch(&sd_share->poc_smt_mask[rct_bit]); + prefetch(&sd_share->poc_smt_mask[tgt_bit]); + prefetch(&sd_share->poc_smt_mask[prv_bit]); + } + } +#endif + if (static_branch_likely(&sched_cluster_active)) + prefetch(&sd_share->poc_cluster_mask[tgt_bit]); + + affinity = poc_cpumask_to_u64(allowed, sd_share); + cpu_mask = poc_idle_cpu_mask(affinity, sd_share); + + /* Level 0: Saturation — no idle CPU */ + if (!cpu_mask) + return -1; + +#ifdef CONFIG_SCHED_SMT + if (sched_smt_active()) { + core_mask = poc_idle_core_mask(cpu_mask, sd_share); + + /* Level 1r: recent's core is idle (warm cache) */ + if (!static_branch_likely(&sched_poc_early_select) && + core_mask && POC_CPU_IN_LLC(rct_bit) && POC_IDLE_CORE(rct_bit)) + POC_RETURN(recent, POC_LV1R); + + /* Level 1s: target CPU sticky — L1/TLB affinity shortcut */ + if (static_branch_unlikely(&sched_poc_target_sticky) && POC_IDLE_CPU(tgt_bit)) + POC_RETURN(target, POC_LV1S); + + if (core_mask) { + /* + * Idle core path: T → P order. + * Target first — wake_affine chose it for data sharing + * and the full core is free. + */ + + /* Level 1t: target CPU's core is idle → return it */ + if (!static_branch_likely(&sched_poc_early_select) && + POC_IDLE_CORE(tgt_bit)) + POC_RETURN(target, POC_LV1T); + + /* Level 1p: prev's core is idle (task's L1/L2 warm) */ + if (prev != target && POC_CPU_IN_LLC(prv_bit) && POC_IDLE_CORE(prv_bit)) + POC_RETURN(prev, POC_LV1P); + + cpu_mask = core_mask; + } else { + int cpu; + + /* Level 4s: sync wakeup + target CPU idle → + * waker will sleep imminently, freeing the core */ + if (sync && POC_IDLE_CPU(tgt_bit)) + POC_RETURN(target, POC_LV4S); + + /* + * No-idle-core path: P → T → R order. + * Target itself was already tried at Level 1s/4s; + * prioritize task's own cache (prev, recent) over + * waker locality (target's sibling). + */ + + /* Level 4p: prev's SMT sibling (cache locality) */ + if (prev != target && POC_CPU_IN_LLC(prv_bit)) { + cpu = POC_IDLE_SMT(prev); + POC_RETURN_IF(cpu, POC_LV4P); + } + + /* Level 4t: target's SMT sibling */ + cpu = POC_IDLE_SMT(target); + POC_RETURN_IF(cpu, POC_LV4T); + + /* Level 4r: recent's SMT sibling (warm cache) */ + if (POC_CPU_IN_LLC(rct_bit)) { + cpu = POC_IDLE_SMT(recent); + POC_RETURN_IF(cpu, POC_LV4R); + } + + /* SIS_UTIL overload gate for Level 5/6 */ + if (!static_branch_likely(&sched_poc_greedy_search) && + sched_feat(SIS_UTIL) && !READ_ONCE(sd_share->nr_idle_scan)) + return -2; + + level_offset = POC_SMT_LEVEL_OFFSET; + } + } + else +#endif + { + /* Level 1r: recent CPU is idle (non-SMT) */ + if (!static_branch_likely(&sched_poc_early_select) && + POC_CPU_IN_LLC(rct_bit) && POC_IDLE_CPU(rct_bit)) + POC_RETURN(recent, POC_LV1R); + /* Level 1t: target CPU is idle → return (non-SMT) */ + if (POC_IDLE_CPU(tgt_bit)) + POC_RETURN(target, POC_LV1T); + /* Level 1p: prev CPU is idle (non-SMT) */ + if (prev != target && POC_CPU_IN_LLC(prv_bit) && POC_IDLE_CPU(prv_bit)) + POC_RETURN(prev, POC_LV1P); + } + + if (static_branch_likely(&sched_poc_packed)) { + /* + * Level 2+3 / 5+6: packed priority search (≤32 CPUs/LLC) + * + * Packs cluster candidates (high priority) into lower 32 bits + * and all LLC candidates (low priority) into upper 32 bits. + * A single TZCNT resolves the highest-priority idle CPU. + * Level discrimination: (raw >> 5) yields 0 (cluster) or 1 (LLC). + * + * rr_improved=ON: rotation amount via golden-ratio scramble. + * rr_improved=OFF: rotation amount is (counter & 31). + */ + unsigned int counter = __this_cpu_inc_return(poc_rr_counter); + int rot; + u32 cls = 0; + u32 all; + u64 packed; + int raw, bit; + + if (static_branch_likely(&sched_poc_rr_improved)) + rot = (int)(POC_SCRAMBLE(counter) >> 27); + else + rot = counter & 31; + + if (static_branch_likely(&sched_cluster_active) && + sd_share->poc_cluster_valid) + cls = ror32((u32)(cpu_mask & + sd_share->poc_cluster_mask[tgt_bit]), rot); + + all = ror32((u32)cpu_mask, rot); + packed = (u64)cls | ((u64)all << 32); + + raw = POC_CTZ64(packed); + bit = ((raw & 31) + rot) & 31; + + POC_RETURN(base + bit, POC_LV2 + (raw >> 5) + level_offset); + } else { + /* Level 2/5: idle core/cpu in target's L2 cluster */ + if (static_branch_likely(&sched_cluster_active) + && sd_share->poc_cluster_valid) { + int cpu = poc_cluster_search( + base, tgt_bit, sd_share, cpu_mask); + if (POC_CPU_VALID(cpu)) + POC_RETURN(cpu, POC_LV2 + level_offset); + } + + /* Level 3/6: idle core/cpu across LLC via RR */ + { + unsigned int counter = __this_cpu_inc_return(poc_rr_counter); + int rr_cpu = poc_select_rr(base, cpu_mask, counter); + POC_RETURN(rr_cpu, POC_LV3 + level_offset); + } + } +} + +/************************************************************** + * Topology setup: + * + * poc_sd_shared_init - Initialize POC fields in sched_domain_shared + * @sd: the LLC-sharing sched_domain whose ->shared was just attached + * @sd_id: first CPU of @sd's span (used as poc_cpu_base) + * + * Called from build_sched_domains() right after sd->shared is attached + * for an SD_SHARE_LLC domain. Computes per-LLC bit-base and pre-builds + * member/SMT/cluster masks for O(1) lookup at wakeup time. + */ +void poc_sd_shared_init(struct sched_domain *sd, int sd_id) +{ + struct cpumask *sd_span = sched_domain_span(sd); + int range = cpumask_last(sd_span) - sd_id + 1; + + sd->shared->poc_cpu_base = sd_id; + sd->shared->poc_affinity_shift = sd_id & 63; + + if (range <= 64) { + sd->shared->poc_fast_eligible = true; + /* + * Disable aligned optimization if this LLC's base CPU + * is not 64-aligned (e.g., Threadripper CCDs). + */ + if (sd_id & 63) + static_branch_disable_cpuslocked(&sched_poc_aligned); + /* + * Disable packed priority search if this LLC + * has more than 32 CPUs. + */ + if (range > 32) + static_branch_disable_cpuslocked(&sched_poc_packed); + } else { + sd->shared->poc_fast_eligible = false; + static_branch_disable_cpuslocked(&sched_poc_packed); + } + memset(sd->shared->poc_idle_cpus, 0, + sizeof(sd->shared->poc_idle_cpus)); + atomic64_set(&sd->shared->poc_idle_cpus_mask, 0); +#ifdef CONFIG_SCHED_SMT + memset(sd->shared->poc_idle_cores, 0, + sizeof(sd->shared->poc_idle_cores)); + atomic64_set(&sd->shared->poc_idle_cores_mask, 0); +#endif + + /* Build LLC member bitmask for reader-side aggregation */ + { + u64 members = 0; + int cpu_iter; + + for_each_cpu(cpu_iter, sd_span) { + int bit = cpu_iter - sd_id; + + if ((unsigned int)bit < 64) + members |= 1ULL << bit; + } + sd->shared->poc_llc_members = members; + } + +#ifdef CONFIG_SCHED_SMT + /* + * Pre-compute SMT sibling masks for Level 4. + * Each entry contains a bitmask of SMT siblings (including self) + * for O(1) lookup via CTZ during wakeup. + */ + memset(sd->shared->poc_smt_mask, 0, + sizeof(sd->shared->poc_smt_mask)); + if (sd->shared->poc_fast_eligible) { + int cpu_iter; + + for_each_cpu(cpu_iter, sd_span) { + int bit = cpu_iter - sd_id; + int sibling; + u64 mask = 0; + + for_each_cpu(sibling, cpu_smt_mask(cpu_iter)) { + int sib_bit; + + sib_bit = sibling - sd_id; + if (sib_bit >= 0 && sib_bit < 64) + mask |= 1ULL << sib_bit; + } + if (bit >= 0 && bit < 64) + sd->shared->poc_smt_mask[bit] = mask; + } + } + + /* + * Detect SMT topology and classify for poc_idle_core_mask(): + * + * Tier 1 (consecutive): uniform 2-way SMT, siblings at + * consecutive bit positions (e.g., 0,1 / 2,3). + * Uses compile-time constants: shift=1, mask=0x5555... + * + * Tier 2 (uniform stride-N): uniform 2-way SMT with + * constant stride between siblings (e.g., Intel Xeon + * stride-8: CPU 0,8 / 1,9 / ...). Uses precomputed + * poc_smt_shift and poc_primary_mask for read-time + * derivation without write-path overhead. + * + * Tier 3 (exotic): >2-way SMT, non-uniform topology, + * or mixed SMT ways. Falls back to write-time + * maintenance of poc_idle_cores_mask atomic64_t. + * + * On pure non-SMT systems, the key values are irrelevant + * because sched_smt_active() gates all SMT paths. + */ + sd->shared->poc_smt_shift = 1; + sd->shared->poc_primary_mask = 0; + + if (sd->shared->poc_fast_eligible) { + int cpu_iter; + bool all_2way = true; + bool all_consecutive = true; + int uniform_stride = -1; + u64 primary_mask = 0; + + for_each_cpu(cpu_iter, sd_span) { + int bit = cpu_iter - sd_id; + + if (bit < 0 || bit >= 64) + continue; + u64 mask = sd->shared->poc_smt_mask[bit]; + int ways = hweight64(mask); + + if (ways != 2) { + all_2way = false; + all_consecutive = false; + break; + } + + int lo = __ffs(mask); + int hi = __fls(mask); + int stride = hi - lo; + + /* Track primary (lowest-numbered sibling) */ + primary_mask |= 1ULL << lo; + + /* Check consecutive: 0b11 at even position */ + if ((lo & 1) || mask != (3ULL << lo)) + all_consecutive = false; + + /* Check uniform stride */ + if (uniform_stride < 0) + uniform_stride = stride; + else if (stride != uniform_stride) + all_2way = false; + } + + if (!all_consecutive) + static_branch_disable_cpuslocked( + &sched_poc_smt_consecutive); + + if (all_2way && uniform_stride > 0) { + sd->shared->poc_smt_shift = (u8)uniform_stride; + sd->shared->poc_primary_mask = primary_mask; + } else { + static_branch_disable_cpuslocked( + &sched_poc_smt_consecutive); + static_branch_disable_cpuslocked( + &sched_poc_smt_uniform); + } + } +#endif /* CONFIG_SCHED_SMT */ + + memset(sd->shared->poc_cluster_mask, 0, + sizeof(sd->shared->poc_cluster_mask)); + + sd->shared->poc_cluster_valid = false; + +#ifdef CONFIG_SCHED_CLUSTER + /* + * Detect cluster (L2-sharing) topology for Level 2/5 + * cluster-local search in POC selector. + * + * Uses cpu_clustergroup_mask() which returns the L2 + * cache sharing mask on x86. Validates that all + * clusters are uniform (same size, power-of-2, and + * naturally aligned in POC bit space). + */ + if (sd->shared->poc_fast_eligible) { + const struct cpumask *cls_mask = cpu_clustergroup_mask(sd_id); + int cls_size = cpumask_weight(cls_mask); + int smt_size = cpumask_weight(cpu_smt_mask(sd_id)); + + if (cls_size > smt_size && is_power_of_2(cls_size)) { + bool valid = true; + int cpu_iter; + + for_each_cpu(cpu_iter, sd_span) { + const struct cpumask *m = + cpu_clustergroup_mask(cpu_iter); + int first = cpumask_first(m); + int rel = first - sd_id; + + if (cpumask_weight(m) != cls_size || + (rel & (cls_size - 1)) != 0) { + valid = false; + break; + } + } + if (valid) { + sd->shared->poc_cluster_valid = true; + + /* + * Pre-compute cluster masks for O(1) lookup. + * Each entry contains a bitmask of cluster + * members (excluding self) for fast search. + */ + for_each_cpu(cpu_iter, sd_span) { + const struct cpumask *m = + cpu_clustergroup_mask(cpu_iter); + int bit = cpu_iter - sd_id; + int member; + u64 cmask = 0; + + for_each_cpu(member, m) { + int mbit; + + if (member == cpu_iter) + continue; + mbit = member - sd_id; + if (mbit >= 0 && mbit < 64) + cmask |= 1ULL << mbit; + } + if (bit >= 0 && bit < 64) + sd->shared->poc_cluster_mask[bit] = cmask; + } + } + } + } +#endif /* CONFIG_SCHED_CLUSTER */ +} + +/************************************************************** + * Sysctl interface and initialization: + */ + +#if defined(CONFIG_SYSCTL) || defined(CONFIG_SCHED_CLASS_EXT) +/* + * poc_resync_idle_state - Resync POC idle bitmaps after re-enable + * + * When POC is re-enabled after a period of being disabled, + * the idle bitmaps may be stale. Walk all online CPUs and push + * the current idle state into poc_idle_cpus_mask (and poc_idle_cores_mask + * on non-consecutive SMT). + * + * Must be called AFTER static_branch_enable() so that concurrent + * idle transitions are also updating the flags. + * Caller must hold cpus_read_lock(). + */ +static void poc_resync_idle_state(void) +{ + int cpu; + + for_each_online_cpu(cpu) { + WRITE_ONCE(cpu_rq(cpu)->poc_idle_committed, 0); + __set_cpu_idle_state_poc(cpu, idle_cpu(cpu)); + } +} + +/* + * poc_reevaluate_active - Recompute poc_selector_active from inputs + * + * poc_selector_active = sched_poc_selector && !poc_selector_skip + * + * On transition to active: enable static key, then resync idle bitmaps. + * On transition to inactive: disable static key. + * Caller must hold cpus_read_lock(). + */ +static void poc_reevaluate_active(void) +{ + bool want = sched_poc_selector && !poc_selector_skip; + bool now = static_branch_likely(&poc_selector_active); + + if (want == now) + return; + + if (want) { + static_branch_enable_cpuslocked(&poc_selector_active); + poc_resync_idle_state(); + } else { + static_branch_disable_cpuslocked(&poc_selector_active); + } +} +#endif /* CONFIG_SYSCTL || CONFIG_SCHED_CLASS_EXT */ + +#ifdef CONFIG_SCHED_CLASS_EXT +/* + * poc_notify_scx - Called by sched_ext on enable/disable transitions + * @scx_active: true when scx scheduler is being enabled + */ +void poc_notify_scx(bool scx_active) +{ + cpus_read_lock(); + poc_selector_skip = scx_active; + poc_reevaluate_active(); + cpus_read_unlock(); +} + +/* + * poc_skip_fallback_work - Workqueue item to re-enable POC after scx fallback. + * + * Scheduled by poc_check_skip_fallback() when an scx scheduler calls + * select_idle_sibling. Runs poc_reevaluate_active() outside the hot path + * to avoid updating the static key and resyncing bitmaps inline. + */ +static void poc_skip_fallback_fn(struct work_struct *work); +static DECLARE_WORK(poc_skip_fallback_work, poc_skip_fallback_fn); + +static void poc_skip_fallback_fn(struct work_struct *work) +{ + cpus_read_lock(); + poc_reevaluate_active(); + cpus_read_unlock(); +} + +/* + * poc_check_skip_fallback - Hot-path detection for scx calling select_idle_sibling + * + * While scx is active, poc_selector_skip=true suppresses idle bitmap updates + * in do_idle. Some scx schedulers still call select_idle_sibling; when that + * happens, flip poc_selector_skip back to false and schedule a workqueue item + * to re-enable poc_selector_active and resync stale bitmaps. + * + * WRITE_ONCE(false) is idempotent across concurrent callers; schedule_work() + * silently drops duplicate requests when the item is already queued. + */ +void poc_check_skip_fallback(void) +{ + if (!sched_poc_selector || !READ_ONCE(poc_selector_skip)) + return; + WRITE_ONCE(poc_selector_skip, false); + schedule_work(&poc_skip_fallback_work); +} +#endif + +#ifdef CONFIG_SYSCTL +static int sched_poc_sysctl_handler(const struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + unsigned int val = sched_poc_selector ? 1 : 0; + struct ctl_table tmp = { + .data = &val, + .maxlen = sizeof(val), + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }; + int ret = proc_douintvec_minmax(&tmp, write, buffer, lenp, ppos); + + if (!ret && write) { + cpus_read_lock(); + sched_poc_selector = !!val; + poc_reevaluate_active(); + cpus_read_unlock(); + } + return ret; +} + +static int sched_poc_smt_fallback_sysctl_handler(const struct ctl_table *table, + int write, void *buffer, + size_t *lenp, loff_t *ppos) +{ + unsigned int val = static_branch_unlikely(&sched_poc_smt_fallback) ? 1 : 0; + struct ctl_table tmp = { + .data = &val, + .maxlen = sizeof(val), + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }; + int ret = proc_douintvec_minmax(&tmp, write, buffer, lenp, ppos); + + if (!ret && write) { + if (val) + static_branch_enable(&sched_poc_smt_fallback); + else + static_branch_disable(&sched_poc_smt_fallback); + } + return ret; +} + +static int sched_poc_rr_improved_sysctl_handler(const struct ctl_table *table, + int write, void *buffer, + size_t *lenp, loff_t *ppos) +{ + unsigned int val = static_branch_likely(&sched_poc_rr_improved) ? 1 : 0; + struct ctl_table tmp = { + .data = &val, + .maxlen = sizeof(val), + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }; + int ret = proc_douintvec_minmax(&tmp, write, buffer, lenp, ppos); + + if (!ret && write) { + if (val) + static_branch_enable(&sched_poc_rr_improved); + else + static_branch_disable(&sched_poc_rr_improved); + } + return ret; +} + +static int sched_poc_target_sticky_sysctl_handler(const struct ctl_table *table, + int write, void *buffer, + size_t *lenp, loff_t *ppos) +{ + unsigned int val = static_branch_unlikely(&sched_poc_target_sticky) ? 1 : 0; + struct ctl_table tmp = { + .data = &val, + .maxlen = sizeof(val), + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }; + int ret = proc_douintvec_minmax(&tmp, write, buffer, lenp, ppos); + + if (!ret && write) { + if (val) + static_branch_enable(&sched_poc_target_sticky); + else + static_branch_disable(&sched_poc_target_sticky); + } + return ret; +} + +static int sched_poc_early_select_handler(const struct ctl_table *table, + int write, void *buffer, + size_t *lenp, loff_t *ppos) +{ + unsigned int val = static_branch_likely(&sched_poc_early_select) ? 1 : 0; + struct ctl_table tmp = { + .data = &val, + .maxlen = sizeof(val), + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }; + int ret = proc_douintvec_minmax(&tmp, write, buffer, lenp, ppos); + + if (!ret && write) { + if (val) + static_branch_enable(&sched_poc_early_select); + else + static_branch_disable(&sched_poc_early_select); + } + return ret; +} + +static int sched_poc_greedy_search_handler(const struct ctl_table *table, + int write, void *buffer, + size_t *lenp, loff_t *ppos) +{ + unsigned int val = static_branch_likely(&sched_poc_greedy_search) ? 1 : 0; + struct ctl_table tmp = { + .data = &val, + .maxlen = sizeof(val), + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }; + int ret = proc_douintvec_minmax(&tmp, write, buffer, lenp, ppos); + + if (!ret && write) { + if (val) + static_branch_enable(&sched_poc_greedy_search); + else + static_branch_disable(&sched_poc_greedy_search); + } + return ret; +} + +static int sched_poc_count_sysctl_handler(const struct ctl_table *table, + int write, void *buffer, + size_t *lenp, loff_t *ppos) +{ + unsigned int val = static_branch_unlikely(&sched_poc_count_enabled) ? 1 : 0; + struct ctl_table tmp = { + .data = &val, + .maxlen = sizeof(val), + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }; + int ret = proc_douintvec_minmax(&tmp, write, buffer, lenp, ppos); + + if (!ret && write) { + if (val) + static_branch_enable(&sched_poc_count_enabled); + else + static_branch_disable(&sched_poc_count_enabled); + } + return ret; +} + +static int sched_poc_lockless_bitmap_sysctl_handler(const struct ctl_table *table, + int write, void *buffer, + size_t *lenp, loff_t *ppos) +{ + unsigned int val = static_branch_unlikely(&sched_poc_lockless_bitmap) ? 1 : 0; + struct ctl_table tmp = { + .data = &val, + .maxlen = sizeof(val), + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }; + int ret = proc_douintvec_minmax(&tmp, write, buffer, lenp, ppos); + + if (!ret && write) { + cpus_read_lock(); + if (val) + static_branch_enable_cpuslocked(&sched_poc_lockless_bitmap); + else + static_branch_disable_cpuslocked(&sched_poc_lockless_bitmap); + /* + * Resync the newly-active representation so readers see + * consistent state immediately after the mode switch. + */ + poc_resync_idle_state(); + cpus_read_unlock(); + } + return ret; +} + +static struct ctl_table sched_poc_sysctls[] = { + { + .procname = "sched_poc_selector", + .data = NULL, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_poc_sysctl_handler, + }, + { + .procname = "sched_poc_smt_fallback", + .data = NULL, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_poc_smt_fallback_sysctl_handler, + }, + { + .procname = "sched_poc_rr_improved", + .data = NULL, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_poc_rr_improved_sysctl_handler, + }, + { + .procname = "sched_poc_target_sticky", + .data = NULL, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_poc_target_sticky_sysctl_handler, + }, + { + .procname = "sched_poc_early_select", + .data = NULL, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_poc_early_select_handler, + }, + { + .procname = "sched_poc_greedy_search", + .data = NULL, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_poc_greedy_search_handler, + }, + { + .procname = "sched_poc_count", + .data = NULL, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_poc_count_sysctl_handler, + }, + { + .procname = "sched_poc_lockless_bitmap", + .data = NULL, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_poc_lockless_bitmap_sysctl_handler, + }, +}; + +static int __init sched_poc_sysctl_init(void) +{ + printk(KERN_INFO "%s %s by %s [CTZ: %s, PTSelect: %s]\n", + SCHED_POC_SELECTOR_PROGNAME, SCHED_POC_SELECTOR_VERSION, + SCHED_POC_SELECTOR_AUTHOR, POC_CTZ64_NAME, POC_PTSELECT_NAME); + + register_sysctl_init("kernel", sched_poc_sysctls); + return 0; +} +late_initcall(sched_poc_sysctl_init); + +#endif /* CONFIG_SYSCTL */ + +/* + * Initialize per-CPU RR counters with CPU ID offset. + * Different starting values shift the FASTRANGE16 phase per CPU, + * reducing cross-CPU collision probability when multiple CPUs + * perform burst wakeups against the same idle bitmap snapshot. + */ +static int __init sched_poc_rr_init(void) +{ + int cpu; + + for_each_possible_cpu(cpu) + per_cpu(poc_rr_counter, cpu) = (u32)cpu; + return 0; +} +early_initcall(sched_poc_rr_init); + +/************************************************************** + * Status: sysfs interface (always available) + * + * Exported at /sys/kernel/poc_selector/status/ for runtime status queries. + * Reports whether POC is actually active (combining all conditions). + */ + +#ifdef CONFIG_SYSFS + +/* Root kobject shared with debug section */ +static struct kobject *kobj_poc_root; + +static bool poc_check_all_llc_eligible(void) +{ + int cpu; + + for_each_online_cpu(cpu) { + struct sched_domain_shared *sd_share; + + scoped_guard(rcu) { + sd_share = rcu_dereference(per_cpu(sd_llc_shared, cpu)); + if (sd_share && !sd_share->poc_fast_eligible) + return false; + } + } + return true; +} + +static ssize_t active_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + bool active = static_branch_likely(&poc_selector_active) && + !sched_asym_cpucap_active() && + poc_check_all_llc_eligible(); + return sysfs_emit(buf, "%d\n", active ? 1 : 0); +} + +static ssize_t symmetric_cpucap_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%d\n", sched_asym_cpucap_active() ? 0 : 1); +} + +static ssize_t all_llc_eligible_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%d\n", poc_check_all_llc_eligible() ? 1 : 0); +} + +static ssize_t version_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%s\n", SCHED_POC_SELECTOR_VERSION); +} + +static struct kobj_attribute poc_status_active_attr = __ATTR_RO(active); +static struct kobj_attribute poc_status_asym_attr = __ATTR_RO(symmetric_cpucap); +static struct kobj_attribute poc_status_eligible_attr = __ATTR_RO(all_llc_eligible); +static struct kobj_attribute poc_status_version_attr = __ATTR_RO(version); + +static struct attribute *poc_status_attrs[] = { + &poc_status_active_attr.attr, + &poc_status_asym_attr.attr, + &poc_status_eligible_attr.attr, + &poc_status_version_attr.attr, + NULL, +}; + +static const struct attribute_group poc_status_group = { + .name = "status", + .attrs = poc_status_attrs, +}; + +/* --- hw_accel: expose which hardware acceleration is in use --- */ + +#define DEFINE_POC_HW_ATTR(fname, namestr) \ +static ssize_t poc_hw_##fname##_show(struct kobject *kobj, \ + struct kobj_attribute *attr, char *buf) \ +{ \ + return sysfs_emit(buf, "%s\n", namestr); \ +} \ +static struct kobj_attribute poc_hw_attr_##fname = { \ + .attr = { .name = #fname, .mode = 0444 }, \ + .show = poc_hw_##fname##_show, \ +} + +DEFINE_POC_HW_ATTR(ctz, POC_CTZ64_NAME); +DEFINE_POC_HW_ATTR(ptselect, POC_PTSELECT_NAME); + +/* popcnt: x86 uses runtime alternatives, detect via boot_cpu_has */ +static ssize_t poc_hw_popcnt_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ +#if defined(__x86_64__) + return sysfs_emit(buf, "%s\n", + boot_cpu_has(X86_FEATURE_POPCNT) ? "HW (POPCNT)" : "SW"); +#elif defined(__aarch64__) + return sysfs_emit(buf, "HW (CNT)\n"); +#elif defined(__riscv) && defined(__riscv_zbb) + return sysfs_emit(buf, "HW (cpop)\n"); +#else + return sysfs_emit(buf, "SW\n"); +#endif +} + +static struct kobj_attribute poc_hw_attr_popcnt = { + .attr = { .name = "popcnt", .mode = 0444 }, + .show = poc_hw_popcnt_show, +}; + +static struct attribute *poc_hw_attrs[] = { + &poc_hw_attr_popcnt.attr, + &poc_hw_attr_ctz.attr, + &poc_hw_attr_ptselect.attr, + NULL, +}; + +static const struct attribute_group poc_hw_group = { + .name = "hw_accel", + .attrs = poc_hw_attrs, +}; + +/* --- count: per-level hit counters (sysctl kernel.sched_poc_count) --- */ + +static unsigned long poc_sum_level(enum poc_level lvl) +{ + unsigned long sum = 0; + int cpu; + + for_each_possible_cpu(cpu) + sum += per_cpu(poc_debug_cnt[lvl], cpu); + return sum; +} + +#define DEFINE_POC_COUNT_ATTR(fname, level) \ +static ssize_t poc_count_##fname##_show(struct kobject *kobj, \ + struct kobj_attribute *attr, char *buf) \ +{ \ + return sysfs_emit(buf, "%lu\n", poc_sum_level(level)); \ +} \ +static struct kobj_attribute poc_count_##fname##_attr = { \ + .attr = { .name = #fname, .mode = 0444 }, \ + .show = poc_count_##fname##_show, \ +} + +DEFINE_POC_COUNT_ATTR(l1s, POC_LV1S); +DEFINE_POC_COUNT_ATTR(l1t, POC_LV1T); +DEFINE_POC_COUNT_ATTR(l1p, POC_LV1P); +DEFINE_POC_COUNT_ATTR(l1r, POC_LV1R); +DEFINE_POC_COUNT_ATTR(l2, POC_LV2); +DEFINE_POC_COUNT_ATTR(l3, POC_LV3); +DEFINE_POC_COUNT_ATTR(l4s, POC_LV4S); +DEFINE_POC_COUNT_ATTR(l4p, POC_LV4P); +DEFINE_POC_COUNT_ATTR(l4r, POC_LV4R); +DEFINE_POC_COUNT_ATTR(l4t, POC_LV4T); +DEFINE_POC_COUNT_ATTR(l5, POC_LV5); +DEFINE_POC_COUNT_ATTR(l6, POC_LV6); +DEFINE_POC_COUNT_ATTR(fallback, POC_FALLBACK); + +static ssize_t poc_count_reset_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int cpu; + + for_each_possible_cpu(cpu) + memset(per_cpu_ptr(poc_debug_cnt, cpu), 0, + sizeof(poc_debug_cnt)); + return count; +} + +static struct kobj_attribute poc_count_reset_attr = { + .attr = { .name = "reset", .mode = 0200 }, + .store = poc_count_reset_store, +}; + +static struct attribute *poc_count_attrs[] = { + &poc_count_l1s_attr.attr, + &poc_count_l1t_attr.attr, + &poc_count_l1p_attr.attr, + &poc_count_l1r_attr.attr, + &poc_count_l2_attr.attr, + &poc_count_l3_attr.attr, + &poc_count_l4s_attr.attr, + &poc_count_l4p_attr.attr, + &poc_count_l4r_attr.attr, + &poc_count_l4t_attr.attr, + &poc_count_l5_attr.attr, + &poc_count_l6_attr.attr, + &poc_count_fallback_attr.attr, + &poc_count_reset_attr.attr, + NULL, +}; + +static const struct attribute_group poc_count_group = { + .name = "count", + .attrs = poc_count_attrs, +}; + +static int __init sched_poc_status_init(void) +{ + int ret; + + kobj_poc_root = kobject_create_and_add("poc_selector", kernel_kobj); + if (!kobj_poc_root) + return -ENOMEM; + + ret = sysfs_create_group(kobj_poc_root, &poc_status_group); + if (ret) + goto err_status; + + ret = sysfs_create_group(kobj_poc_root, &poc_hw_group); + if (ret) + goto err_hw; + + ret = sysfs_create_group(kobj_poc_root, &poc_count_group); + if (ret) + goto err_selected; + + return 0; + +err_selected: + sysfs_remove_group(kobj_poc_root, &poc_hw_group); +err_hw: + sysfs_remove_group(kobj_poc_root, &poc_status_group); +err_status: + kobject_put(kobj_poc_root); + kobj_poc_root = NULL; + return ret; +} +late_initcall(sched_poc_status_init); + +#endif /* CONFIG_SYSFS */ +#endif /* CONFIG_SCHED_POC_SELECTOR */ diff -ruN --no-dereference -x .git a/kernel/sched/sched.h b/kernel/sched/sched.h --- a/kernel/sched/sched.h 2026-10-11 15:27:50.000000000 +0200 +++ b/kernel/sched/sched.h 2026-10-13 14:33:30.000000000 +0200 @@ -493,6 +493,8 @@ * will also be accessed at each tick. */ atomic_long_t load_avg ____cacheline_aligned; + atomic_long_t runnable_avg; + #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_RT_GROUP_SCHED @@ -528,21 +530,8 @@ }; -#ifdef CONFIG_GROUP_SCHED_WEIGHT #define ROOT_TASK_GROUP_LOAD NICE_0_LOAD -/* - * A weight of 0 or 1 can cause arithmetics problems. - * A weight of a cfs_rq is the sum of weights of which entities - * are queued on this cfs_rq, so a weight of a entity should not be - * too large, so as the shares value of a task group. - * (The default weight is 1024 - so there's no practical - * limitation from this.) - */ -#define MIN_SHARES (1UL << 1) -#define MAX_SHARES (1UL << 18) -#endif - typedef int (*tg_visitor)(struct task_group *, void *); extern int walk_tg_tree_from(struct task_group *from, @@ -571,6 +560,7 @@ extern int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent); extern void online_fair_sched_group(struct task_group *tg); extern void unregister_fair_sched_group(struct task_group *tg); +extern void __sched_cgroup_mode_update(int mode); #else /* !CONFIG_FAIR_GROUP_SCHED: */ static inline void free_fair_sched_group(struct task_group *tg) { } static inline int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) @@ -628,6 +618,17 @@ #endif /* !CONFIG_CGROUP_SCHED */ +/* + * A weight of 0 or 1 can cause arithmetics problems. + * A weight of a cfs_rq is the sum of weights of which entities + * are queued on this cfs_rq, so a weight of a entity should not be + * too large, so as the shares value of a task group. + * (The default weight is 1024 - so there's no practical + * limitation from this.) + */ +#define MIN_SHARES (1UL << 1) +#define MAX_SHARES (1UL << 18) + extern void unregister_rt_sched_group(struct task_group *tg); extern void free_rt_sched_group(struct task_group *tg); extern int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent); @@ -706,6 +707,7 @@ /* * CFS load tracking */ + struct sched_entity *h_curr; struct sched_avg avg; #ifndef CONFIG_64BIT u64 last_update_time_copy; @@ -721,6 +723,7 @@ #ifdef CONFIG_FAIR_GROUP_SCHED u64 last_update_tg_load_avg; unsigned long tg_load_avg_contrib; + unsigned long tg_runnable_avg_contrib; long propagate; long prop_runnable_sum; @@ -1179,6 +1182,10 @@ call_single_data_t nohz_csd; #endif /* CONFIG_NO_HZ_COMMON */ +#ifdef CONFIG_SCHED_POC_SELECTOR + unsigned int poc_idle_committed; +#endif + #ifdef CONFIG_UCLAMP_TASK /* Utilization clamp values based on CPU's RUNNABLE tasks */ struct uclamp_rq uclamp[UCLAMP_CNT] ____cacheline_aligned; @@ -1460,12 +1467,12 @@ DECLARE_STATIC_KEY_FALSE(__sched_core_enabled); -static inline bool sched_core_enabled(struct rq *rq) +static __always_inline bool sched_core_enabled(struct rq *rq) { return static_branch_unlikely(&__sched_core_enabled) && rq->core_enabled; } -static inline bool sched_core_disabled(void) +static __always_inline bool sched_core_disabled(void) { return !static_branch_unlikely(&__sched_core_enabled); } @@ -1474,7 +1481,7 @@ * Be careful with this function; not for general use. The return value isn't * stable unless you actually hold a relevant rq->__lock. */ -static inline raw_spinlock_t *rq_lockp(struct rq *rq) +static __always_inline raw_spinlock_t *rq_lockp(struct rq *rq) { if (sched_core_enabled(rq)) return &rq->core->__lock; @@ -1482,7 +1489,7 @@ return &rq->__lock; } -static inline raw_spinlock_t *__rq_lockp(struct rq *rq) +static __always_inline raw_spinlock_t *__rq_lockp(struct rq *rq) __returns_ctx_lock(rq_lockp(rq)) /* alias them */ { if (rq->core_enabled) @@ -1585,12 +1592,12 @@ return true; } -static inline raw_spinlock_t *rq_lockp(struct rq *rq) +static __always_inline raw_spinlock_t *rq_lockp(struct rq *rq) { return &rq->__lock; } -static inline raw_spinlock_t *__rq_lockp(struct rq *rq) +static __always_inline raw_spinlock_t *__rq_lockp(struct rq *rq) __returns_ctx_lock(rq_lockp(rq)) /* alias them */ { return &rq->__lock; @@ -1650,33 +1657,33 @@ extern bool raw_spin_rq_trylock(struct rq *rq) __cond_acquires(true, __rq_lockp(rq)); -static inline void raw_spin_rq_lock(struct rq *rq) +static __always_inline void raw_spin_rq_lock(struct rq *rq) __acquires(__rq_lockp(rq)) { raw_spin_rq_lock_nested(rq, 0); } -static inline void raw_spin_rq_unlock(struct rq *rq) +static __always_inline void raw_spin_rq_unlock(struct rq *rq) __releases(__rq_lockp(rq)) { raw_spin_unlock(rq_lockp(rq)); } -static inline void raw_spin_rq_lock_irq(struct rq *rq) +static __always_inline void raw_spin_rq_lock_irq(struct rq *rq) __acquires(__rq_lockp(rq)) { local_irq_disable(); raw_spin_rq_lock(rq); } -static inline void raw_spin_rq_unlock_irq(struct rq *rq) +static __always_inline void raw_spin_rq_unlock_irq(struct rq *rq) __releases(__rq_lockp(rq)) { raw_spin_rq_unlock(rq); local_irq_enable(); } -static inline unsigned long _raw_spin_rq_lock_irqsave(struct rq *rq) +static __always_inline unsigned long _raw_spin_rq_lock_irqsave(struct rq *rq) __acquires(__rq_lockp(rq)) { unsigned long flags; @@ -1687,7 +1694,7 @@ return flags; } -static inline void raw_spin_rq_unlock_irqrestore(struct rq *rq, unsigned long flags) +static __always_inline void raw_spin_rq_unlock_irqrestore(struct rq *rq, unsigned long flags) __releases(__rq_lockp(rq)) { raw_spin_rq_unlock(rq); @@ -2374,6 +2381,119 @@ #endif /* !CONFIG_CGROUP_SCHED */ +#ifdef CONFIG_SCHED_POC_SELECTOR +extern struct static_key_true poc_selector_active; +#ifdef CONFIG_SCHED_CLASS_EXT +extern void poc_notify_scx(bool scx_active); +extern void poc_check_skip_fallback(void); +#else +static inline void poc_check_skip_fallback(void) {} +#endif +extern struct static_key_true sched_poc_aligned; +extern struct static_key_true sched_poc_smt_consecutive; +extern struct static_key_true sched_poc_smt_uniform; +extern struct static_key_false sched_poc_target_sticky; +extern struct static_key_true sched_poc_packed; +extern struct static_key_false sched_poc_lockless_bitmap; +extern void __set_cpu_idle_state_poc(int cpu, int state); +extern void poc_sd_shared_init(struct sched_domain *sd, int sd_id); +static __always_inline void set_cpu_idle_state_poc(int cpu, int state) +{ + if (static_branch_likely(&poc_selector_active) && + !sched_asym_cpucap_active()) + __set_cpu_idle_state_poc(cpu, state); +} + +/* + * POC_CTZ64 - Count trailing zeros (find first set bit) + * + * Architecture-optimized CTZ for POC idle CPU selection. + * Returns 64 for input 0 (important for BSF-based implementations). + */ +#if defined(__x86_64__) && defined(__BMI__) +/* Tier 1: x86-64 with BMI1 - TZCNT is zero-safe */ +#define POC_CTZ64(v) ((int)__builtin_ctzll(v)) + +#elif defined(__aarch64__) +/* Tier 1: ARM64 - RBIT+CLZ is zero-safe */ +#define POC_CTZ64(v) ((int)__builtin_ctzll(v)) + +#elif defined(__riscv) && defined(__riscv_zbb) +/* Tier 1: RISC-V with Zbb - CTZ is zero-safe */ +#define POC_CTZ64(v) ((int)__builtin_ctzll(v)) + +#elif defined(__x86_64__) +/* Tier 2: x86-64 without BMI1 - BSF needs zero check */ +static __always_inline int poc_ctz64_bsf(u64 v) +{ + if (unlikely(!v)) + return 64; + return (int)__builtin_ctzll(v); +} +#define POC_CTZ64(v) poc_ctz64_bsf(v) + +#else +/* Tier 3: De Bruijn fallback for other architectures */ +#define POC_DEBRUIJN_CTZ64_CONST 0x03F79D71B4CA8B09ULL +static const u8 poc_debruijn_ctz64_tab[64] = { + 0, 1, 56, 2, 57, 49, 28, 3, + 61, 58, 42, 50, 38, 29, 17, 4, + 62, 47, 59, 36, 45, 43, 51, 22, + 53, 39, 33, 30, 24, 18, 12, 5, + 63, 55, 48, 27, 60, 41, 37, 16, + 46, 35, 44, 21, 52, 32, 23, 11, + 54, 26, 40, 15, 34, 20, 31, 10, + 25, 14, 19, 9, 13, 8, 7, 6, +}; +static __always_inline int poc_debruijn_ctz64(u64 v) +{ + u64 lsb; + u32 idx; + + if (unlikely(!v)) + return 64; + lsb = v & (-(s64)v); + idx = (u32)((lsb * POC_DEBRUIJN_CTZ64_CONST) >> 58); + return (int)poc_debruijn_ctz64_tab[idx & 63]; +} +#define POC_CTZ64(v) poc_debruijn_ctz64(v) + +#endif /* POC_CTZ64 */ + +/* + * POC helper: convert cpumask region to POC-relative u64 + * + * Extracts the 64-bit region of @mask corresponding to this LLC's + * CPU range and shifts it to align with POC's bit positions. + * + * Used by load balancer functions that need to intersect cpumasks + * with POC idle bitmaps. + */ +static __always_inline u64 poc_cpumask_to_u64(const struct cpumask *mask, + struct sched_domain_shared *sd_share) +{ + int base = sd_share->poc_cpu_base; + int base_word = base >> 6; + int shift = sd_share->poc_affinity_shift; + + if (likely(shift == 0)) { + /* + * Fast path: this LLC's own base is 64-aligned, so no + * cross-word shift is needed here even if some other LLC + * in the system is misaligned (sched_poc_aligned is a + * single system-wide flag and must not gate this branch). + */ + return cpumask_bits(mask)[base_word]; + } else { + /* Slow path: shift required (e.g., Threadripper) */ + u64 lo = cpumask_bits(mask)[base_word]; + u64 hi = (base_word + 1 < BITS_TO_LONGS(nr_cpumask_bits)) ? + cpumask_bits(mask)[base_word + 1] : 0; + return (lo >> shift) | (hi << (64 - shift)); + } +} +#endif /* CONFIG_SCHED_POC_SELECTOR */ + static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) { set_task_rq(p, cpu); @@ -2574,6 +2694,7 @@ #define ENQUEUE_MIGRATED 0x00040000 #define ENQUEUE_INITIAL 0x00080000 #define ENQUEUE_RQ_SELECTED 0x00100000 +#define ENQUEUE_QUEUED 0x00200000 #define RETRY_TASK ((void *)-1UL) @@ -3103,7 +3224,7 @@ attach_task(rq, p); } -#ifdef CONFIG_PREEMPT_RT +#if defined(CONFIG_PREEMPT_RT) || defined(CONFIG_CACHY) # define SCHED_NR_MIGRATE_BREAK 8 #else # define SCHED_NR_MIGRATE_BREAK 32 @@ -3452,6 +3573,7 @@ static inline void nohz_run_idle_balance(int cpu) { } #endif + #include "stats.h" #if defined(CONFIG_SCHED_CORE) && defined(CONFIG_SCHEDSTATS) diff -ruN --no-dereference -x .git a/kernel/sched/topology.c b/kernel/sched/topology.c --- a/kernel/sched/topology.c 2026-10-11 15:27:50.000000000 +0200 +++ b/kernel/sched/topology.c 2026-10-13 14:33:30.000000000 +0200 @@ -3174,6 +3174,9 @@ if (sd->flags & SD_SHARE_LLC) { init_sched_domain_shared(&d, sd, SD_SHARE_LLC); +#ifdef CONFIG_SCHED_POC_SELECTOR + poc_sd_shared_init(sd, cpumask_first(sched_domain_span(sd))); +#endif /* * In presence of higher domains, adjust the diff -ruN --no-dereference -x .git a/kernel/smp.c b/kernel/smp.c --- a/kernel/smp.c 2026-10-11 15:27:50.000000000 +0200 +++ b/kernel/smp.c 2026-10-13 14:33:30.000000000 +0200 @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -63,7 +64,14 @@ free_cpumask_var(cfd->cpumask); return -ENOMEM; } - cfd->csd = alloc_percpu(call_single_data_t); + + /* + * Allocate the per-CPU CSD the first time a CPU comes up. It is + * not freed when the CPU is offlined, so csd_lock_wait() can access + * it even when the CPU was offlined after preemption was re-enabled. + */ + if (!cfd->csd) + cfd->csd = alloc_percpu(call_single_data_t); if (!cfd->csd) { free_cpumask_var(cfd->cpumask); free_cpumask_var(cfd->cpumask_ipi); @@ -79,7 +87,6 @@ free_cpumask_var(cfd->cpumask); free_cpumask_var(cfd->cpumask_ipi); - free_percpu(cfd->csd); return 0; } @@ -323,6 +330,8 @@ int bug_id = 0; u64 ts0, ts1; + guard(preempt)(); + ts1 = ts0 = ktime_get_mono_fast_ns(); for (;;) { if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id, &nmessages)) @@ -659,17 +668,9 @@ local_irq_restore(flags); } -/** - * smp_call_function_single - Run a function on a specific CPU - * @cpu: Specific target CPU for this function. - * @func: The function to run. This must be fast and non-blocking. - * @info: An arbitrary pointer to pass to the function. - * @wait: If true, wait until function has completed on other CPUs. - * - * Returns: %0 on success, else a negative status code. - */ -int smp_call_function_single(int cpu, smp_call_func_t func, void *info, - int wait) +static int __smp_call_function_single(int cpu, smp_call_func_t func, + void *info, const struct cpumask *mask, + bool wait) { call_single_data_t *csd; call_single_data_t csd_stack = { @@ -686,6 +687,14 @@ */ this_cpu = get_cpu(); + if (mask) { + /* Try for same CPU (cheapest) */ + if (!cpumask_test_cpu(this_cpu, mask)) + cpu = sched_numa_find_nth_cpu(mask, 0, cpu_to_node(this_cpu)); + else + cpu = this_cpu; + } + /* * Can deadlock when called with interrupts disabled. * We allow cpu's that are not yet online though, as no one else can @@ -718,13 +727,32 @@ err = generic_exec_single(cpu, csd); + /* + * @csd is stack-allocated when @wait is true. No concurrent access + * except from the IPI completion path, so we can re-enable preemption + * early to reduce latency. + */ + put_cpu(); + if (wait) csd_lock_wait(csd); - put_cpu(); - return err; } + +/** + * smp_call_function_single - Run a function on a specific CPU + * @cpu: Specific target CPU for this function. + * @func: The function to run. This must be fast and non-blocking. + * @info: An arbitrary pointer to pass to the function. + * @wait: If true, wait until function has completed on other CPUs. + * + * Returns: %0 on success, else a negative status code. + */ +int smp_call_function_single(int cpu, smp_call_func_t func, void *info, bool wait) +{ + return __smp_call_function_single(cpu, func, info, NULL, wait); +} EXPORT_SYMBOL(smp_call_function_single); /** @@ -775,10 +803,10 @@ /** * smp_call_function_any - Run a function on any of the given cpus - * @mask: The mask of cpus it can run on. - * @func: The function to run. This must be fast and non-blocking. - * @info: An arbitrary pointer to pass to the function. - * @wait: If true, wait until function has completed. + * @mask: The mask of cpus it can run on. + * @func: The function to run. This must be fast and non-blocking. + * @info: An arbitrary pointer to pass to the function. + * @wait: If true, wait until function has completed. * * Selection preference: * 1) current cpu if in @mask @@ -789,19 +817,53 @@ int smp_call_function_any(const struct cpumask *mask, smp_call_func_t func, void *info, int wait) { - unsigned int cpu; - int ret; + return __smp_call_function_single(-1, func, info, mask, wait); +} +EXPORT_SYMBOL_GPL(smp_call_function_any); - /* Try for same CPU (cheapest) */ - cpu = get_cpu(); - if (!cpumask_test_cpu(cpu, mask)) - cpu = sched_numa_find_nth_cpu(mask, 0, cpu_to_node(cpu)); +static DEFINE_STATIC_KEY_FALSE(ipi_mask_inlined); - ret = smp_call_function_single(cpu, func, info, wait); - put_cpu(); - return ret; +#ifdef CONFIG_PREEMPTION + +int smp_task_ipi_mask_alloc(struct task_struct *task) +{ + if (static_branch_unlikely(&ipi_mask_inlined)) + return 0; + + ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr = + kmalloc(cpumask_size(), GFP_KERNEL); + if (!ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr) + return -ENOMEM; + + return 0; } -EXPORT_SYMBOL_GPL(smp_call_function_any); + +void smp_task_ipi_mask_free(struct task_struct *task) +{ + if (static_branch_unlikely(&ipi_mask_inlined)) + return; + + kfree(ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr); +} + +static cpumask_t *smp_task_ipi_mask(struct task_struct *cur) +{ + /* + * If cpumask_size() is smaller than or equal to the pointer + * size, it stashes the cpumask in the pointer itself to + * avoid extra memory allocations. + */ + if (static_branch_unlikely(&ipi_mask_inlined)) + return (cpumask_t *)&ACCESS_PRIVATE(cur, ipi_mask).ipi_mask_val; + + return ACCESS_PRIVATE(cur, ipi_mask).ipi_mask_ptr; +} +#else +static cpumask_t *smp_task_ipi_mask(struct task_struct *cur) +{ + return NULL; +} +#endif /* * Flags to be used as scf_flags argument of smp_call_function_many_cond(). @@ -817,13 +879,20 @@ unsigned int scf_flags, smp_cond_func_t cond_func) { - int cpu, last_cpu, this_cpu = smp_processor_id(); - struct call_function_data *cfd; + struct cpumask *cpumask, *task_mask; bool wait = scf_flags & SCF_WAIT; - int nr_cpus = 0; + struct call_function_data *cfd; + int cpu, last_cpu, this_cpu; bool run_remote = false; + int nr_cpus = 0; - lockdep_assert_preemption_disabled(); + this_cpu = get_cpu(); + cfd = this_cpu_ptr(&cfd_data); + task_mask = smp_task_ipi_mask(current); + if (task_mask) + cpumask = task_mask; + else + cpumask = cfd->cpumask; /* * Can deadlock when called with interrupts disabled. @@ -845,16 +914,15 @@ /* Check if we need remote execution, i.e., any CPU excluding this one. */ if (cpumask_any_and_but(mask, cpu_online_mask, this_cpu) < nr_cpu_ids) { - cfd = this_cpu_ptr(&cfd_data); - cpumask_and(cfd->cpumask, mask, cpu_online_mask); - __cpumask_clear_cpu(this_cpu, cfd->cpumask); + cpumask_and(cpumask, mask, cpu_online_mask); + __cpumask_clear_cpu(this_cpu, cpumask); cpumask_clear(cfd->cpumask_ipi); - for_each_cpu(cpu, cfd->cpumask) { + for_each_cpu(cpu, cpumask) { call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu); if (cond_func && !cond_func(cpu, info)) { - __cpumask_clear_cpu(cpu, cfd->cpumask); + __cpumask_clear_cpu(cpu, cpumask); continue; } @@ -904,8 +972,18 @@ local_irq_restore(flags); } + /* + * The IPI work has been queued and dispatched. On PREEMPT kernels, + * tasks created through dup_task_struct() have task-local wait masks. + * The boot init_task can fall back to cfd->cpumask when the mask is + * not inlined, but other tasks still use task-local masks and cannot + * overwrite it. On !PREEMPT kernels, preempt_enable() cannot schedule + * another task, so the per-CPU mask remains protected. + */ + put_cpu(); + if (run_remote && wait) { - for_each_cpu(cpu, cfd->cpumask) { + for_each_cpu(cpu, cpumask) { call_single_data_t *csd; csd = per_cpu_ptr(cfd->csd, cpu); @@ -916,15 +994,14 @@ /** * smp_call_function_many() - Run a function on a set of CPUs. - * @mask: The set of cpus to run on (only runs on online subset). - * @func: The function to run. This must be fast and non-blocking. - * @info: An arbitrary pointer to pass to the function. - * @wait: If true, wait (atomically) until function has completed - * on other CPUs. + * @mask: The set of cpus to run on (only runs on online subset). + * @func: The function to run. This must be fast and non-blocking. + * @info: An arbitrary pointer to pass to the function. + * @wait: If true, wait (atomically) until function has completed + * on other CPUs. * * You must not call this function with disabled interrupts or from a - * hardware interrupt handler or from a bottom half handler. Preemption - * must be disabled when calling this function. + * hardware interrupt handler or from a bottom half handler. * * @func is not called on the local CPU even if @mask contains it. Consider * using on_each_cpu_cond_mask() instead if this is not desirable. @@ -938,10 +1015,10 @@ /** * smp_call_function() - Run a function on all other CPUs. - * @func: The function to run. This must be fast and non-blocking. - * @info: An arbitrary pointer to pass to the function. - * @wait: If true, wait (atomically) until function has completed - * on other CPUs. + * @func: The function to run. This must be fast and non-blocking. + * @info: An arbitrary pointer to pass to the function. + * @wait: If true, wait (atomically) until function has completed + * on other CPUs. * * If @wait is true, then returns once @func has returned; otherwise * it returns just before the target cpu calls @func. @@ -951,9 +1028,8 @@ */ void smp_call_function(smp_call_func_t func, void *info, int wait) { - preempt_disable(); - smp_call_function_many(cpu_online_mask, func, info, wait); - preempt_enable(); + smp_call_function_many_cond(cpu_online_mask, func, info, + wait ? SCF_WAIT : 0, NULL); } EXPORT_SYMBOL(smp_call_function); @@ -1019,6 +1095,9 @@ void __init setup_nr_cpu_ids(void) { set_nr_cpu_ids(find_last_bit(cpumask_bits(cpu_possible_mask), NR_CPUS) + 1); + + if (IS_ENABLED(CONFIG_PREEMPTION) && cpumask_size() <= sizeof(unsigned long)) + static_branch_enable(&ipi_mask_inlined); } /* Called by boot processor to activate the rest. */ @@ -1055,12 +1134,14 @@ * @func: The function to run on all applicable CPUs. * This must be fast and non-blocking. * @info: An arbitrary pointer to pass to both functions. - * @wait: If true, wait (atomically) until function has - * completed on other CPUs. + * @wait: If true, wait until function has completed on other CPUs. * @mask: The set of cpus to run on (only runs on online subset). * - * Preemption is disabled to protect against CPUs going offline but not online. - * CPUs going online during the call will not be seen or sent an IPI. + * Target CPU selection and work queueing are done with preemption + * disabled. This protects against CPUs going offline, but not against + * CPUs coming online concurrently; newly online CPUs are not guaranteed + * to be seen or sent an IPI. If @wait is true, the final wait for remote + * completion happens after that preemption-disabled section. * * You must not call this function with disabled interrupts or * from a hardware interrupt handler or from a bottom half handler. @@ -1073,9 +1154,7 @@ if (wait) scf_flags |= SCF_WAIT; - preempt_disable(); smp_call_function_many_cond(mask, func, info, scf_flags, cond_func); - preempt_enable(); } EXPORT_SYMBOL(on_each_cpu_cond_mask); diff -ruN --no-dereference -x .git a/kernel/up.c b/kernel/up.c --- a/kernel/up.c 2026-10-11 15:27:50.000000000 +0200 +++ b/kernel/up.c 2026-10-13 14:33:30.000000000 +0200 @@ -9,8 +9,7 @@ #include #include -int smp_call_function_single(int cpu, void (*func) (void *info), void *info, - int wait) +int smp_call_function_single(int cpu, void (*func)(void *info), void *info, bool wait) { unsigned long flags; diff -ruN --no-dereference -x .git a/mm/compaction.c b/mm/compaction.c --- a/mm/compaction.c 2026-10-11 15:27:50.000000000 +0200 +++ b/mm/compaction.c 2026-10-13 14:33:30.000000000 +0200 @@ -1935,7 +1935,11 @@ * aggressively the kernel should compact memory in the * background. It takes values in the range [0, 100]. */ +#ifdef CONFIG_CACHY +static unsigned int __read_mostly sysctl_compaction_proactiveness; +#else static unsigned int __read_mostly sysctl_compaction_proactiveness = 20; +#endif static int sysctl_extfrag_threshold = 500; static int __read_mostly sysctl_compact_memory; diff -ruN --no-dereference -x .git a/mm/huge_memory.c b/mm/huge_memory.c --- a/mm/huge_memory.c 2026-10-11 15:27:50.000000000 +0200 +++ b/mm/huge_memory.c 2026-10-13 14:33:30.000000000 +0200 @@ -65,7 +65,11 @@ #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE (1<mm; @@ -787,6 +798,11 @@ * to undo, we also need to drop a reference to the anon_vma. */ put_anon_vma(rmap_item->anon_vma); + /* + * Reset linear_page_index that might overlay age-related + * information. (it's still unstable node) + */ + rmap_item->linear_page_index = 0; mmap_read_lock(mm); vma = find_mergeable_vma(mm, addr); @@ -899,6 +915,8 @@ VM_BUG_ON(stable_node->rmap_hlist_len <= 0); stable_node->rmap_hlist_len--; put_anon_vma(rmap_item->anon_vma); + /* Reset linear_page_index that might overlay age-related information. */ + rmap_item->linear_page_index = 0; rmap_item->address &= PAGE_MASK; cond_resched(); } @@ -1052,6 +1070,8 @@ stable_node->rmap_hlist_len--; put_anon_vma(rmap_item->anon_vma); + /* Reset linear_page_index that might overlay age-related information. */ + rmap_item->linear_page_index = 0; rmap_item->head = NULL; rmap_item->address &= PAGE_MASK; @@ -1598,8 +1618,15 @@ /* Unstable nid is in union with stable anon_vma: remove first */ remove_rmap_item_from_tree(rmap_item); - /* Must get reference to anon_vma while still holding mmap_lock */ + /* + * We can consider the VMA only while still holding the mmap lock, + * so lock, so reference the anon_vma and calculate the linear + * page index early, before stable_tree_append(). If anything goes + * wrong that prevents the rmap_item from being added to the + * stable_tree, break_cow() will clean it up. + */ rmap_item->anon_vma = vma->anon_vma; + rmap_item->linear_page_index = linear_page_index(vma, rmap_item->address); get_anon_vma(vma->anon_vma); out: mmap_read_unlock(mm); @@ -2458,6 +2485,13 @@ if (folio_test_ksm(folio)) return false; + /* + * There is no age information in stable-tree nodes. We might end up + * here without a KSM page for example after COW. + */ + if (rmap_item->address & STABLE_FLAG) + return false; + age = rmap_item->age; if (age != U8_MAX) rmap_item->age++; @@ -3173,6 +3207,7 @@ hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { /* Ignore the stable/unstable/sqnr flags */ const unsigned long addr = rmap_item->address & PAGE_MASK; + const unsigned long index = rmap_item->linear_page_index; struct anon_vma *anon_vma = rmap_item->anon_vma; struct anon_vma_chain *vmac; struct vm_area_struct *vma; @@ -3186,8 +3221,18 @@ anon_vma_lock_read(anon_vma); } + /* + * Currently, KSM folios are always small folios, so it's + * sufficient to search for a single page. We can simply use + * the linear_page_index of the original de-duplicate + * anonymous page that we remembered in the rmap_item while + * de-duplicating. Note that mremap() always de-duplicates KSM + * folios: so if there was mremap() in our parent or our child, + * we wouldn't have the KSM folio mapped in these processes + * anymore. + */ anon_vma_interval_tree_foreach(vmac, &anon_vma->rb_root, - 0, ULONG_MAX) { + index, index) { cond_resched(); vma = vmac->vma; diff -ruN --no-dereference -x .git a/mm/page_alloc.c b/mm/page_alloc.c --- a/mm/page_alloc.c 2026-10-11 15:27:50.000000000 +0200 +++ b/mm/page_alloc.c 2026-10-13 14:33:30.000000000 +0200 @@ -270,7 +270,11 @@ int min_free_kbytes = 1024; int user_min_free_kbytes = -1; +#ifdef CONFIG_CACHY +static int watermark_boost_factor __read_mostly; +#else static int watermark_boost_factor __read_mostly = 15000; +#endif static int watermark_scale_factor = 10; int defrag_mode; diff -ruN --no-dereference -x .git a/mm/rmap.c b/mm/rmap.c --- a/mm/rmap.c 2026-10-11 15:27:50.000000000 +0200 +++ b/mm/rmap.c 2026-10-13 14:33:30.000000000 +0200 @@ -911,7 +911,7 @@ struct folio_referenced_arg { int mapcount; int referenced; - vm_flags_t vm_flags; + vma_flags_t vma_flags; struct mem_cgroup *memcg; }; @@ -930,7 +930,7 @@ address = pvmw.address; nr = 1; - if (vma->vm_flags & VM_LOCKED) { + if (vma_test(vma, VMA_LOCKED_BIT)) { ptes++; pra->mapcount--; @@ -951,7 +951,7 @@ /* Restore the mlock which got missed */ mlock_vma_folio(folio, vma); page_vma_mapped_walk_done(&pvmw); - pra->vm_flags |= VM_LOCKED; + vma_flags_set(&pra->vma_flags, VMA_LOCKED_BIT); return false; /* To break the loop */ } @@ -1019,8 +1019,11 @@ referenced++; if (referenced) { + vma_flags_t vma_flags = vma->flags; + pra->referenced++; - pra->vm_flags |= vma->vm_flags & ~VM_LOCKED; + vma_flags_clear(&vma_flags, VMA_LOCKED_BIT); + vma_flags_set_mask(&pra->vma_flags, vma_flags); } if (!pra->mapcount) @@ -1058,7 +1061,7 @@ * @folio: The folio to test. * @is_locked: Caller holds lock on the folio. * @memcg: target memory cgroup - * @vm_flags: A combination of all the vma->vm_flags which referenced the folio. + * @vma_flags: A combination of all the vma->flags which referenced the folio. * * Quick test_and_clear_referenced for all mappings of a folio, * @@ -1066,7 +1069,7 @@ * the function bailed out due to rmap lock contention. */ int folio_referenced(struct folio *folio, int is_locked, - struct mem_cgroup *memcg, vm_flags_t *vm_flags) + struct mem_cgroup *memcg, vma_flags_t *vma_flags) { bool we_locked = false; struct folio_referenced_arg pra = { @@ -1082,7 +1085,7 @@ }; VM_WARN_ON_ONCE_FOLIO(folio_is_zone_device(folio), folio); - *vm_flags = 0; + vma_flags_clear_all(vma_flags); if (!pra.mapcount) return 0; @@ -1096,7 +1099,7 @@ } rmap_walk(folio, &rwc); - *vm_flags = pra.vm_flags; + vma_flags_set_mask(vma_flags, pra.vma_flags); if (we_locked) folio_unlock(folio); diff -ruN --no-dereference -x .git a/mm/swap.c b/mm/swap.c --- a/mm/swap.c 2026-10-11 15:27:50.000000000 +0200 +++ b/mm/swap.c 2026-10-13 14:33:30.000000000 +0200 @@ -1198,17 +1198,8 @@ */ void __init swap_setup(void) { - unsigned long megs = PAGES_TO_MB(totalram_pages()); - - /* Use a smaller cluster for small-memory machines */ - if (megs < 16) - page_cluster = 2; - else - page_cluster = 3; - /* - * Right now other parts of the system means that we - * _really_ don't want to cluster much more - */ + /* Only swap-in pages requested, avoid readahead */ + page_cluster = 0; register_sysctl_init("vm", swap_sysctl_table); } diff -ruN --no-dereference -x .git a/mm/vmscan.c b/mm/vmscan.c --- a/mm/vmscan.c 2026-10-11 15:27:50.000000000 +0200 +++ b/mm/vmscan.c 2026-10-13 14:33:30.000000000 +0200 @@ -196,7 +196,11 @@ /* * From 0 .. MAX_SWAPPINESS. Higher means more swappy. */ +#ifdef CONFIG_CACHY +int vm_swappiness = 100; +#else int vm_swappiness = 60; +#endif static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) { @@ -262,6 +266,12 @@ } #endif +static inline bool is_exec_file_folio(const struct folio *folio, + const vma_flags_t *vma_flags) +{ + return vma_flags_test(vma_flags, VMA_EXEC_BIT) && folio_is_file_lru(folio); +} + static void set_task_reclaim_state(struct task_struct *task, struct reclaim_state *rs) { @@ -830,10 +840,16 @@ * with PG_active set. In contrast, the aging (page table walk) path uses * folio_update_gen(). */ -static bool lru_gen_set_refs(struct folio *folio) +static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags) { /* see the comment on LRU_REFS_FLAGS */ if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { + /* Activate file-backed executable folios after first usage. */ + if (is_exec_file_folio(folio, vma_flags)) { + set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset)); + return true; + } + set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); return false; } @@ -846,7 +862,7 @@ return true; } #else -static bool lru_gen_set_refs(struct folio *folio) +static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags) { return false; } @@ -856,16 +872,16 @@ struct scan_control *sc) { int referenced_ptes, referenced_folio; - vm_flags_t vm_flags; + vma_flags_t vma_flags; referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup, - &vm_flags); + &vma_flags); /* * The supposedly reclaimable folio was found to be in a VM_LOCKED vma. * Let the folio, now marked Mlocked, be moved to the unevictable list. */ - if (vm_flags & VM_LOCKED) + if (vma_flags_test(&vma_flags, VMA_LOCKED_BIT)) return FOLIOREF_ACTIVATE; /* @@ -881,7 +897,7 @@ if (!referenced_ptes) return FOLIOREF_RECLAIM; - return lru_gen_set_refs(folio) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP; + return lru_gen_set_refs(folio, &vma_flags) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP; } referenced_folio = folio_test_clear_referenced(folio); @@ -909,7 +925,7 @@ /* * Activate file-backed executable folios after first usage. */ - if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) + if (is_exec_file_folio(folio, &vma_flags)) return FOLIOREF_ACTIVATE; return FOLIOREF_KEEP; @@ -2067,7 +2083,7 @@ { unsigned long nr_taken; unsigned long nr_scanned; - vm_flags_t vm_flags; + vma_flags_t vma_flags; LIST_HEAD(l_hold); /* The folios which were snipped off */ LIST_HEAD(l_active); LIST_HEAD(l_inactive); @@ -2111,7 +2127,7 @@ /* Referenced or rmap lock contention: rotate */ if (folio_referenced(folio, 0, sc->target_mem_cgroup, - &vm_flags) != 0) { + &vma_flags) != 0) { /* * Identify referenced, file-backed active folios and * give them one more trip around the active list. So @@ -2121,7 +2137,7 @@ * IO, plus JVM can create lots of anon VM_EXEC folios, * so we ignore them here. */ - if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) { + if (is_exec_file_folio(folio, &vma_flags)) { nr_rotated += folio_nr_pages(folio); list_add(&folio->lru, &l_active); continue; @@ -3190,14 +3206,19 @@ ******************************************************************************/ /* promote pages accessed through page tables */ -static int folio_update_gen(struct folio *folio, int gen) +static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma_flags) { unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); - /* see the comment on LRU_REFS_FLAGS */ - if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { + /* + * See the comment on LRU_REFS_FLAGS, and activate file-backed + * executable folios after first usage to avoid typical IO + * thrashing from reclaiming. + */ + if (!folio_test_referenced(folio) && !folio_test_workingset(folio) && + !is_exec_file_folio(folio, vma_flags)) { set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); return -1; } @@ -3430,8 +3451,8 @@ return young * n >= total; } -static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio, - int new_gen, bool dirty) +static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struct *vma, + struct folio *folio, int new_gen, bool dirty) { int old_gen; @@ -3444,10 +3465,10 @@ folio_mark_dirty(folio); if (walk) { - old_gen = folio_update_gen(folio, new_gen); + old_gen = folio_update_gen(folio, new_gen, &vma->flags); if (old_gen >= 0 && old_gen != new_gen) update_batch_size(walk, folio, old_gen, new_gen); - } else if (lru_gen_set_refs(folio)) { + } else if (lru_gen_set_refs(folio, &vma->flags)) { old_gen = folio_lru_gen(folio); if (old_gen >= 0 && old_gen != new_gen) folio_activate(folio); @@ -3520,7 +3541,7 @@ continue; if (last != folio) { - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, args->vma, last, gen, dirty); last = folio; dirty = false; @@ -3533,7 +3554,7 @@ walk->mm_stats[MM_LEAF_YOUNG] += nr; } - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, args->vma, last, gen, dirty); last = NULL; if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) @@ -3611,7 +3632,7 @@ goto next; if (last != folio) { - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); last = folio; dirty = false; @@ -3625,7 +3646,7 @@ i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; } while (i <= MIN_LRU_BATCH); - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); lazy_mmu_mode_disable(); spin_unlock(ptl); @@ -4123,7 +4144,11 @@ } /* to protect the working set of the last N jiffies */ +#ifdef CONFIG_CACHY +static unsigned long lru_gen_min_ttl __read_mostly = 100; +#else static unsigned long lru_gen_min_ttl __read_mostly; +#endif static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) { @@ -4260,7 +4285,7 @@ continue; if (last != folio) { - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); last = folio; dirty = false; @@ -4272,7 +4297,7 @@ young += nr; } - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); lazy_mmu_mode_disable(); diff -ruN --no-dereference -x .git a/net/ipv4/Kconfig b/net/ipv4/Kconfig --- a/net/ipv4/Kconfig 2026-10-11 15:27:50.000000000 +0200 +++ b/net/ipv4/Kconfig 2026-10-13 14:33:30.000000000 +0200 @@ -668,15 +668,33 @@ default n help - BBR (Bottleneck Bandwidth and RTT) TCP congestion control aims to - maximize network utilization and minimize queues. It builds an explicit - model of the bottleneck delivery rate and path round-trip propagation - delay. It tolerates packet loss and delay unrelated to congestion. It - can operate over LAN, WAN, cellular, wifi, or cable modem links. It can - coexist with flows that use loss-based congestion control, and can - operate with shallow buffers, deep buffers, bufferbloat, policers, or - AQM schemes that do not provide a delay signal. It requires the fq - ("Fair Queue") pacing packet scheduler. + BBR (Bottleneck Bandwidth and RTT) TCP congestion control aims to + maximize network utilization and minimize queues. It builds an explicit + model of the bottleneck delivery rate and path round-trip propagation + delay. It tolerates packet loss and delay unrelated to congestion. It + can operate over LAN, WAN, cellular, wifi, or cable modem links. It can + coexist with flows that use loss-based congestion control, and can + operate with shallow buffers, deep buffers, bufferbloat, policers, or + AQM schemes that do not provide a delay signal. It requires the fq + ("Fair Queue") pacing packet scheduler. + +config TCP_CONG_BBR3 + tristate "BBR3 TCP" + default n + help + + BBR (Bottleneck Bandwidth and RTT) TCP congestion control is a + model-based congestion control algorithm that aims to maximize + network utilization, keep queues and retransmit rates low, and to be + able to coexist with Reno/CUBIC in common scenarios. It builds an + explicit model of the network path. It tolerates a targeted degree + of random packet loss and delay. It can operate over LAN, WAN, + cellular, wifi, or cable modem links, and can use shallow-threshold + ECN signals. It can coexist to some degree with flows that use + loss-based congestion control, and can operate with shallow buffers, + deep buffers, bufferbloat, policers, or AQM schemes that do not + provide a delay signal. It requires pacing, using either TCP internal + pacing or the fq ("Fair Queue") pacing packet scheduler. choice prompt "Default TCP congestion control" @@ -715,6 +733,9 @@ config DEFAULT_BBR bool "BBR" if TCP_CONG_BBR=y + config DEFAULT_BBR3 + bool "BBR3" if TCP_CONG_BBR3=y + config DEFAULT_RENO bool "Reno" endchoice @@ -739,6 +760,7 @@ default "dctcp" if DEFAULT_DCTCP default "cdg" if DEFAULT_CDG default "bbr" if DEFAULT_BBR + default "bbr3" if DEFAULT_BBR3 default "cubic" config TCP_AO diff -ruN --no-dereference -x .git a/net/ipv4/Makefile b/net/ipv4/Makefile --- a/net/ipv4/Makefile 2026-10-11 15:27:50.000000000 +0200 +++ b/net/ipv4/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -45,6 +45,7 @@ obj-$(CONFIG_INET_UDP_DIAG) += udp_diag.o obj-$(CONFIG_INET_RAW_DIAG) += raw_diag.o obj-$(CONFIG_TCP_CONG_BBR) += tcp_bbr.o +obj-$(CONFIG_TCP_CONG_BBR3) += tcp_bbr3.o obj-$(CONFIG_TCP_CONG_BIC) += tcp_bic.o obj-$(CONFIG_TCP_CONG_CDG) += tcp_cdg.o obj-$(CONFIG_TCP_CONG_CUBIC) += tcp_cubic.o diff -ruN --no-dereference -x .git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c --- a/net/ipv4/bpf_tcp_ca.c 2026-10-11 15:27:50.000000000 +0200 +++ b/net/ipv4/bpf_tcp_ca.c 2026-10-13 14:33:30.000000000 +0200 @@ -284,7 +284,7 @@ { } -static u32 bpf_tcp_ca_min_tso_segs(struct sock *sk) +static u32 bpf_tcp_ca_tso_segs(struct sock *sk, unsigned int mss_now) { return 0; } @@ -320,7 +320,7 @@ .cwnd_event_tx_start = bpf_tcp_ca_cwnd_event_tx_start, .in_ack_event = bpf_tcp_ca_in_ack_event, .pkts_acked = bpf_tcp_ca_pkts_acked, - .min_tso_segs = bpf_tcp_ca_min_tso_segs, + .tso_segs = bpf_tcp_ca_tso_segs, .cong_control = bpf_tcp_ca_cong_control, .undo_cwnd = bpf_tcp_ca_undo_cwnd, .sndbuf_expand = bpf_tcp_ca_sndbuf_expand, diff -ruN --no-dereference -x .git a/net/ipv4/tcp.c b/net/ipv4/tcp.c --- a/net/ipv4/tcp.c 2026-10-11 15:27:50.000000000 +0200 +++ b/net/ipv4/tcp.c 2026-10-13 14:33:30.000000000 +0200 @@ -3482,6 +3482,7 @@ tp->rx_opt.dsack = 0; tp->rx_opt.num_sacks = 0; tp->rcv_ooopack = 0; + tp->fast_ack_mode = 0; /* Clean up fastopen related fields */ @@ -4267,6 +4268,8 @@ info->tcpi_options |= TCPI_OPT_ECN; if (tp->ecn_flags & TCP_ECN_SEEN) info->tcpi_options |= TCPI_OPT_ECN_SEEN; + if (tp->ecn_flags & TCP_ECN_LOW) + info->tcpi_options |= TCPI_OPT_ECN_LOW; if (tp->syn_data_acked) info->tcpi_options |= TCPI_OPT_SYN_DATA; if (tp->tcp_usec_ts) diff -ruN --no-dereference -x .git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c --- a/net/ipv4/tcp_bbr.c 2026-10-11 15:27:50.000000000 +0200 +++ b/net/ipv4/tcp_bbr.c 2026-10-13 14:33:30.000000000 +0200 @@ -302,6 +302,23 @@ return READ_ONCE(sk->sk_pacing_rate) < (bbr_min_tso_rate >> 3) ? 1 : 2; } +__bpf_kfunc static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) +{ + unsigned long bytes; + u32 r; + + bytes = READ_ONCE(sk->sk_pacing_rate) >> READ_ONCE(sk->sk_pacing_shift); + + r = tcp_min_rtt(tcp_sk(sk)) >> + READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tso_rtt_log); + if (r < BITS_PER_TYPE(sk->sk_gso_max_size)) + bytes += sk->sk_gso_max_size >> r; + + bytes = min_t(unsigned long, bytes, sk->sk_gso_max_size); + + return max_t(u32, bytes / mss_now, bbr_min_tso_segs(sk)); +} + static u32 bbr_tso_segs_goal(struct sock *sk) { struct tcp_sock *tp = tcp_sk(sk); @@ -1151,7 +1168,7 @@ .undo_cwnd = bbr_undo_cwnd, .cwnd_event_tx_start = bbr_cwnd_event_tx_start, .ssthresh = bbr_ssthresh, - .min_tso_segs = bbr_min_tso_segs, + .tso_segs = bbr_tso_segs, .get_info = bbr_get_info, .set_state = bbr_set_state, }; diff -ruN --no-dereference -x .git a/net/ipv4/tcp_bbr3.c b/net/ipv4/tcp_bbr3.c --- a/net/ipv4/tcp_bbr3.c 1970-01-01 01:00:00.000000000 +0100 +++ b/net/ipv4/tcp_bbr3.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,2418 @@ +// SPDX-License-Identifier: GPL-2.0 +/* BBR (Bottleneck Bandwidth and RTT) congestion control + * + * BBR is a model-based congestion control algorithm that aims for low queues, + * low loss, and (bounded) Reno/CUBIC coexistence. To maintain a model of the + * network path, it uses measurements of bandwidth and RTT, as well as (if they + * occur) packet loss and/or shallow-threshold ECN signals. Note that although + * it can use ECN or loss signals explicitly, it does not require either; it + * can bound its in-flight data based on its estimate of the BDP. + * + * The model has both higher and lower bounds for the operating range: + * lo: bw_lo, inflight_lo: conservative short-term lower bound + * hi: bw_hi, inflight_hi: robust long-term upper bound + * The bandwidth-probing time scale is (a) extended dynamically based on + * estimated BDP to improve coexistence with Reno/CUBIC; (b) bounded by + * an interactive wall-clock time-scale to be more scalable and responsive + * than Reno and CUBIC. + * + * Here is a state transition diagram for BBR: + * + * | + * V + * +---> STARTUP ----+ + * | | | + * | V | + * | DRAIN ----+ + * | | | + * | V | + * +---> PROBE_BW ----+ + * | ^ | | + * | | | | + * | +----+ | + * | | + * +---- PROBE_RTT <--+ + * + * A BBR flow starts in STARTUP, and ramps up its sending rate quickly. + * When it estimates the pipe is full, it enters DRAIN to drain the queue. + * In steady state a BBR flow only uses PROBE_BW and PROBE_RTT. + * A long-lived BBR flow spends the vast majority of its time remaining + * (repeatedly) in PROBE_BW, fully probing and utilizing the pipe's bandwidth + * in a fair manner, with a small, bounded queue. *If* a flow has been + * continuously sending for the entire min_rtt window, and hasn't seen an RTT + * sample that matches or decreases its min_rtt estimate for 10 seconds, then + * it briefly enters PROBE_RTT to cut inflight to a minimum value to re-probe + * the path's two-way propagation delay (min_rtt). When exiting PROBE_RTT, if + * we estimated that we reached the full bw of the pipe then we enter PROBE_BW; + * otherwise we enter STARTUP to try to fill the pipe. + * + * BBR is described in detail in: + * "BBR: Congestion-Based Congestion Control", + * Neal Cardwell, Yuchung Cheng, C. Stephen Gunn, Soheil Hassas Yeganeh, + * Van Jacobson. ACM Queue, Vol. 14 No. 5, September-October 2016. + * + * There is a public e-mail list for discussing BBR development and testing: + * https://groups.google.com/forum/#!forum/bbr-dev + * + * NOTE: BBR might be used with the fq qdisc ("man tc-fq") with pacing enabled, + * otherwise TCP stack falls back to an internal pacing using one high + * resolution timer per TCP socket and may use more resources. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "tcp_dctcp.h" + +#define BBR_VERSION 3 + +#define bbr_param(sk,name) (bbr_ ## name) + +/* Scale factor for rate in pkt/uSec unit to avoid truncation in bandwidth + * estimation. The rate unit ~= (1500 bytes / 1 usec / 2^24) ~= 715 bps. + * This handles bandwidths from 0.06pps (715bps) to 256Mpps (3Tbps) in a u32. + * Since the minimum window is >=4 packets, the lower bound isn't + * an issue. The upper bound isn't an issue with existing technologies. + */ +#define BW_SCALE 24 +#define BW_UNIT (1 << BW_SCALE) + +#define BBR_SCALE 8 /* scaling factor for fractions in BBR (e.g. gains) */ +#define BBR_UNIT (1 << BBR_SCALE) + +/* BBR has the following modes for deciding how fast to send: */ +enum bbr_mode { + BBR_STARTUP, /* ramp up sending rate rapidly to fill pipe */ + BBR_DRAIN, /* drain any queue created during startup */ + BBR_PROBE_BW, /* discover, share bw: pace around estimated bw */ + BBR_PROBE_RTT, /* cut inflight to min to probe min_rtt */ +}; + +/* How does the incoming ACK stream relate to our bandwidth probing? */ +enum bbr_ack_phase { + BBR_ACKS_INIT, /* not probing; not getting probe feedback */ + BBR_ACKS_REFILLING, /* sending at est. bw to fill pipe */ + BBR_ACKS_PROBE_STARTING, /* inflight rising to probe bw */ + BBR_ACKS_PROBE_FEEDBACK, /* getting feedback from bw probing */ + BBR_ACKS_PROBE_STOPPING, /* stopped probing; still getting feedback */ +}; + +/* BBR congestion control block */ +struct bbr { + u32 min_rtt_us; /* min RTT in min_rtt_win_sec window */ + u32 min_rtt_stamp; /* timestamp of min_rtt_us */ + u32 probe_rtt_done_stamp; /* end time for BBR_PROBE_RTT mode */ + u32 probe_rtt_min_us; /* min RTT in probe_rtt_win_ms win */ + u32 probe_rtt_min_stamp; /* timestamp of probe_rtt_min_us*/ + u32 next_rtt_delivered; /* scb->tx.delivered at end of round */ + u64 cycle_mstamp; /* time of this cycle phase start */ + u32 mode:2, /* current bbr_mode in state machine */ + prev_ca_state:3, /* CA state on previous ACK */ + round_start:1, /* start of packet-timed tx->ack round? */ + ce_state:1, /* If most recent data has CE bit set */ + bw_probe_up_rounds:5, /* cwnd-limited rounds in PROBE_UP */ + try_fast_path:1, /* can we take fast path? */ + idle_restart:1, /* restarting after idle? */ + probe_rtt_round_done:1, /* a BBR_PROBE_RTT round at 4 pkts? */ + init_cwnd:7, /* initial cwnd */ + unused_1:10; + u32 pacing_gain:10, /* current gain for setting pacing rate */ + cwnd_gain:10, /* current gain for setting cwnd */ + full_bw_reached:1, /* reached full bw in Startup? */ + full_bw_cnt:2, /* number of rounds without large bw gains */ + cycle_idx:2, /* current index in pacing_gain cycle array */ + has_seen_rtt:1, /* have we seen an RTT sample yet? */ + unused_2:6; + u32 prior_cwnd; /* prior cwnd upon entering loss recovery */ + u32 full_bw; /* recent bw, to estimate if pipe is full */ + + /* For tracking ACK aggregation: */ + u64 ack_epoch_mstamp; /* start of ACK sampling epoch */ + u16 extra_acked[2]; /* max excess data ACKed in epoch */ + u32 ack_epoch_acked:20, /* packets (S)ACKed in sampling epoch */ + extra_acked_win_rtts:5, /* age of extra_acked, in round trips */ + extra_acked_win_idx:1, /* current index in extra_acked array */ + /* BBR v3 state: */ + full_bw_now:1, /* recently reached full bw plateau? */ + startup_ecn_rounds:2, /* consecutive hi ECN STARTUP rounds */ + loss_in_cycle:1, /* packet loss in this cycle? */ + ecn_in_cycle:1, /* ECN in this cycle? */ + unused_3:1; + u32 loss_round_delivered; /* scb->tx.delivered ending loss round */ + u32 undo_bw_lo; /* bw_lo before latest losses */ + u32 undo_inflight_lo; /* inflight_lo before latest losses */ + u32 undo_inflight_hi; /* inflight_hi before latest losses */ + u32 bw_latest; /* max delivered bw in last round trip */ + u32 bw_lo; /* lower bound on sending bandwidth */ + u32 bw_hi[2]; /* max recent measured bw sample */ + u32 inflight_latest; /* max delivered data in last round trip */ + u32 inflight_lo; /* lower bound of inflight data range */ + u32 inflight_hi; /* upper bound of inflight data range */ + u32 bw_probe_up_cnt; /* packets delivered per inflight_hi incr */ + u32 bw_probe_up_acks; /* packets (S)ACKed since inflight_hi incr */ + u32 probe_wait_us; /* PROBE_DOWN until next clock-driven probe */ + u32 prior_rcv_nxt; /* tp->rcv_nxt when CE state last changed */ + u32 ecn_eligible:1, /* sender can use ECN (RTT, handshake)? */ + ecn_alpha:9, /* EWMA delivered_ce/delivered; 0..256 */ + bw_probe_samples:1, /* rate samples reflect bw probing? */ + prev_probe_too_high:1, /* did last PROBE_UP go too high? */ + stopped_risky_probe:1, /* last PROBE_UP stopped due to risk? */ + rounds_since_probe:8, /* packet-timed rounds since probed bw */ + loss_round_start:1, /* loss_round_delivered round trip? */ + loss_in_round:1, /* loss marked in this round trip? */ + ecn_in_round:1, /* ECN marked in this round trip? */ + ack_phase:3, /* bbr_ack_phase: meaning of ACKs */ + loss_events_in_round:4,/* losses in STARTUP round */ + initialized:1; /* has bbr_init() been called? */ + u32 alpha_last_delivered; /* tp->delivered at alpha update */ + u32 alpha_last_delivered_ce; /* tp->delivered_ce at alpha update */ + + u8 unused_4; /* to preserve alignment */ + struct tcp_plb_state plb; +}; + +struct bbr_context { + u32 sample_bw; +}; + +/* Window length of min_rtt filter (in sec): */ +static const u32 bbr_min_rtt_win_sec = 10; +/* Minimum time (in ms) spent at bbr_cwnd_min_target in BBR_PROBE_RTT mode: */ +static const u32 bbr_probe_rtt_mode_ms = 200; +/* Window length of probe_rtt_min_us filter (in ms), and consequently the + * typical interval between PROBE_RTT mode entries. The default is 5000ms. + * Note that bbr_probe_rtt_win_ms must be <= bbr_min_rtt_win_sec * MSEC_PER_SEC + */ +static const u32 bbr_probe_rtt_win_ms = 5000; +/* Proportion of cwnd to estimated BDP in PROBE_RTT, in units of BBR_UNIT: */ +static const u32 bbr_probe_rtt_cwnd_gain = BBR_UNIT * 1 / 2; + +/* Use min_rtt to help adapt TSO burst size, with smaller min_rtt resulting + * in bigger TSO bursts. We cut the RTT-based allowance in half + * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance + * is below 1500 bytes after 6 * ~500 usec = 3ms. + */ +static const u32 bbr_tso_rtt_shift = 9; + +/* Pace at ~1% below estimated bw, on average, to reduce queue at bottleneck. + * In order to help drive the network toward lower queues and low latency while + * maintaining high utilization, the average pacing rate aims to be slightly + * lower than the estimated bandwidth. This is an important aspect of the + * design. + */ +static const int bbr_pacing_margin_percent = 1; + +/* We use a startup_pacing_gain of 4*ln(2) because it's the smallest value + * that will allow a smoothly increasing pacing rate that will double each RTT + * and send the same number of packets per RTT that an un-paced, slow-starting + * Reno or CUBIC flow would: + */ +static const int bbr_startup_pacing_gain = BBR_UNIT * 277 / 100 + 1; +/* The gain for deriving startup cwnd: */ +static const int bbr_startup_cwnd_gain = BBR_UNIT * 2; +/* The pacing gain in BBR_DRAIN is calculated to typically drain + * the queue created in BBR_STARTUP in a single round: + */ +static const int bbr_drain_gain = BBR_UNIT * 1000 / 2885; +/* The gain for deriving steady-state cwnd tolerates delayed/stretched ACKs: */ +static const int bbr_cwnd_gain = BBR_UNIT * 2; +/* The pacing_gain values for the PROBE_BW gain cycle, to discover/share bw: */ +static const int bbr_pacing_gain[] = { + BBR_UNIT * 5 / 4, /* UP: probe for more available bw */ + BBR_UNIT * 91 / 100, /* DOWN: drain queue and/or yield bw */ + BBR_UNIT, /* CRUISE: try to use pipe w/ some headroom */ + BBR_UNIT, /* REFILL: refill pipe to estimated 100% */ +}; +enum bbr_pacing_gain_phase { + BBR_BW_PROBE_UP = 0, /* push up inflight to probe for bw/vol */ + BBR_BW_PROBE_DOWN = 1, /* drain excess inflight from the queue */ + BBR_BW_PROBE_CRUISE = 2, /* use pipe, w/ headroom in queue/pipe */ + BBR_BW_PROBE_REFILL = 3, /* refill the pipe again to 100% */ +}; + +/* Try to keep at least this many packets in flight, if things go smoothly. For + * smooth functioning, a sliding window protocol ACKing every other packet + * needs at least 4 packets in flight: + */ +static const u32 bbr_cwnd_min_target = 4; + +/* To estimate if BBR_STARTUP or BBR_BW_PROBE_UP has filled pipe... */ +/* If bw has increased significantly (1.25x), there may be more bw available: */ +static const u32 bbr_full_bw_thresh = BBR_UNIT * 5 / 4; +/* But after 3 rounds w/o significant bw growth, estimate pipe is full: */ +static const u32 bbr_full_bw_cnt = 3; + +/* Gain factor for adding extra_acked to target cwnd: */ +static const int bbr_extra_acked_gain = BBR_UNIT; +/* Window length of extra_acked window. */ +static const u32 bbr_extra_acked_win_rtts = 5; +/* Max allowed val for ack_epoch_acked, after which sampling epoch is reset */ +static const u32 bbr_ack_epoch_acked_reset_thresh = 1U << 20; +/* Time period for clamping cwnd increment due to ack aggregation */ +static const u32 bbr_extra_acked_max_us = 100 * 1000; + +/* Flags to control BBR ECN-related behavior... */ + +/* Ensure ACKs only ACK packets with consistent ECN CE status? */ +static const bool bbr_precise_ece_ack = true; + +/* Max RTT (in usec) at which to use sender-side ECN logic. + * Disabled when 0 (ECN allowed at any RTT). + */ +static const u32 bbr_ecn_max_rtt_us = 5000; + +/* On losses, scale down inflight and pacing rate by beta scaled by BBR_SCALE. + * No loss response when 0. + */ +static const u32 bbr_beta = BBR_UNIT * 30 / 100; + +/* Gain factor for ECN mark ratio samples, scaled by BBR_SCALE (1/16 = 6.25%) */ +static const u32 bbr_ecn_alpha_gain = BBR_UNIT * 1 / 16; + +/* The initial value for ecn_alpha; 1.0 allows a flow to respond quickly + * to congestion if the bottleneck is congested when the flow starts up. + */ +static const u32 bbr_ecn_alpha_init = BBR_UNIT; + +/* On ECN, cut inflight_lo to (1 - ecn_factor * ecn_alpha) scaled by BBR_SCALE. + * No ECN based bounding when 0. + */ +static const u32 bbr_ecn_factor = BBR_UNIT * 1 / 3; /* 1/3 = 33% */ + +/* Estimate bw probing has gone too far if CE ratio exceeds this threshold. + * Scaled by BBR_SCALE. Disabled when 0. + */ +static const u32 bbr_ecn_thresh = BBR_UNIT * 1 / 2; /* 1/2 = 50% */ + +/* If non-zero, if in a cycle with no losses but some ECN marks, after ECN + * clears then make the first round's increment to inflight_hi the following + * fraction of inflight_hi. + */ +static const u32 bbr_ecn_reprobe_gain = BBR_UNIT * 1 / 2; + +/* Estimate bw probing has gone too far if loss rate exceeds this level. */ +static const u32 bbr_loss_thresh = BBR_UNIT * 2 / 100; /* 2% loss */ + +/* Slow down for a packet loss recovered by TLP? */ +static const bool bbr_loss_probe_recovery = true; + +/* Exit STARTUP if number of loss marking events in a Recovery round is >= N, + * and loss rate is higher than bbr_loss_thresh. + * Disabled if 0. + */ +static const u32 bbr_full_loss_cnt = 6; + +/* Exit STARTUP if number of round trips with ECN mark rate above ecn_thresh + * meets this count. + */ +static const u32 bbr_full_ecn_cnt = 2; + +/* Fraction of unutilized headroom to try to leave in path upon high loss. */ +static const u32 bbr_inflight_headroom = BBR_UNIT * 15 / 100; + +/* How much do we increase cwnd_gain when probing for bandwidth in + * BBR_BW_PROBE_UP? This specifies the increment in units of + * BBR_UNIT/4. The default is 1, meaning 0.25. + * The min value is 0 (meaning 0.0); max is 3 (meaning 0.75). + */ +static const u32 bbr_bw_probe_cwnd_gain = 1; + +/* Max number of packet-timed rounds to wait before probing for bandwidth. If + * we want to tolerate 1% random loss per round, and not have this cut our + * inflight too much, we must probe for bw periodically on roughly this scale. + * If low, limits Reno/CUBIC coexistence; if high, limits loss tolerance. + * We aim to be fair with Reno/CUBIC up to a BDP of at least: + * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets + */ +static const u32 bbr_bw_probe_max_rounds = 63; + +/* Max amount of randomness to inject in round counting for Reno-coexistence. + */ +static const u32 bbr_bw_probe_rand_rounds = 2; + +/* Use BBR-native probe time scale starting at this many usec. + * We aim to be fair with Reno/CUBIC up to an inter-loss time epoch of at least: + * BDP*RTT = 25Mbps * .030sec /(1514bytes) * 0.030sec = 1.9 secs + */ +static const u32 bbr_bw_probe_base_us = 2 * USEC_PER_SEC; /* 2 secs */ + +/* Use BBR-native probes spread over this many usec: */ +static const u32 bbr_bw_probe_rand_us = 1 * USEC_PER_SEC; /* 1 secs */ + +/* Use fast path if app-limited, no loss/ECN, and target cwnd was reached? */ +static const bool bbr_fast_path = true; + +/* Use fast ack mode? */ +static const bool bbr_fast_ack_mode = true; + +static u32 bbr_max_bw(const struct sock *sk); +static u32 bbr_bw(const struct sock *sk); +static void bbr_exit_probe_rtt(struct sock *sk); +static void bbr_reset_congestion_signals(struct sock *sk); +static void bbr_run_loss_probe_recovery(struct sock *sk); + +static void bbr_check_probe_rtt_done(struct sock *sk); + +/* This connection can use ECN if both endpoints have signaled ECN support in + * the handshake and the per-route settings indicated this is a + * shallow-threshold ECN environment, meaning both: + * (a) ECN CE marks indicate low-latency/shallow-threshold congestion, and + * (b) TCP endpoints provide precise ACKs that only ACK data segments + * with consistent ECN CE status + */ +static bool bbr_can_use_ecn(const struct sock *sk) +{ + const struct tcp_sock *tp = tcp_sk(sk); + + return tcp_ecn_mode_any(tp) && + (tcp_sk(sk)->ecn_flags & TCP_ECN_LOW); +} + +/* Do we estimate that STARTUP filled the pipe? */ +static bool bbr_full_bw_reached(const struct sock *sk) +{ + const struct bbr *bbr = inet_csk_ca(sk); + + return bbr->full_bw_reached; +} + +/* Return the windowed max recent bandwidth sample, in pkts/uS << BW_SCALE. */ +static u32 bbr_max_bw(const struct sock *sk) +{ + const struct bbr *bbr = inet_csk_ca(sk); + + return max(bbr->bw_hi[0], bbr->bw_hi[1]); +} + +/* Return the estimated bandwidth of the path, in pkts/uS << BW_SCALE. */ +static u32 bbr_bw(const struct sock *sk) +{ + const struct bbr *bbr = inet_csk_ca(sk); + + return min(bbr_max_bw(sk), bbr->bw_lo); +} + +/* Return maximum extra acked in past k-2k round trips, + * where k = bbr_extra_acked_win_rtts. + */ +static u16 bbr_extra_acked(const struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + return max(bbr->extra_acked[0], bbr->extra_acked[1]); +} + +/* Return rate in bytes per second, optionally with a gain. + * The order here is chosen carefully to avoid overflow of u64. This should + * work for input rates of up to 2.9Tbit/sec and gain of 2.89x. + */ +static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain, + int margin) +{ + unsigned int mss = tcp_sk(sk)->mss_cache; + + rate *= mss; + rate *= gain; + rate >>= BBR_SCALE; + rate *= USEC_PER_SEC / 100 * (100 - margin); + rate >>= BW_SCALE; + rate = max(rate, 1ULL); + return rate; +} + +static u64 bbr_bw_bytes_per_sec(struct sock *sk, u64 rate) +{ + return bbr_rate_bytes_per_sec(sk, rate, BBR_UNIT, 0); +} + +/* Convert a BBR bw and gain factor to a pacing rate in bytes per second. */ +static unsigned long bbr_bw_to_pacing_rate(struct sock *sk, u32 bw, int gain) +{ + u64 rate = bw; + + rate = bbr_rate_bytes_per_sec(sk, rate, gain, + bbr_pacing_margin_percent); + rate = min_t(u64, rate, READ_ONCE(sk->sk_max_pacing_rate)); + return rate; +} + +/* Initialize pacing rate to: startup_pacing_gain * init_cwnd / RTT. */ +static void bbr_init_pacing_rate_from_rtt(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u64 bw; + u32 rtt_us; + + if (tp->srtt_us) { /* any RTT sample yet? */ + rtt_us = max(tp->srtt_us >> 3, 1U); + bbr->has_seen_rtt = 1; + } else { /* no RTT sample yet */ + rtt_us = USEC_PER_MSEC; /* use nominal default RTT */ + } + bw = (u64)tcp_snd_cwnd(tp) * BW_UNIT; + do_div(bw, rtt_us); + WRITE_ONCE(sk->sk_pacing_rate, + bbr_bw_to_pacing_rate(sk, bw, + bbr_param(sk, startup_pacing_gain))); +} + +/* Pace using current bw estimate and a gain factor. */ +static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + unsigned long rate = bbr_bw_to_pacing_rate(sk, bw, gain); + + if (unlikely(!bbr->has_seen_rtt && tp->srtt_us)) + bbr_init_pacing_rate_from_rtt(sk); + if (bbr_full_bw_reached(sk) || rate > READ_ONCE(sk->sk_pacing_rate)) + WRITE_ONCE(sk->sk_pacing_rate, rate); +} + +/* Return the number of segments BBR would like in a TSO/GSO skb, given a + * particular max gso size as a constraint. TODO: make this simpler and more + * consistent by switching bbr to just call tcp_tso_autosize(). + */ +static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now, + u32 gso_max_size) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 segs, r; + u64 bytes; + + /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */ + bytes = READ_ONCE(sk->sk_pacing_rate) >> READ_ONCE(sk->sk_pacing_shift); + + /* Budget a TSO/GSO burst size allowance based on min_rtt. For every + * K = 2^tso_rtt_shift microseconds of min_rtt, halve the burst. + * The min_rtt-based burst allowance is: 64 KBytes / 2^(min_rtt/K) + */ + if (bbr_param(sk, tso_rtt_shift)) { + r = bbr->min_rtt_us >> bbr_param(sk, tso_rtt_shift); + if (r < BITS_PER_TYPE(u32)) /* prevent undefined behavior */ + bytes += GSO_LEGACY_MAX_SIZE >> r; + } + + bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER); + segs = max_t(u32, bytes / mss_now, + sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs); + return segs; +} + +/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */ +__bpf_kfunc static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) +{ + return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size); +} + +/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */ +static u32 bbr_tso_segs_goal(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + + return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_LEGACY_MAX_SIZE); +} + +/* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ +static void bbr_save_cwnd(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->prev_ca_state < TCP_CA_Recovery && bbr->mode != BBR_PROBE_RTT) + bbr->prior_cwnd = tcp_snd_cwnd(tp); /* this cwnd is good enough */ + else /* loss recovery or BBR_PROBE_RTT have temporarily cut cwnd */ + bbr->prior_cwnd = max(bbr->prior_cwnd, tcp_snd_cwnd(tp)); +} + +__bpf_kfunc static void bbr_cwnd_event(struct sock *sk, enum tcp_ca_event event) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if ((event == CA_EVENT_ECN_IS_CE || + event == CA_EVENT_ECN_NO_CE) && + bbr_can_use_ecn(sk) && + bbr_param(sk, precise_ece_ack)) { + u32 state = bbr->ce_state; + dctcp_ece_ack_update(sk, event, &bbr->prior_rcv_nxt, &state); + bbr->ce_state = state; + } else if (event == CA_EVENT_TLP_RECOVERY && + bbr_param(sk, loss_probe_recovery)) { + bbr_run_loss_probe_recovery(sk); + } +} + +__bpf_kfunc static void bbr_cwnd_event_tx_start(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + if (tp->app_limited) { + bbr->idle_restart = 1; + bbr->ack_epoch_mstamp = tp->tcp_mstamp; + bbr->ack_epoch_acked = 0; + /* Avoid pointless buffer overflows: pace at est. bw if we don't + * need more speed (we're restarting from idle and app-limited). + */ + if (bbr->mode == BBR_PROBE_BW) + bbr_set_pacing_rate(sk, bbr_bw(sk), BBR_UNIT); + else if (bbr->mode == BBR_PROBE_RTT) + bbr_check_probe_rtt_done(sk); + } +} + +/* Calculate bdp based on min RTT and the estimated bottleneck bandwidth: + * + * bdp = ceil(bw * min_rtt * gain) + * + * The key factor, gain, controls the amount of queue. While a small gain + * builds a smaller queue, it becomes more vulnerable to noise in RTT + * measurements (e.g., delayed ACKs or other ACK compression effects). This + * noise may cause BBR to under-estimate the rate. + */ +static u32 bbr_bdp(struct sock *sk, u32 bw, int gain) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 bdp; + u64 w; + + /* If we've never had a valid RTT sample, cap cwnd at the initial + * default. This should only happen when the connection is not using TCP + * timestamps and has retransmitted all of the SYN/SYNACK/data packets + * ACKed so far. In this case, an RTO can cut cwnd to 1, in which + * case we need to slow-start up toward something safe: initial cwnd. + */ + if (unlikely(bbr->min_rtt_us == ~0U)) /* no valid RTT samples yet? */ + return bbr->init_cwnd; /* be safe: cap at initial cwnd */ + + w = (u64)bw * bbr->min_rtt_us; + + /* Apply a gain to the given value, remove the BW_SCALE shift, and + * round the value up to avoid a negative feedback loop. + */ + bdp = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; + + return bdp; +} + +/* To achieve full performance in high-speed paths, we budget enough cwnd to + * fit full-sized skbs in-flight on both end hosts to fully utilize the path: + * - one skb in sending host Qdisc, + * - one skb in sending host TSO/GSO engine + * - one skb being received by receiver host LRO/GRO/delayed-ACK engine + * Don't worry, at low rates this won't bloat cwnd because + * in such cases tso_segs_goal is small. The minimum cwnd is 4 packets, + * which allows 2 outstanding 2-packet sequences, to try to keep pipe + * full even with ACK-every-other-packet delayed ACKs. + */ +static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 tso_segs_goal; + + tso_segs_goal = 3 * bbr_tso_segs_goal(sk); + + /* Allow enough full-sized skbs in flight to utilize end systems. */ + cwnd = max_t(u32, cwnd, tso_segs_goal); + cwnd = max_t(u32, cwnd, bbr_param(sk, cwnd_min_target)); + /* Ensure gain cycling gets inflight above BDP even for small BDPs. */ + if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) + cwnd += 2; + + return cwnd; +} + +/* Find inflight based on min RTT and the estimated bottleneck bandwidth. */ +static u32 bbr_inflight(struct sock *sk, u32 bw, int gain) +{ + u32 inflight; + + inflight = bbr_bdp(sk, bw, gain); + inflight = bbr_quantization_budget(sk, inflight); + + return inflight; +} + +/* With pacing at lower layers, there's often less data "in the network" than + * "in flight". With TSQ and departure time pacing at lower layers (e.g. fq), + * we often have several skbs queued in the pacing layer with a pre-scheduled + * earliest departure time (EDT). BBR adapts its pacing rate based on the + * inflight level that it estimates has already been "baked in" by previous + * departure time decisions. We calculate a rough estimate of the number of our + * packets that might be in the network at the earliest departure time for the + * next skb scheduled: + * in_network_at_edt = inflight_at_edt - (EDT - now) * bw + * If we're increasing inflight, then we want to know if the transmit of the + * EDT skb will push inflight above the target, so inflight_at_edt includes + * bbr_tso_segs_goal() from the skb departing at EDT. If decreasing inflight, + * then estimate if inflight will sink too low just before the EDT transmit. + */ +static u32 bbr_packets_in_net_at_edt(struct sock *sk, u32 inflight_now) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u64 now_ns, edt_ns, interval_us; + u32 interval_delivered, inflight_at_edt; + + now_ns = tp->tcp_clock_cache; + edt_ns = max(tp->tcp_wstamp_ns, now_ns); + interval_us = div_u64(edt_ns - now_ns, NSEC_PER_USEC); + interval_delivered = (u64)bbr_bw(sk) * interval_us >> BW_SCALE; + inflight_at_edt = inflight_now; + if (bbr->pacing_gain > BBR_UNIT) /* increasing inflight */ + inflight_at_edt += bbr_tso_segs_goal(sk); /* include EDT skb */ + if (interval_delivered >= inflight_at_edt) + return 0; + return inflight_at_edt - interval_delivered; +} + +/* Find the cwnd increment based on estimate of ack aggregation */ +static u32 bbr_ack_aggregation_cwnd(struct sock *sk) +{ + u32 max_aggr_cwnd, aggr_cwnd = 0; + + if (bbr_param(sk, extra_acked_gain)) { + max_aggr_cwnd = ((u64)bbr_bw(sk) * bbr_extra_acked_max_us) + / BW_UNIT; + aggr_cwnd = (bbr_param(sk, extra_acked_gain) * bbr_extra_acked(sk)) + >> BBR_SCALE; + aggr_cwnd = min(aggr_cwnd, max_aggr_cwnd); + } + + return aggr_cwnd; +} + +/* Returns the cwnd for PROBE_RTT mode. */ +static u32 bbr_probe_rtt_cwnd(struct sock *sk) +{ + return max_t(u32, bbr_param(sk, cwnd_min_target), + bbr_bdp(sk, bbr_bw(sk), bbr_param(sk, probe_rtt_cwnd_gain))); +} + +/* Slow-start up toward target cwnd (if bw estimate is growing, or packet loss + * has drawn us down below target), or snap down to target if we're above it. + */ +static void bbr_set_cwnd(struct sock *sk, const struct rate_sample *rs, + u32 acked, u32 bw, int gain, u32 cwnd, + struct bbr_context *ctx) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 target_cwnd = 0; + + if (!acked) + goto done; /* no packet fully ACKed; just apply caps */ + + target_cwnd = bbr_bdp(sk, bw, gain); + + /* Increment the cwnd to account for excess ACKed data that seems + * due to aggregation (of data and/or ACKs) visible in the ACK stream. + */ + target_cwnd += bbr_ack_aggregation_cwnd(sk); + target_cwnd = bbr_quantization_budget(sk, target_cwnd); + + /* Update cwnd and enable fast path if cwnd reaches target_cwnd. */ + bbr->try_fast_path = 0; + if (bbr_full_bw_reached(sk)) { /* only cut cwnd if we filled the pipe */ + cwnd += acked; + if (cwnd >= target_cwnd) { + cwnd = target_cwnd; + bbr->try_fast_path = 1; + } + } else if (cwnd < target_cwnd || cwnd < 2 * bbr->init_cwnd) { + cwnd += acked; + } else { + bbr->try_fast_path = 1; + } + + cwnd = max_t(u32, cwnd, bbr_param(sk, cwnd_min_target)); +done: + tcp_snd_cwnd_set(tp, min(cwnd, tp->snd_cwnd_clamp)); /* global cap */ + if (bbr->mode == BBR_PROBE_RTT) /* drain queue, refresh min_rtt */ + tcp_snd_cwnd_set(tp, min_t(u32, tcp_snd_cwnd(tp), + bbr_probe_rtt_cwnd(sk))); +} + +static void bbr_reset_startup_mode(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->mode = BBR_STARTUP; +} + +/* See if we have reached next round trip. Upon start of the new round, + * returns packets delivered since previous round start plus this ACK. + */ +static u32 bbr_update_round_start(struct sock *sk, + const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 round_delivered = 0; + + bbr->round_start = 0; + + /* See if we've reached the next RTT */ + if (rs->interval_us > 0 && + !before(rs->prior_delivered, bbr->next_rtt_delivered)) { + round_delivered = tp->delivered - bbr->next_rtt_delivered; + bbr->next_rtt_delivered = tp->delivered; + bbr->round_start = 1; + } + return round_delivered; +} + +/* Calculate the bandwidth based on how fast packets are delivered */ +static void bbr_calculate_bw_sample(struct sock *sk, + const struct rate_sample *rs, struct bbr_context *ctx) +{ + u64 bw = 0; + + /* Divide delivered by the interval to find a (lower bound) bottleneck + * bandwidth sample. Delivered is in packets and interval_us in uS and + * ratio will be <<1 for most connections. So delivered is first scaled. + * Round up to allow growth at low rates, even with integer division. + */ + if (rs->interval_us > 0) { + if (WARN_ONCE(rs->delivered < 0, + "negative delivered: %d interval_us: %ld\n", + rs->delivered, rs->interval_us)) + return; + + bw = DIV_ROUND_UP_ULL((u64)rs->delivered * BW_UNIT, rs->interval_us); + } + + ctx->sample_bw = bw; +} + +/* Estimates the windowed max degree of ack aggregation. + * This is used to provision extra in-flight data to keep sending during + * inter-ACK silences. + * + * Degree of ack aggregation is estimated as extra data acked beyond expected. + * + * max_extra_acked = "maximum recent excess data ACKed beyond max_bw * interval" + * cwnd += max_extra_acked + * + * Max extra_acked is clamped by cwnd and bw * bbr_extra_acked_max_us (100 ms). + * Max filter is an approximate sliding window of 5-10 (packet timed) round + * trips for non-startup phase, and 1-2 round trips for startup. + */ +static void bbr_update_ack_aggregation(struct sock *sk, + const struct rate_sample *rs) +{ + u32 epoch_us, expected_acked, extra_acked; + struct bbr *bbr = inet_csk_ca(sk); + struct tcp_sock *tp = tcp_sk(sk); + u32 extra_acked_win_rtts_thresh = bbr_param(sk, extra_acked_win_rtts); + + if (!bbr_param(sk, extra_acked_gain) || rs->acked_sacked <= 0 || + rs->delivered < 0 || rs->interval_us <= 0) + return; + + if (bbr->round_start) { + bbr->extra_acked_win_rtts = min(0x1F, + bbr->extra_acked_win_rtts + 1); + if (!bbr_full_bw_reached(sk)) + extra_acked_win_rtts_thresh = 1; + if (bbr->extra_acked_win_rtts >= + extra_acked_win_rtts_thresh) { + bbr->extra_acked_win_rtts = 0; + bbr->extra_acked_win_idx = bbr->extra_acked_win_idx ? + 0 : 1; + bbr->extra_acked[bbr->extra_acked_win_idx] = 0; + } + } + + /* Compute how many packets we expected to be delivered over epoch. */ + epoch_us = tcp_stamp_us_delta(tp->delivered_mstamp, + bbr->ack_epoch_mstamp); + expected_acked = ((u64)bbr_bw(sk) * epoch_us) / BW_UNIT; + + /* Reset the aggregation epoch if ACK rate is below expected rate or + * significantly large no. of ack received since epoch (potentially + * quite old epoch). + */ + if (bbr->ack_epoch_acked <= expected_acked || + (bbr->ack_epoch_acked + rs->acked_sacked >= + bbr_ack_epoch_acked_reset_thresh)) { + bbr->ack_epoch_acked = 0; + bbr->ack_epoch_mstamp = tp->delivered_mstamp; + expected_acked = 0; + } + + /* Compute excess data delivered, beyond what was expected. */ + bbr->ack_epoch_acked = min_t(u32, 0xFFFFF, + bbr->ack_epoch_acked + rs->acked_sacked); + extra_acked = bbr->ack_epoch_acked - expected_acked; + extra_acked = min(extra_acked, tcp_snd_cwnd(tp)); + if (extra_acked > bbr->extra_acked[bbr->extra_acked_win_idx]) + bbr->extra_acked[bbr->extra_acked_win_idx] = extra_acked; +} + +static void bbr_check_probe_rtt_done(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + if (!(bbr->probe_rtt_done_stamp && + after(tcp_jiffies32, bbr->probe_rtt_done_stamp))) + return; + + bbr->probe_rtt_min_stamp = tcp_jiffies32; /* schedule next PROBE_RTT */ + tcp_snd_cwnd_set(tp, max(tcp_snd_cwnd(tp), bbr->prior_cwnd)); + bbr_exit_probe_rtt(sk); +} + +/* The goal of PROBE_RTT mode is to have BBR flows cooperatively and + * periodically drain the bottleneck queue, to converge to measure the true + * min_rtt (unloaded propagation delay). This allows the flows to keep queues + * small (reducing queuing delay and packet loss) and achieve fairness among + * BBR flows. + * + * The min_rtt filter window is 10 seconds. When the min_rtt estimate expires, + * we enter PROBE_RTT mode and cap the cwnd at bbr_cwnd_min_target=4 packets. + * After at least bbr_probe_rtt_mode_ms=200ms and at least one packet-timed + * round trip elapsed with that flight size <= 4, we leave PROBE_RTT mode and + * re-enter the previous mode. BBR uses 200ms to approximately bound the + * performance penalty of PROBE_RTT's cwnd capping to roughly 2% (200ms/10s). + * + * Note that flows need only pay 2% if they are busy sending over the last 10 + * seconds. Interactive applications (e.g., Web, RPCs, video chunks) often have + * natural silences or low-rate periods within 10 seconds where the rate is low + * enough for long enough to drain its queue in the bottleneck. We pick up + * these min RTT measurements opportunistically with our min_rtt filter. :-) + */ +static void bbr_update_min_rtt(struct sock *sk, const struct rate_sample *rs) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + bool probe_rtt_expired, min_rtt_expired; + u32 expire; + + /* Track min RTT in probe_rtt_win_ms to time next PROBE_RTT state. */ + expire = bbr->probe_rtt_min_stamp + + msecs_to_jiffies(bbr_param(sk, probe_rtt_win_ms)); + probe_rtt_expired = after(tcp_jiffies32, expire); + if (rs->rtt_us >= 0 && + (rs->rtt_us < bbr->probe_rtt_min_us || + (probe_rtt_expired && !rs->is_ack_delayed))) { + bbr->probe_rtt_min_us = rs->rtt_us; + bbr->probe_rtt_min_stamp = tcp_jiffies32; + } + /* Track min RTT seen in the min_rtt_win_sec filter window: */ + expire = bbr->min_rtt_stamp + bbr_param(sk, min_rtt_win_sec) * HZ; + min_rtt_expired = after(tcp_jiffies32, expire); + if (bbr->probe_rtt_min_us <= bbr->min_rtt_us || + min_rtt_expired) { + bbr->min_rtt_us = bbr->probe_rtt_min_us; + bbr->min_rtt_stamp = bbr->probe_rtt_min_stamp; + } + + if (bbr_param(sk, probe_rtt_mode_ms) > 0 && probe_rtt_expired && + !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) { + bbr->mode = BBR_PROBE_RTT; /* dip, drain queue */ + bbr_save_cwnd(sk); /* note cwnd so we can restore it */ + bbr->probe_rtt_done_stamp = 0; + bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; + bbr->next_rtt_delivered = tp->delivered; + } + + if (bbr->mode == BBR_PROBE_RTT) { + /* Ignore low rate samples during this mode. */ + tp->app_limited = + (tp->delivered + tcp_packets_in_flight(tp)) ? : 1; + /* Maintain min packets in flight for max(200 ms, 1 round). */ + if (!bbr->probe_rtt_done_stamp && + tcp_packets_in_flight(tp) <= bbr_probe_rtt_cwnd(sk)) { + bbr->probe_rtt_done_stamp = tcp_jiffies32 + + msecs_to_jiffies(bbr_param(sk, probe_rtt_mode_ms)); + bbr->probe_rtt_round_done = 0; + bbr->next_rtt_delivered = tp->delivered; + } else if (bbr->probe_rtt_done_stamp) { + if (bbr->round_start) + bbr->probe_rtt_round_done = 1; + if (bbr->probe_rtt_round_done) + bbr_check_probe_rtt_done(sk); + } + } + /* Restart after idle ends only once we process a new S/ACK for data */ + if (rs->delivered > 0) + bbr->idle_restart = 0; +} + +static void bbr_update_gains(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + switch (bbr->mode) { + case BBR_STARTUP: + bbr->pacing_gain = bbr_param(sk, startup_pacing_gain); + bbr->cwnd_gain = bbr_param(sk, startup_cwnd_gain); + break; + case BBR_DRAIN: + bbr->pacing_gain = bbr_param(sk, drain_gain); /* slow, to drain */ + bbr->cwnd_gain = bbr_param(sk, startup_cwnd_gain); /* keep cwnd */ + break; + case BBR_PROBE_BW: + bbr->pacing_gain = bbr_pacing_gain[bbr->cycle_idx]; + bbr->cwnd_gain = bbr_param(sk, cwnd_gain); + if (bbr_param(sk, bw_probe_cwnd_gain) && + bbr->cycle_idx == BBR_BW_PROBE_UP) + bbr->cwnd_gain += + BBR_UNIT * bbr_param(sk, bw_probe_cwnd_gain) / 4; + break; + case BBR_PROBE_RTT: + bbr->pacing_gain = BBR_UNIT; + bbr->cwnd_gain = BBR_UNIT; + break; + default: + WARN_ONCE(1, "BBR bad mode: %u\n", bbr->mode); + break; + } +} + +__bpf_kfunc static u32 bbr_sndbuf_expand(struct sock *sk) +{ + /* Provision 3 * cwnd since BBR may slow-start even during recovery. */ + return 3; +} + +/* Incorporate a new bw sample into the current window of our max filter. */ +static void bbr_take_max_bw_sample(struct sock *sk, u32 bw) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->bw_hi[1] = max(bw, bbr->bw_hi[1]); +} + +/* Keep max of last 1-2 cycles. Each PROBE_BW cycle, flip filter window. */ +static void bbr_advance_max_bw_filter(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (!bbr->bw_hi[1]) + return; /* no samples in this window; remember old window */ + bbr->bw_hi[0] = bbr->bw_hi[1]; + bbr->bw_hi[1] = 0; +} + +/* Reset the estimator for reaching full bandwidth based on bw plateau. */ +static void bbr_reset_full_bw(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->full_bw = 0; + bbr->full_bw_cnt = 0; + bbr->full_bw_now = 0; +} + +/* How much do we want in flight? Our BDP, unless congestion cut cwnd. */ +static u32 bbr_target_inflight(struct sock *sk) +{ + u32 bdp = bbr_inflight(sk, bbr_bw(sk), BBR_UNIT); + + return min(bdp, tcp_sk(sk)->snd_cwnd); +} + +static bool bbr_is_probing_bandwidth(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + return (bbr->mode == BBR_STARTUP) || + (bbr->mode == BBR_PROBE_BW && + (bbr->cycle_idx == BBR_BW_PROBE_REFILL || + bbr->cycle_idx == BBR_BW_PROBE_UP)); +} + +/* Has the given amount of time elapsed since we marked the phase start? */ +static bool bbr_has_elapsed_in_phase(const struct sock *sk, u32 interval_us) +{ + const struct tcp_sock *tp = tcp_sk(sk); + const struct bbr *bbr = inet_csk_ca(sk); + + return tcp_stamp_us_delta(tp->tcp_mstamp, + bbr->cycle_mstamp + interval_us) > 0; +} + +static void bbr_handle_queue_too_high_in_startup(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 bdp; /* estimated BDP in packets, with quantization budget */ + + bbr->full_bw_reached = 1; + + bdp = bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); + bbr->inflight_hi = max(bdp, bbr->inflight_latest); +} + +/* Exit STARTUP upon N consecutive rounds with ECN mark rate > ecn_thresh. */ +static void bbr_check_ecn_too_high_in_startup(struct sock *sk, u32 ce_ratio) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr_full_bw_reached(sk) || !bbr->ecn_eligible || + !bbr_param(sk, full_ecn_cnt) || !bbr_param(sk, ecn_thresh)) + return; + + if (ce_ratio >= bbr_param(sk, ecn_thresh)) + bbr->startup_ecn_rounds++; + else + bbr->startup_ecn_rounds = 0; + + if (bbr->startup_ecn_rounds >= bbr_param(sk, full_ecn_cnt)) { + bbr_handle_queue_too_high_in_startup(sk); + return; + } +} + +/* Updates ecn_alpha and returns ce_ratio. -1 if not available. */ +static int bbr_update_ecn_alpha(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct net *net = sock_net(sk); + struct bbr *bbr = inet_csk_ca(sk); + s32 delivered, delivered_ce; + u64 alpha, ce_ratio; + u32 gain; + bool want_ecn_alpha; + + /* See if we should use ECN sender logic for this connection. */ + if (!bbr->ecn_eligible && bbr_can_use_ecn(sk) && + !!bbr_param(sk, ecn_factor) && + (bbr->min_rtt_us <= bbr_ecn_max_rtt_us || + !bbr_ecn_max_rtt_us)) + bbr->ecn_eligible = 1; + + /* Skip updating alpha only if not ECN-eligible and PLB is disabled. */ + want_ecn_alpha = (bbr->ecn_eligible || + (bbr_can_use_ecn(sk) && + READ_ONCE(net->ipv4.sysctl_tcp_plb_enabled))); + if (!want_ecn_alpha) + return -1; + + delivered = tp->delivered - bbr->alpha_last_delivered; + delivered_ce = tp->delivered_ce - bbr->alpha_last_delivered_ce; + + if (delivered == 0 || /* avoid divide by zero */ + WARN_ON_ONCE(delivered < 0 || delivered_ce < 0)) /* backwards? */ + return -1; + + BUILD_BUG_ON(BBR_SCALE != TCP_PLB_SCALE); + ce_ratio = (u64)delivered_ce << BBR_SCALE; + do_div(ce_ratio, delivered); + + gain = bbr_param(sk, ecn_alpha_gain); + alpha = ((BBR_UNIT - gain) * bbr->ecn_alpha) >> BBR_SCALE; + alpha += (gain * ce_ratio) >> BBR_SCALE; + bbr->ecn_alpha = min_t(u32, alpha, BBR_UNIT); + + bbr->alpha_last_delivered = tp->delivered; + bbr->alpha_last_delivered_ce = tp->delivered_ce; + + bbr_check_ecn_too_high_in_startup(sk, ce_ratio); + return (int)ce_ratio; +} + +/* Protective Load Balancing (PLB). PLB rehashes outgoing data (to a new IPv6 + * flow label) if it encounters sustained congestion in the form of ECN marks. + */ +static void bbr_plb(struct sock *sk, const struct rate_sample *rs, int ce_ratio) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->round_start && ce_ratio >= 0) + tcp_plb_update_state(sk, &bbr->plb, ce_ratio); + + tcp_plb_check_rehash(sk, &bbr->plb); +} + +/* Each round trip of BBR_BW_PROBE_UP, double volume of probing data. */ +static void bbr_raise_inflight_hi_slope(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 growth_this_round, cnt; + + /* Calculate "slope": packets S/Acked per inflight_hi increment. */ + growth_this_round = 1 << bbr->bw_probe_up_rounds; + bbr->bw_probe_up_rounds = min(bbr->bw_probe_up_rounds + 1, 30); + cnt = tcp_snd_cwnd(tp) / growth_this_round; + cnt = max(cnt, 1U); + bbr->bw_probe_up_cnt = cnt; +} + +/* In BBR_BW_PROBE_UP, not seeing high loss/ECN/queue, so raise inflight_hi. */ +static void bbr_probe_inflight_hi_upward(struct sock *sk, + const struct rate_sample *rs) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 delta; + + if (!tp->is_cwnd_limited || tcp_snd_cwnd(tp) < bbr->inflight_hi) + return; /* not fully using inflight_hi, so don't grow it */ + + /* For each bw_probe_up_cnt packets ACKed, increase inflight_hi by 1. */ + bbr->bw_probe_up_acks += rs->acked_sacked; + if (bbr->bw_probe_up_acks >= bbr->bw_probe_up_cnt) { + delta = bbr->bw_probe_up_acks / bbr->bw_probe_up_cnt; + bbr->bw_probe_up_acks -= delta * bbr->bw_probe_up_cnt; + bbr->inflight_hi += delta; + bbr->try_fast_path = 0; /* Need to update cwnd */ + } + + if (bbr->round_start) + bbr_raise_inflight_hi_slope(sk); +} + +/* Does loss/ECN rate for this sample say inflight is "too high"? + * This is used by both the bbr_check_loss_too_high_in_startup() function, + * and in PROBE_UP. + */ +static bool bbr_is_inflight_too_high(const struct sock *sk, + const struct rate_sample *rs) +{ + const struct bbr *bbr = inet_csk_ca(sk); + u32 loss_thresh, ecn_thresh; + + if (rs->lost > 0 && rs->tx_in_flight) { + loss_thresh = (u64)rs->tx_in_flight * bbr_param(sk, loss_thresh) >> + BBR_SCALE; + if (rs->lost > loss_thresh) { + return true; + } + } + + if (rs->delivered_ce > 0 && rs->delivered > 0 && + bbr->ecn_eligible && !!bbr_param(sk, ecn_thresh)) { + ecn_thresh = (u64)rs->delivered * bbr_param(sk, ecn_thresh) >> + BBR_SCALE; + if (rs->delivered_ce > ecn_thresh) { + return true; + } + } + + return false; +} + +/* Calculate the tx_in_flight level that corresponded to excessive loss. + * We find "lost_prefix" segs of the skb where loss rate went too high, + * by solving for "lost_prefix" in the following equation: + * lost / inflight >= loss_thresh + * (lost_prev + lost_prefix) / (inflight_prev + lost_prefix) >= loss_thresh + * Then we take that equation, convert it to fixed point, and + * round up to the nearest packet. + */ +static u32 bbr_inflight_hi_from_lost_skb(const struct sock *sk, + const struct rate_sample *rs, + const struct sk_buff *skb) +{ + const struct tcp_sock *tp = tcp_sk(sk); + u32 loss_thresh = bbr_param(sk, loss_thresh); + u32 pcount, divisor, inflight_hi; + s32 inflight_prev, lost_prev; + u64 loss_budget, lost_prefix; + + pcount = tcp_skb_pcount(skb); + + /* How much data was in flight before this skb? */ + inflight_prev = rs->tx_in_flight - pcount; + if (inflight_prev < 0) { + WARN_ONCE(tcp_skb_tx_in_flight_is_suspicious( + pcount, + TCP_SKB_CB(skb)->sacked, + rs->tx_in_flight), + "tx_in_flight: %u pcount: %u reneg: %u", + rs->tx_in_flight, pcount, tcp_sk(sk)->is_sack_reneg); + return ~0U; + } + + /* How much inflight data was marked lost before this skb? */ + lost_prev = rs->lost - pcount; + if (WARN_ONCE(lost_prev < 0, + "cwnd: %u ca: %d out: %u lost: %u pif: %u " + "tx_in_flight: %u tx.lost: %u tp->lost: %u rs->lost: %d " + "lost_prev: %d pcount: %d seq: %u end_seq: %u reneg: %u", + tcp_snd_cwnd(tp), inet_csk(sk)->icsk_ca_state, + tp->packets_out, tp->lost_out, tcp_packets_in_flight(tp), + rs->tx_in_flight, TCP_SKB_CB(skb)->tx.lost, tp->lost, + rs->lost, lost_prev, pcount, + TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq, + tp->is_sack_reneg)) + return ~0U; + + /* At what prefix of this lost skb did losss rate exceed loss_thresh? */ + loss_budget = (u64)inflight_prev * loss_thresh + BBR_UNIT - 1; + loss_budget >>= BBR_SCALE; + if (lost_prev >= loss_budget) { + lost_prefix = 0; /* previous losses crossed loss_thresh */ + } else { + lost_prefix = loss_budget - lost_prev; + lost_prefix <<= BBR_SCALE; + divisor = BBR_UNIT - loss_thresh; + if (WARN_ON_ONCE(!divisor)) /* loss_thresh is 8 bits */ + return ~0U; + do_div(lost_prefix, divisor); + } + + inflight_hi = inflight_prev + lost_prefix; + return inflight_hi; +} + +/* If loss/ECN rates during probing indicated we may have overfilled a + * buffer, return an operating point that tries to leave unutilized headroom in + * the path for other flows, for fairness convergence and lower RTTs and loss. + */ +static u32 bbr_inflight_with_headroom(const struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 headroom, headroom_fraction; + + if (bbr->inflight_hi == ~0U) + return ~0U; + + headroom_fraction = bbr_param(sk, inflight_headroom); + headroom = ((u64)bbr->inflight_hi * headroom_fraction) >> BBR_SCALE; + headroom = max(headroom, 1U); + return max_t(s32, bbr->inflight_hi - headroom, + bbr_param(sk, cwnd_min_target)); +} + +/* Bound cwnd to a sensible level, based on our current probing state + * machine phase and model of a good inflight level (inflight_lo, inflight_hi). + */ +static void bbr_bound_cwnd_for_inflight_model(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 cap; + + /* tcp_rcv_synsent_state_process() currently calls tcp_ack() + * and thus cong_control() without first initializing us(!). + */ + if (!bbr->initialized) + return; + + cap = ~0U; + if (bbr->mode == BBR_PROBE_BW && + bbr->cycle_idx != BBR_BW_PROBE_CRUISE) { + /* Probe to see if more packets fit in the path. */ + cap = bbr->inflight_hi; + } else { + if (bbr->mode == BBR_PROBE_RTT || + (bbr->mode == BBR_PROBE_BW && + bbr->cycle_idx == BBR_BW_PROBE_CRUISE)) + cap = bbr_inflight_with_headroom(sk); + } + /* Adapt to any loss/ECN since our last bw probe. */ + cap = min(cap, bbr->inflight_lo); + + cap = max_t(u32, cap, bbr_param(sk, cwnd_min_target)); + tcp_snd_cwnd_set(tp, min(cap, tcp_snd_cwnd(tp))); +} + +/* How should we multiplicatively cut bw or inflight limits based on ECN? */ +static u32 bbr_ecn_cut(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + return BBR_UNIT - + ((bbr->ecn_alpha * bbr_param(sk, ecn_factor)) >> BBR_SCALE); +} + +/* Init lower bounds if have not inited yet. */ +static void bbr_init_lower_bounds(struct sock *sk, bool init_bw) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + if (init_bw && bbr->bw_lo == ~0U) + bbr->bw_lo = bbr_max_bw(sk); + if (bbr->inflight_lo == ~0U) + bbr->inflight_lo = tcp_snd_cwnd(tp); +} + +/* Reduce bw and inflight to (1 - beta). */ +static void bbr_loss_lower_bounds(struct sock *sk, u32 *bw, u32 *inflight) +{ + struct bbr* bbr = inet_csk_ca(sk); + u32 loss_cut = BBR_UNIT - bbr_param(sk, beta); + + *bw = max_t(u32, bbr->bw_latest, + (u64)bbr->bw_lo * loss_cut >> BBR_SCALE); + *inflight = max_t(u32, bbr->inflight_latest, + (u64)bbr->inflight_lo * loss_cut >> BBR_SCALE); +} + +/* Reduce inflight to (1 - alpha*ecn_factor). */ +static void bbr_ecn_lower_bounds(struct sock *sk, u32 *inflight) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 ecn_cut = bbr_ecn_cut(sk); + + *inflight = (u64)bbr->inflight_lo * ecn_cut >> BBR_SCALE; +} + +/* Estimate a short-term lower bound on the capacity available now, based + * on measurements of the current delivery process and recent history. When we + * are seeing loss/ECN at times when we are not probing bw, then conservatively + * move toward flow balance by multiplicatively cutting our short-term + * estimated safe rate and volume of data (bw_lo and inflight_lo). We use a + * multiplicative decrease in order to converge to a lower capacity in time + * logarithmic in the magnitude of the decrease. + * + * However, we do not cut our short-term estimates lower than the current rate + * and volume of delivered data from this round trip, since from the current + * delivery process we can estimate the measured capacity available now. + * + * Anything faster than that approach would knowingly risk high loss, which can + * cause low bw for Reno/CUBIC and high loss recovery latency for + * request/response flows using any congestion control. + */ +static void bbr_adapt_lower_bounds(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 ecn_inflight_lo = ~0U; + + /* We only use lower-bound estimates when not probing bw. + * When probing we need to push inflight higher to probe bw. + */ + if (bbr_is_probing_bandwidth(sk)) + return; + + /* ECN response. */ + if (bbr->ecn_in_round && !!bbr_param(sk, ecn_factor)) { + bbr_init_lower_bounds(sk, false); + bbr_ecn_lower_bounds(sk, &ecn_inflight_lo); + } + + /* Loss response. */ + if (bbr->loss_in_round) { + bbr_init_lower_bounds(sk, true); + bbr_loss_lower_bounds(sk, &bbr->bw_lo, &bbr->inflight_lo); + } + + /* Adjust to the lower of the levels implied by loss/ECN. */ + bbr->inflight_lo = min(bbr->inflight_lo, ecn_inflight_lo); + bbr->bw_lo = max(1U, bbr->bw_lo); +} + +/* Reset any short-term lower-bound adaptation to congestion, so that we can + * push our inflight up. + */ +static void bbr_reset_lower_bounds(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->bw_lo = ~0U; + bbr->inflight_lo = ~0U; +} + +/* After bw probing (STARTUP/PROBE_UP), reset signals before entering a state + * machine phase where we adapt our lower bound based on congestion signals. + */ +static void bbr_reset_congestion_signals(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->loss_in_round = 0; + bbr->ecn_in_round = 0; + bbr->loss_in_cycle = 0; + bbr->ecn_in_cycle = 0; + bbr->bw_latest = 0; + bbr->inflight_latest = 0; +} + +static void bbr_exit_loss_recovery(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + tcp_snd_cwnd_set(tp, max(tcp_snd_cwnd(tp), bbr->prior_cwnd)); + bbr->try_fast_path = 0; /* bound cwnd using latest model */ +} + +/* Update rate and volume of delivered data from latest round trip. */ +static void bbr_update_latest_delivery_signals( + struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr->loss_round_start = 0; + if (rs->interval_us <= 0 || !rs->acked_sacked) + return; /* Not a valid observation */ + + bbr->bw_latest = max_t(u32, bbr->bw_latest, ctx->sample_bw); + bbr->inflight_latest = max_t(u32, bbr->inflight_latest, rs->delivered); + + if (!before(rs->prior_delivered, bbr->loss_round_delivered)) { + bbr->loss_round_delivered = tp->delivered; + bbr->loss_round_start = 1; /* mark start of new round trip */ + } +} + +/* Once per round, reset filter for latest rate and volume of delivered data. */ +static void bbr_advance_latest_delivery_signals( + struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + /* If ACK matches a TLP retransmit, persist the filter. If we detect + * that a TLP retransmit plugged a tail loss, we'll want to remember + * how much data the path delivered before the tail loss. + */ + if (bbr->loss_round_start && !rs->is_acking_tlp_retrans_seq) { + bbr->bw_latest = ctx->sample_bw; + bbr->inflight_latest = rs->delivered; + } +} + +/* Update (most of) our congestion signals: track the recent rate and volume of + * delivered data, presence of loss, and EWMA degree of ECN marking. + */ +static void bbr_update_congestion_signals( + struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + u64 bw; + + if (rs->interval_us <= 0 || !rs->acked_sacked) + return; /* Not a valid observation */ + bw = ctx->sample_bw; + + if (!rs->is_app_limited || bw >= bbr_max_bw(sk)) + bbr_take_max_bw_sample(sk, bw); + + bbr->loss_in_round |= (rs->losses > 0); + + if (!bbr->loss_round_start) + return; /* skip the per-round-trip updates */ + /* Now do per-round-trip updates. */ + bbr_adapt_lower_bounds(sk, rs); + + bbr->loss_in_round = 0; + bbr->ecn_in_round = 0; +} + +/* Bandwidth probing can cause loss. To help coexistence with loss-based + * congestion control we spread out our probing in a Reno-conscious way. Due to + * the shape of the Reno sawtooth, the time required between loss epochs for an + * idealized Reno flow is a number of round trips that is the BDP of that + * flow. We count packet-timed round trips directly, since measured RTT can + * vary widely, and Reno is driven by packet-timed round trips. + */ +static bool bbr_is_reno_coexistence_probe_time(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 rounds; + + /* Random loss can shave some small percentage off of our inflight + * in each round. To survive this, flows need robust periodic probes. + */ + rounds = min_t(u32, bbr_param(sk, bw_probe_max_rounds), bbr_target_inflight(sk)); + return bbr->rounds_since_probe >= rounds; +} + +/* How long do we want to wait before probing for bandwidth (and risking + * loss)? We randomize the wait, for better mixing and fairness convergence. + * + * We bound the Reno-coexistence inter-bw-probe time to be 62-63 round trips. + * This is calculated to allow fairness with a 25Mbps, 30ms Reno flow, + * (eg 4K video to a broadband user): + * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets + * + * We bound the BBR-native inter-bw-probe wall clock time to be: + * (a) higher than 2 sec: to try to avoid causing loss for a long enough time + * to allow Reno at 30ms to get 4K video bw, the inter-bw-probe time must + * be at least: 25Mbps * .030sec / (1514bytes) * 0.030sec = 1.9secs + * (b) lower than 3 sec: to ensure flows can start probing in a reasonable + * amount of time to discover unutilized bw on human-scale interactive + * time-scales (e.g. perhaps traffic from a web page download that we + * were competing with is now complete). + */ +static void bbr_pick_probe_wait(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + /* Decide the random round-trip bound for wait until probe: */ + bbr->rounds_since_probe = + get_random_u32_below(bbr_param(sk, bw_probe_rand_rounds)); + /* Decide the random wall clock bound for wait until probe: */ + bbr->probe_wait_us = bbr_param(sk, bw_probe_base_us) + + get_random_u32_below(bbr_param(sk, bw_probe_rand_us)); +} + +static void bbr_set_cycle_idx(struct sock *sk, int cycle_idx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->cycle_idx = cycle_idx; + /* New phase, so need to update cwnd and pacing rate. */ + bbr->try_fast_path = 0; +} + +/* Send at estimated bw to fill the pipe, but not queue. We need this phase + * before PROBE_UP, because as soon as we send faster than the available bw + * we will start building a queue, and if the buffer is shallow we can cause + * loss. If we do not fill the pipe before we cause this loss, our bw_hi and + * inflight_hi estimates will underestimate. + */ +static void bbr_start_bw_probe_refill(struct sock *sk, u32 bw_probe_up_rounds) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr_reset_lower_bounds(sk); + bbr->bw_probe_up_rounds = bw_probe_up_rounds; + bbr->bw_probe_up_acks = 0; + bbr->stopped_risky_probe = 0; + bbr->ack_phase = BBR_ACKS_REFILLING; + bbr->next_rtt_delivered = tp->delivered; + bbr_set_cycle_idx(sk, BBR_BW_PROBE_REFILL); +} + +/* Now probe max deliverable data rate and volume. */ +static void bbr_start_bw_probe_up(struct sock *sk, struct bbr_context *ctx) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr->ack_phase = BBR_ACKS_PROBE_STARTING; + bbr->next_rtt_delivered = tp->delivered; + bbr->cycle_mstamp = tp->tcp_mstamp; + bbr_reset_full_bw(sk); + bbr->full_bw = ctx->sample_bw; + bbr_set_cycle_idx(sk, BBR_BW_PROBE_UP); + bbr_raise_inflight_hi_slope(sk); +} + +/* Start a new PROBE_BW probing cycle of some wall clock length. Pick a wall + * clock time at which to probe beyond an inflight that we think to be + * safe. This will knowingly risk packet loss, so we want to do this rarely, to + * keep packet loss rates low. Also start a round-trip counter, to probe faster + * if we estimate a Reno flow at our BDP would probe faster. + */ +static void bbr_start_bw_probe_down(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr_reset_congestion_signals(sk); + bbr->bw_probe_up_cnt = ~0U; /* not growing inflight_hi any more */ + bbr_pick_probe_wait(sk); + bbr->cycle_mstamp = tp->tcp_mstamp; /* start wall clock */ + bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; + bbr->next_rtt_delivered = tp->delivered; + bbr_set_cycle_idx(sk, BBR_BW_PROBE_DOWN); +} + +/* Cruise: maintain what we estimate to be a neutral, conservative + * operating point, without attempting to probe up for bandwidth or down for + * RTT, and only reducing inflight in response to loss/ECN signals. + */ +static void bbr_start_bw_probe_cruise(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->inflight_lo != ~0U) + bbr->inflight_lo = min(bbr->inflight_lo, bbr->inflight_hi); + + bbr_set_cycle_idx(sk, BBR_BW_PROBE_CRUISE); +} + +/* Loss and/or ECN rate is too high while probing. + * Adapt (once per bw probe) by cutting inflight_hi and then restarting cycle. + */ +static void bbr_handle_inflight_too_high(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + const u32 beta = bbr_param(sk, beta); + + bbr->prev_probe_too_high = 1; + bbr->bw_probe_samples = 0; /* only react once per probe */ + /* If we are app-limited then we are not robustly + * probing the max volume of inflight data we think + * might be safe (analogous to how app-limited bw + * samples are not known to be robustly probing bw). + */ + if (!rs->is_app_limited) { + bbr->inflight_hi = max_t(u32, rs->tx_in_flight, + (u64)bbr_target_inflight(sk) * + (BBR_UNIT - beta) >> BBR_SCALE); + } + if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) + bbr_start_bw_probe_down(sk); +} + +/* If we're seeing bw and loss samples reflecting our bw probing, adapt + * using the signals we see. If loss or ECN mark rate gets too high, then adapt + * inflight_hi downward. If we're able to push inflight higher without such + * signals, push higher: adapt inflight_hi upward. + */ +static bool bbr_adapt_upper_bounds(struct sock *sk, + const struct rate_sample *rs, + struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + /* Track when we'll see bw/loss samples resulting from our bw probes. */ + if (bbr->ack_phase == BBR_ACKS_PROBE_STARTING && bbr->round_start) + bbr->ack_phase = BBR_ACKS_PROBE_FEEDBACK; + if (bbr->ack_phase == BBR_ACKS_PROBE_STOPPING && bbr->round_start) { + /* End of samples from bw probing phase. */ + bbr->bw_probe_samples = 0; + bbr->ack_phase = BBR_ACKS_INIT; + /* At this point in the cycle, our current bw sample is also + * our best recent chance at finding the highest available bw + * for this flow. So now is the best time to forget the bw + * samples from the previous cycle, by advancing the window. + */ + if (bbr->mode == BBR_PROBE_BW && !rs->is_app_limited) + bbr_advance_max_bw_filter(sk); + /* If we had an inflight_hi, then probed and pushed inflight all + * the way up to hit that inflight_hi without seeing any + * high loss/ECN in all the resulting ACKs from that probing, + * then probe up again, this time letting inflight persist at + * inflight_hi for a round trip, then accelerating beyond. + */ + if (bbr->mode == BBR_PROBE_BW && + bbr->stopped_risky_probe && !bbr->prev_probe_too_high) { + bbr_start_bw_probe_refill(sk, 0); + return true; /* yes, decided state transition */ + } + } + if (bbr_is_inflight_too_high(sk, rs)) { + if (bbr->bw_probe_samples) /* sample is from bw probing? */ + bbr_handle_inflight_too_high(sk, rs); + } else { + /* Loss/ECN rate is declared safe. Adjust upper bound upward. */ + + if (bbr->inflight_hi == ~0U) + return false; /* no excess queue signals yet */ + + /* To be resilient to random loss, we must raise bw/inflight_hi + * if we observe in any phase that a higher level is safe. + */ + if (rs->tx_in_flight > bbr->inflight_hi) { + bbr->inflight_hi = rs->tx_in_flight; + } + + if (bbr->mode == BBR_PROBE_BW && + bbr->cycle_idx == BBR_BW_PROBE_UP) + bbr_probe_inflight_hi_upward(sk, rs); + } + + return false; +} + +/* Check if it's time to probe for bandwidth now, and if so, kick it off. */ +static bool bbr_check_time_to_probe_bw(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 n; + + /* If we seem to be at an operating point where we are not seeing loss + * but we are seeing ECN marks, then when the ECN marks cease we reprobe + * quickly (in case cross-traffic has ceased and freed up bw). + */ + if (bbr_param(sk, ecn_reprobe_gain) && bbr->ecn_eligible && + bbr->ecn_in_cycle && !bbr->loss_in_cycle && + inet_csk(sk)->icsk_ca_state == TCP_CA_Open) { + /* Calculate n so that when bbr_raise_inflight_hi_slope() + * computes growth_this_round as 2^n it will be roughly the + * desired volume of data (inflight_hi*ecn_reprobe_gain). + */ + n = ilog2((((u64)bbr->inflight_hi * + bbr_param(sk, ecn_reprobe_gain)) >> BBR_SCALE)); + bbr_start_bw_probe_refill(sk, n); + return true; + } + + if (bbr_has_elapsed_in_phase(sk, bbr->probe_wait_us) || + bbr_is_reno_coexistence_probe_time(sk)) { + bbr_start_bw_probe_refill(sk, 0); + return true; + } + return false; +} + +/* Is it time to transition from PROBE_DOWN to PROBE_CRUISE? */ +static bool bbr_check_time_to_cruise(struct sock *sk, u32 inflight, u32 bw) +{ + /* Always need to pull inflight down to leave headroom in queue. */ + if (inflight > bbr_inflight_with_headroom(sk)) + return false; + + return inflight <= bbr_inflight(sk, bw, BBR_UNIT); +} + +/* PROBE_BW state machine: cruise, refill, probe for bw, or drain? */ +static void bbr_update_cycle_phase(struct sock *sk, + const struct rate_sample *rs, + struct bbr_context *ctx) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + bool is_bw_probe_done = false; + u32 inflight, bw; + + if (!bbr_full_bw_reached(sk)) + return; + + /* In DRAIN, PROBE_BW, or PROBE_RTT, adjust upper bounds. */ + if (bbr_adapt_upper_bounds(sk, rs, ctx)) + return; /* already decided state transition */ + + if (bbr->mode != BBR_PROBE_BW) + return; + + inflight = bbr_packets_in_net_at_edt(sk, rs->prior_in_flight); + bw = bbr_max_bw(sk); + + switch (bbr->cycle_idx) { + /* First we spend most of our time cruising with a pacing_gain of 1.0, + * which paces at the estimated bw, to try to fully use the pipe + * without building queue. If we encounter loss/ECN marks, we adapt + * by slowing down. + */ + case BBR_BW_PROBE_CRUISE: + if (bbr_check_time_to_probe_bw(sk, rs)) + return; /* already decided state transition */ + break; + + /* After cruising, when it's time to probe, we first "refill": we send + * at the estimated bw to fill the pipe, before probing higher and + * knowingly risking overflowing the bottleneck buffer (causing loss). + */ + case BBR_BW_PROBE_REFILL: + if (bbr->round_start) { + /* After one full round trip of sending in REFILL, we + * start to see bw samples reflecting our REFILL, which + * may be putting too much data in flight. + */ + bbr->bw_probe_samples = 1; + bbr_start_bw_probe_up(sk, ctx); + } + break; + + /* After we refill the pipe, we probe by using a pacing_gain > 1.0, to + * probe for bw. If we have not seen loss/ECN, we try to raise inflight + * to at least pacing_gain*BDP; note that this may take more than + * min_rtt if min_rtt is small (e.g. on a LAN). + * + * We terminate PROBE_UP bandwidth probing upon any of the following: + * + * (1) We've pushed inflight up to hit the inflight_hi target set in the + * most recent previous bw probe phase. Thus we want to start + * draining the queue immediately because it's very likely the most + * recently sent packets will fill the queue and cause drops. + * (2) If inflight_hi has not limited bandwidth growth recently, and + * yet delivered bandwidth has not increased much recently + * (bbr->full_bw_now). + * (3) Loss filter says loss rate is "too high". + * (4) ECN filter says ECN mark rate is "too high". + * + * (1) (2) checked here, (3) (4) checked in bbr_is_inflight_too_high() + */ + case BBR_BW_PROBE_UP: + if (bbr->prev_probe_too_high && + inflight >= bbr->inflight_hi) { + bbr->stopped_risky_probe = 1; + is_bw_probe_done = true; + } else { + if (tp->is_cwnd_limited && + tcp_snd_cwnd(tp) >= bbr->inflight_hi) { + /* inflight_hi is limiting bw growth */ + bbr_reset_full_bw(sk); + bbr->full_bw = ctx->sample_bw; + } else if (bbr->full_bw_now) { + /* Plateau in estimated bw. Pipe looks full. */ + is_bw_probe_done = true; + } + } + if (is_bw_probe_done) { + bbr->prev_probe_too_high = 0; /* no loss/ECN (yet) */ + bbr_start_bw_probe_down(sk); /* restart w/ down */ + } + break; + + /* After probing in PROBE_UP, we have usually accumulated some data in + * the bottleneck buffer (if bw probing didn't find more bw). We next + * enter PROBE_DOWN to try to drain any excess data from the queue. To + * do this, we use a pacing_gain < 1.0. We hold this pacing gain until + * our inflight is less then that target cruising point, which is the + * minimum of (a) the amount needed to leave headroom, and (b) the + * estimated BDP. Once inflight falls to match the target, we estimate + * the queue is drained; persisting would underutilize the pipe. + */ + case BBR_BW_PROBE_DOWN: + if (bbr_check_time_to_probe_bw(sk, rs)) + return; /* already decided state transition */ + if (bbr_check_time_to_cruise(sk, inflight, bw)) + bbr_start_bw_probe_cruise(sk); + break; + + default: + WARN_ONCE(1, "BBR invalid cycle index %u\n", bbr->cycle_idx); + } +} + +/* Exiting PROBE_RTT, so return to bandwidth probing in STARTUP or PROBE_BW. */ +static void bbr_exit_probe_rtt(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr_reset_lower_bounds(sk); + if (bbr_full_bw_reached(sk)) { + bbr->mode = BBR_PROBE_BW; + /* Raising inflight after PROBE_RTT may cause loss, so reset + * the PROBE_BW clock and schedule the next bandwidth probe for + * a friendly and randomized future point in time. + */ + bbr_start_bw_probe_down(sk); + /* Since we are exiting PROBE_RTT, we know inflight is + * below our estimated BDP, so it is reasonable to cruise. + */ + bbr_start_bw_probe_cruise(sk); + } else { + bbr->mode = BBR_STARTUP; + } +} + +/* Exit STARTUP based on loss rate > 1% and loss gaps in round >= N. Wait until + * the end of the round in recovery to get a good estimate of how many packets + * have been lost, and how many we need to drain with a low pacing rate. + */ +static void bbr_check_loss_too_high_in_startup(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr_full_bw_reached(sk)) + return; + + /* For STARTUP exit, check the loss rate at the end of each round trip + * of Recovery episodes in STARTUP. We check the loss rate at the end + * of the round trip to filter out noisy/low loss and have a better + * sense of inflight (extent of loss), so we can drain more accurately. + */ + if (rs->losses && bbr->loss_events_in_round < 0xf) + bbr->loss_events_in_round++; /* update saturating counter */ + if (bbr_param(sk, full_loss_cnt) && bbr->loss_round_start && + inet_csk(sk)->icsk_ca_state == TCP_CA_Recovery && + bbr->loss_events_in_round >= bbr_param(sk, full_loss_cnt) && + bbr_is_inflight_too_high(sk, rs)) { + bbr_handle_queue_too_high_in_startup(sk); + return; + } + if (bbr->loss_round_start) + bbr->loss_events_in_round = 0; +} + +/* Estimate when the pipe is full, using the change in delivery rate: BBR + * estimates bw probing filled the pipe if the estimated bw hasn't changed by + * at least bbr_full_bw_thresh (25%) after bbr_full_bw_cnt (3) non-app-limited + * rounds. Why 3 rounds: 1: rwin autotuning grows the rwin, 2: we fill the + * higher rwin, 3: we get higher delivery rate samples. Or transient + * cross-traffic or radio noise can go away. CUBIC Hystart shares a similar + * design goal, but uses delay and inter-ACK spacing instead of bandwidth. + */ +static void bbr_check_full_bw_reached(struct sock *sk, + const struct rate_sample *rs, + struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 bw_thresh, full_cnt, thresh; + + if (bbr->full_bw_now || rs->is_app_limited) + return; + + thresh = bbr_param(sk, full_bw_thresh); + full_cnt = bbr_param(sk, full_bw_cnt); + bw_thresh = (u64)bbr->full_bw * thresh >> BBR_SCALE; + if (ctx->sample_bw >= bw_thresh) { + bbr_reset_full_bw(sk); + bbr->full_bw = ctx->sample_bw; + return; + } + if (!bbr->round_start) + return; + ++bbr->full_bw_cnt; + bbr->full_bw_now = bbr->full_bw_cnt >= full_cnt; + bbr->full_bw_reached |= bbr->full_bw_now; +} + +/* If pipe is probably full, drain the queue and then enter steady-state. */ +static void bbr_check_drain(struct sock *sk, const struct rate_sample *rs, + struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { + bbr->mode = BBR_DRAIN; /* drain queue we created */ + /* Set ssthresh to export purely for monitoring, to signal + * completion of initial STARTUP by setting to a non- + * TCP_INFINITE_SSTHRESH value (ssthresh is not used by BBR). + */ + tcp_sk(sk)->snd_ssthresh = + bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); + bbr_reset_congestion_signals(sk); + } /* fall through to check if in-flight is already small: */ + if (bbr->mode == BBR_DRAIN && + bbr_packets_in_net_at_edt(sk, tcp_packets_in_flight(tcp_sk(sk))) <= + bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT)) { + bbr->mode = BBR_PROBE_BW; + bbr_start_bw_probe_down(sk); + } +} + +static void bbr_update_model(struct sock *sk, const struct rate_sample *rs, + struct bbr_context *ctx) +{ + bbr_update_congestion_signals(sk, rs, ctx); + bbr_update_ack_aggregation(sk, rs); + bbr_check_loss_too_high_in_startup(sk, rs); + bbr_check_full_bw_reached(sk, rs, ctx); + bbr_check_drain(sk, rs, ctx); + bbr_update_cycle_phase(sk, rs, ctx); + bbr_update_min_rtt(sk, rs); +} + +/* Fast path for app-limited case. + * + * On each ack, we execute bbr state machine, which primarily consists of: + * 1) update model based on new rate sample, and + * 2) update control based on updated model or state change. + * + * There are certain workload/scenarios, e.g. app-limited case, where + * either we can skip updating model or we can skip update of both model + * as well as control. This provides signifcant softirq cpu savings for + * processing incoming acks. + * + * In case of app-limited, if there is no congestion (loss/ecn) and + * if observed bw sample is less than current estimated bw, then we can + * skip some of the computation in bbr state processing: + * + * - if there is no rtt/mode/phase change: In this case, since all the + * parameters of the network model are constant, we can skip model + * as well control update. + * + * - else we can skip rest of the model update. But we still need to + * update the control to account for the new rtt/mode/phase. + * + * Returns whether we can take fast path or not. + */ +static bool bbr_run_fast_path(struct sock *sk, bool *update_model, + const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 prev_min_rtt_us, prev_mode; + + if (bbr_param(sk, fast_path) && bbr->try_fast_path && + rs->is_app_limited && ctx->sample_bw < bbr_max_bw(sk) && + !bbr->loss_in_round && !bbr->ecn_in_round ) { + prev_mode = bbr->mode; + prev_min_rtt_us = bbr->min_rtt_us; + bbr_check_drain(sk, rs, ctx); + bbr_update_cycle_phase(sk, rs, ctx); + bbr_update_min_rtt(sk, rs); + + if (bbr->mode == prev_mode && + bbr->min_rtt_us == prev_min_rtt_us && + bbr->try_fast_path) { + return true; + } + + /* Skip model update, but control still needs to be updated */ + *update_model = false; + } + return false; +} + +__bpf_kfunc static void bbr_main(struct sock *sk, u32 ack, int flag, + const struct rate_sample *rs) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + struct bbr_context ctx = { 0 }; + bool update_model = true; + u32 bw, round_delivered; + int ce_ratio = -1; + + round_delivered = bbr_update_round_start(sk, rs, &ctx); + if (bbr->round_start) { + bbr->rounds_since_probe = + min_t(s32, bbr->rounds_since_probe + 1, 0xFF); + ce_ratio = bbr_update_ecn_alpha(sk); + } + bbr_plb(sk, rs, ce_ratio); + + bbr->ecn_in_round |= (bbr->ecn_eligible && rs->is_ece); + bbr_calculate_bw_sample(sk, rs, &ctx); + bbr_update_latest_delivery_signals(sk, rs, &ctx); + + if (bbr_run_fast_path(sk, &update_model, rs, &ctx)) + goto out; + + if (update_model) + bbr_update_model(sk, rs, &ctx); + + bbr_update_gains(sk); + bw = bbr_bw(sk); + bbr_set_pacing_rate(sk, bw, bbr->pacing_gain); + bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain, + tcp_snd_cwnd(tp), &ctx); + bbr_bound_cwnd_for_inflight_model(sk); + +out: + bbr_advance_latest_delivery_signals(sk, rs, &ctx); + bbr->prev_ca_state = inet_csk(sk)->icsk_ca_state; + bbr->loss_in_cycle |= rs->lost > 0; + bbr->ecn_in_cycle |= rs->delivered_ce > 0; +} + +__bpf_kfunc static void bbr_init(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr->initialized = 1; + + bbr->init_cwnd = min(0x7FU, tcp_snd_cwnd(tp)); + bbr->prior_cwnd = tp->prior_cwnd; + WRITE_ONCE(tp->snd_ssthresh, TCP_INFINITE_SSTHRESH); + bbr->next_rtt_delivered = tp->delivered; + bbr->prev_ca_state = TCP_CA_Open; + + bbr->probe_rtt_done_stamp = 0; + bbr->probe_rtt_round_done = 0; + bbr->probe_rtt_min_us = tcp_min_rtt(tp); + bbr->probe_rtt_min_stamp = tcp_jiffies32; + bbr->min_rtt_us = tcp_min_rtt(tp); + bbr->min_rtt_stamp = tcp_jiffies32; + + bbr->has_seen_rtt = 0; + bbr_init_pacing_rate_from_rtt(sk); + + bbr->round_start = 0; + bbr->idle_restart = 0; + bbr->full_bw_reached = 0; + bbr->full_bw = 0; + bbr->full_bw_cnt = 0; + bbr->cycle_mstamp = 0; + bbr->cycle_idx = 0; + + bbr_reset_startup_mode(sk); + + bbr->ack_epoch_mstamp = tp->tcp_mstamp; + bbr->ack_epoch_acked = 0; + bbr->extra_acked_win_rtts = 0; + bbr->extra_acked_win_idx = 0; + bbr->extra_acked[0] = 0; + bbr->extra_acked[1] = 0; + + bbr->ce_state = 0; + bbr->prior_rcv_nxt = tp->rcv_nxt; + bbr->try_fast_path = 0; + + cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED); + + /* Start sampling ECN mark rate after first full flight is ACKed: */ + bbr->loss_round_delivered = tp->delivered + 1; + bbr->loss_round_start = 0; + bbr->undo_bw_lo = 0; + bbr->undo_inflight_lo = 0; + bbr->undo_inflight_hi = 0; + bbr->loss_events_in_round = 0; + bbr->startup_ecn_rounds = 0; + bbr_reset_congestion_signals(sk); + bbr->bw_lo = ~0U; + bbr->bw_hi[0] = 0; + bbr->bw_hi[1] = 0; + bbr->inflight_lo = ~0U; + bbr->inflight_hi = ~0U; + bbr_reset_full_bw(sk); + bbr->bw_probe_up_cnt = ~0U; + bbr->bw_probe_up_acks = 0; + bbr->bw_probe_up_rounds = 0; + bbr->probe_wait_us = 0; + bbr->stopped_risky_probe = 0; + bbr->ack_phase = BBR_ACKS_INIT; + bbr->rounds_since_probe = 0; + bbr->bw_probe_samples = 0; + bbr->prev_probe_too_high = 0; + bbr->ecn_eligible = 0; + bbr->ecn_alpha = bbr_param(sk, ecn_alpha_init); + bbr->alpha_last_delivered = 0; + bbr->alpha_last_delivered_ce = 0; + bbr->plb.pause_until = 0; + + tp->fast_ack_mode = bbr_fast_ack_mode ? 1 : 0; + + if (bbr_can_use_ecn(sk)) + tp->ecn_flags |= TCP_ECN_ECT_PERMANENT; +} + +/* BBR marks the current round trip as a loss round. */ +static void bbr_note_loss(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + /* Capture "current" data over the full round trip of loss, to + * have a better chance of observing the full capacity of the path. + */ + if (!bbr->loss_in_round) /* first loss in this round trip? */ + bbr->loss_round_delivered = tp->delivered; /* set round trip */ + bbr->loss_in_round = 1; + bbr->loss_in_cycle = 1; +} + +/* Core TCP stack informs us that the given skb was just marked lost. */ +__bpf_kfunc static void bbr_skb_marked_lost(struct sock *sk, + const struct sk_buff *skb) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + struct tcp_skb_cb *scb = TCP_SKB_CB(skb); + struct rate_sample rs = {}; + + bbr_note_loss(sk); + + if (!bbr->bw_probe_samples) + return; /* not an skb sent while probing for bandwidth */ + if (unlikely(!scb->tx.delivered_mstamp)) + return; /* skb was SACKed, reneged, marked lost; ignore it */ + /* We are probing for bandwidth. Construct a rate sample that + * estimates what happened in the flight leading up to this lost skb, + * then see if the loss rate went too high, and if so at which packet. + */ + rs.tx_in_flight = scb->tx.in_flight; + rs.lost = tp->lost - scb->tx.lost; + rs.is_app_limited = scb->tx.is_app_limited; + if (bbr_is_inflight_too_high(sk, &rs)) { + rs.tx_in_flight = bbr_inflight_hi_from_lost_skb(sk, &rs, skb); + bbr_handle_inflight_too_high(sk, &rs); + } +} + +static void bbr_run_loss_probe_recovery(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + struct rate_sample rs = {0}; + + bbr_note_loss(sk); + + if (!bbr->bw_probe_samples) + return; /* not sent while probing for bandwidth */ + /* We are probing for bandwidth. Construct a rate sample that + * estimates what happened in the flight leading up to this + * loss, then see if the loss rate went too high. + */ + rs.lost = 1; /* TLP probe repaired loss of a single segment */ + rs.tx_in_flight = bbr->inflight_latest + rs.lost; + rs.is_app_limited = tp->tlp_orig_data_app_limited; + if (bbr_is_inflight_too_high(sk, &rs)) + bbr_handle_inflight_too_high(sk, &rs); +} + +/* Revert short-term model if current loss recovery event was spurious. */ +__bpf_kfunc static u32 bbr_undo_cwnd(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr_reset_full_bw(sk); /* spurious slow-down; reset full bw detector */ + bbr->loss_in_round = 0; + + /* Revert to cwnd and other state saved before loss episode. */ + bbr->bw_lo = max(bbr->bw_lo, bbr->undo_bw_lo); + bbr->inflight_lo = max(bbr->inflight_lo, bbr->undo_inflight_lo); + bbr->inflight_hi = max(bbr->inflight_hi, bbr->undo_inflight_hi); + bbr->try_fast_path = 0; /* take slow path to set proper cwnd, pacing */ + return bbr->prior_cwnd; +} + +/* Entering loss recovery, so save state for when we undo recovery. */ +__bpf_kfunc static u32 bbr_ssthresh(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr_save_cwnd(sk); + /* For undo, save state that adapts based on loss signal. */ + bbr->undo_bw_lo = bbr->bw_lo; + bbr->undo_inflight_lo = bbr->inflight_lo; + bbr->undo_inflight_hi = bbr->inflight_hi; + return tcp_sk(sk)->snd_ssthresh; +} + +static enum tcp_bbr_phase bbr_get_phase(struct bbr *bbr) +{ + switch (bbr->mode) { + case BBR_STARTUP: + return BBR_PHASE_STARTUP; + case BBR_DRAIN: + return BBR_PHASE_DRAIN; + case BBR_PROBE_BW: + break; + case BBR_PROBE_RTT: + return BBR_PHASE_PROBE_RTT; + default: + return BBR_PHASE_INVALID; + } + switch (bbr->cycle_idx) { + case BBR_BW_PROBE_UP: + return BBR_PHASE_PROBE_BW_UP; + case BBR_BW_PROBE_DOWN: + return BBR_PHASE_PROBE_BW_DOWN; + case BBR_BW_PROBE_CRUISE: + return BBR_PHASE_PROBE_BW_CRUISE; + case BBR_BW_PROBE_REFILL: + return BBR_PHASE_PROBE_BW_REFILL; + default: + return BBR_PHASE_INVALID; + } +} + +static size_t bbr_get_info(struct sock *sk, u32 ext, int *attr, + union tcp_cc_info *info) +{ + if (ext & (1 << (INET_DIAG_BBRINFO - 1)) || + ext & (1 << (INET_DIAG_VEGASINFO - 1))) { + struct bbr *bbr = inet_csk_ca(sk); + u64 bw = bbr_bw_bytes_per_sec(sk, bbr_bw(sk)); + u64 bw_hi = bbr_bw_bytes_per_sec(sk, bbr_max_bw(sk)); + u64 bw_lo = bbr->bw_lo == ~0U ? + ~0ULL : bbr_bw_bytes_per_sec(sk, bbr->bw_lo); + struct tcp_bbr_info *bbr_info = &info->bbr; + + memset(bbr_info, 0, sizeof(*bbr_info)); + bbr_info->bbr_bw_lo = (u32)bw; + bbr_info->bbr_bw_hi = (u32)(bw >> 32); + bbr_info->bbr_min_rtt = bbr->min_rtt_us; + bbr_info->bbr_pacing_gain = bbr->pacing_gain; + bbr_info->bbr_cwnd_gain = bbr->cwnd_gain; + bbr_info->bbr_bw_hi_lsb = (u32)bw_hi; + bbr_info->bbr_bw_hi_msb = (u32)(bw_hi >> 32); + bbr_info->bbr_bw_lo_lsb = (u32)bw_lo; + bbr_info->bbr_bw_lo_msb = (u32)(bw_lo >> 32); + bbr_info->bbr_mode = bbr->mode; + bbr_info->bbr_phase = (__u8)bbr_get_phase(bbr); + bbr_info->bbr_version = (__u8)BBR_VERSION; + bbr_info->bbr_inflight_lo = bbr->inflight_lo; + bbr_info->bbr_inflight_hi = bbr->inflight_hi; + bbr_info->bbr_extra_acked = bbr_extra_acked(sk); + *attr = INET_DIAG_BBRINFO; + return sizeof(*bbr_info); + } + return 0; +} + +__bpf_kfunc static void bbr_set_state(struct sock *sk, u8 new_state) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + if (new_state == TCP_CA_Loss) { + + bbr->prev_ca_state = TCP_CA_Loss; + tcp_plb_update_state_upon_rto(sk, &bbr->plb); + /* The tcp_write_timeout() call to sk_rethink_txhash() likely + * repathed this flow, so re-learn the min network RTT on the + * new path: + */ + bbr_reset_full_bw(sk); + if (!bbr_is_probing_bandwidth(sk) && bbr->inflight_lo == ~0U) { + /* bbr_adapt_lower_bounds() needs cwnd before + * we suffered an RTO, to update inflight_lo: + */ + bbr->inflight_lo = + max(tcp_snd_cwnd(tp), bbr->prior_cwnd); + } + } else if (bbr->prev_ca_state == TCP_CA_Loss && + new_state != TCP_CA_Loss) { + bbr_exit_loss_recovery(sk); + } +} + + +static struct tcp_congestion_ops tcp_bbr_cong_ops __read_mostly = { + .flags = TCP_CONG_NON_RESTRICTED | TCP_CONG_WANTS_CE_EVENTS, + .name = "bbr3", + .owner = THIS_MODULE, + .init = bbr_init, + .cong_control = bbr_main, + .sndbuf_expand = bbr_sndbuf_expand, + .skb_marked_lost = bbr_skb_marked_lost, + .undo_cwnd = bbr_undo_cwnd, + .cwnd_event = bbr_cwnd_event, + .cwnd_event_tx_start = bbr_cwnd_event_tx_start, + .ssthresh = bbr_ssthresh, + .tso_segs = bbr_tso_segs, + .get_info = bbr_get_info, + .set_state = bbr_set_state, +}; + +BTF_KFUNCS_START(tcp_bbr_check_kfunc_ids) +BTF_ID_FLAGS(func, bbr_init) +BTF_ID_FLAGS(func, bbr_main) +BTF_ID_FLAGS(func, bbr_sndbuf_expand) +BTF_ID_FLAGS(func, bbr_skb_marked_lost) +BTF_ID_FLAGS(func, bbr_undo_cwnd) +BTF_ID_FLAGS(func, bbr_cwnd_event) +BTF_ID_FLAGS(func, bbr_cwnd_event_tx_start) +BTF_ID_FLAGS(func, bbr_ssthresh) +BTF_ID_FLAGS(func, bbr_tso_segs) +BTF_ID_FLAGS(func, bbr_set_state) +BTF_KFUNCS_END(tcp_bbr_check_kfunc_ids) + +static const struct btf_kfunc_id_set tcp_bbr_kfunc_set = { + .owner = THIS_MODULE, + .set = &tcp_bbr_check_kfunc_ids, +}; + +static int __init bbr_register(void) +{ + int ret; + + BUILD_BUG_ON(sizeof(struct bbr) > ICSK_CA_PRIV_SIZE); + + ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &tcp_bbr_kfunc_set); + if (ret < 0) + return ret; + return tcp_register_congestion_control(&tcp_bbr_cong_ops); +} + +static void __exit bbr_unregister(void) +{ + tcp_unregister_congestion_control(&tcp_bbr_cong_ops); +} + +module_init(bbr_register); +module_exit(bbr_unregister); + +MODULE_AUTHOR("Van Jacobson "); +MODULE_AUTHOR("Neal Cardwell "); +MODULE_AUTHOR("Yuchung Cheng "); +MODULE_AUTHOR("Soheil Hassas Yeganeh "); +MODULE_AUTHOR("Priyaranjan Jha "); +MODULE_AUTHOR("Yousuk Seung "); +MODULE_AUTHOR("Kevin Yang "); +MODULE_AUTHOR("Arjun Roy "); +MODULE_AUTHOR("David Morley "); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("TCP BBR (Bottleneck Bandwidth and RTT)"); +MODULE_VERSION(__stringify(BBR_VERSION)); diff -ruN --no-dereference -x .git a/net/ipv4/tcp_cong.c b/net/ipv4/tcp_cong.c --- a/net/ipv4/tcp_cong.c 2026-10-11 15:27:50.000000000 +0200 +++ b/net/ipv4/tcp_cong.c 2026-10-13 14:33:30.000000000 +0200 @@ -238,6 +238,7 @@ struct inet_connection_sock *icsk = inet_csk(sk); tcp_sk(sk)->prior_ssthresh = 0; + tcp_sk(sk)->fast_ack_mode = 0; if (icsk->icsk_ca_ops->init) icsk->icsk_ca_ops->init(sk); if (tcp_ca_needs_ecn(sk)) diff -ruN --no-dereference -x .git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c --- a/net/ipv4/tcp_input.c 2026-10-11 15:27:50.000000000 +0200 +++ b/net/ipv4/tcp_input.c 2026-10-13 14:33:30.000000000 +0200 @@ -358,7 +358,7 @@ tcp_enter_quickack_mode(sk, 2); break; case INET_ECN_CE: - if (tcp_ca_needs_ecn(sk)) + if (tcp_ca_wants_ce_events(sk)) tcp_ca_event(sk, CA_EVENT_ECN_IS_CE); if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR) && @@ -376,7 +376,7 @@ tp->ecn_flags |= TCP_ECN_SEEN; break; default: - if (tcp_ca_needs_ecn(sk)) + if (tcp_ca_wants_ce_events(sk)) tcp_ca_event(sk, CA_EVENT_ECN_NO_CE); if (!tcp_ecn_mode_rfc3168(tp)) break; @@ -1323,7 +1323,12 @@ */ static void tcp_notify_skb_loss_event(struct tcp_sock *tp, const struct sk_buff *skb) { + struct sock *sk = (struct sock *)tp; + const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; + tp->lost += tcp_skb_pcount(skb); + if (ca_ops->skb_marked_lost) + ca_ops->skb_marked_lost(sk, skb); } void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb) @@ -1751,6 +1756,7 @@ return; } rs->delivered = tp->delivered - rs->prior_delivered; + rs->lost = tp->lost - rs->prior_lost; rs->delivered_ce = tp->delivered_ce - rs->prior_delivered_ce; /* delivered_ce occupies less than 32 bits in the skb control block */ @@ -1762,7 +1768,7 @@ * longer phase. */ snd_us = rs->interval_us; /* send phase */ - ack_us = tcp_stamp_us_delta(tp->tcp_mstamp, + ack_us = tcp_stamp32_us_delta(tp->tcp_mstamp, rs->prior_mstamp); /* ack phase */ rs->interval_us = max(snd_us, ack_us); @@ -1819,19 +1825,21 @@ if (!rs->prior_delivered || tcp_skb_sent_after(tx_tstamp, tp->first_tx_mstamp, scb->end_seq, rs->last_end_seq)) { + rs->prior_lost = scb->tx.lost; rs->prior_delivered_ce = scb->tx.delivered_ce; rs->prior_delivered = scb->tx.delivered; rs->prior_mstamp = scb->tx.delivered_mstamp; rs->is_app_limited = scb->tx.is_app_limited; rs->is_retrans = scb->sacked & TCPCB_RETRANS; + rs->tx_in_flight = scb->tx.in_flight; rs->last_end_seq = scb->end_seq; /* Record send time of most recently ACKed packet: */ tp->first_tx_mstamp = tx_tstamp; /* Find the duration of the "send phase" of this window: */ - rs->interval_us = tcp_stamp_us_delta(tp->first_tx_mstamp, - scb->tx.first_tx_mstamp); - + rs->interval_us = tcp_stamp32_us_delta( + tp->first_tx_mstamp, + scb->tx.first_tx_mstamp); } /* Mark off the skb delivered once it's sacked to avoid being * used again when it's cumulatively acked. For acked packets @@ -1874,6 +1882,17 @@ WARN_ON_ONCE(tcp_skb_pcount(skb) < pcount); tcp_skb_pcount_add(skb, -pcount); + /* Adjust tx.in_flight as pcount is shifted from skb to prev. */ + if (WARN_ONCE(TCP_SKB_CB(skb)->tx.in_flight < pcount, + "prev in_flight: %u skb in_flight: %u pcount: %u", + TCP_SKB_CB(prev)->tx.in_flight, + TCP_SKB_CB(skb)->tx.in_flight, + pcount)) + TCP_SKB_CB(skb)->tx.in_flight = 0; + else + TCP_SKB_CB(skb)->tx.in_flight -= pcount; + TCP_SKB_CB(prev)->tx.in_flight += pcount; + /* When we're adding to gso_segs == 1, gso_size will be zero, * in theory this shouldn't be necessary but as long as DSACK * code can come after this skb later on it's better to keep @@ -4139,7 +4158,8 @@ /* This routine deals with acks during a TLP episode and ends an episode by * resetting tlp_high_seq. Ref: TLP algorithm in RFC8985 */ -static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag) +static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag, + struct rate_sample *rs) { struct tcp_sock *tp = tcp_sk(sk); @@ -4156,6 +4176,7 @@ /* ACK advances: there was a loss, so reduce cwnd. Reset * tlp_high_seq in tcp_init_cwnd_reduction() */ + tcp_ca_event(sk, CA_EVENT_TLP_RECOVERY); tcp_init_cwnd_reduction(sk); tcp_set_ca_state(sk, TCP_CA_CWR); tcp_end_cwnd_reduction(sk); @@ -4166,6 +4187,11 @@ FLAG_NOT_DUP | FLAG_DATA_SACKED))) { /* Pure dupack: original and TLP probe arrived; no loss */ tp->tlp_high_seq = 0; + } else { + /* This ACK matches a TLP retransmit. We cannot yet tell if + * this ACK is for the original or the TLP retransmit. + */ + rs->is_acking_tlp_retrans_seq = 1; } } @@ -4339,6 +4365,7 @@ prior_fack = tcp_is_sack(tp) ? tcp_highest_sack_seq(tp) : tp->snd_una; rs.prior_in_flight = tcp_packets_in_flight(tp); + tcp_rate_check_app_limited(sk); /* ts_recent update must be made after we are sure that the packet * is in window. @@ -4411,7 +4438,7 @@ tcp_in_ack_event(sk, flag); if (unlikely(tp->tlp_high_seq)) - tcp_process_tlp_ack(sk, ack, flag); + tcp_process_tlp_ack(sk, ack, flag, &rs); if (tcp_ack_is_dubious(sk, flag)) { if (!(flag & (FLAG_SND_UNA_ADVANCED | @@ -4436,6 +4463,7 @@ lost = tp->lost - lost; /* freshly marked lost */ rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED); + rs.is_ece = !!(flag & FLAG_ECE); tcp_rate_gen(sk, delivered, lost, is_sack_reneg, sack_state.rate); tcp_cong_control(sk, ack, delivered, flag, sack_state.rate); tcp_xmit_recovery(sk, rexmit); @@ -4461,7 +4489,7 @@ tcp_ack_probe(sk); if (unlikely(tp->tlp_high_seq)) - tcp_process_tlp_ack(sk, ack, flag); + tcp_process_tlp_ack(sk, ack, flag, &rs); return 1; old_ack: @@ -6127,13 +6155,14 @@ /* More than one full frame received... */ if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss && + (tp->fast_ack_mode == 1 || /* ... and right edge of window advances far enough. * (tcp_recvmsg() will send ACK otherwise). * If application uses SO_RCVLOWAT, we want send ack now if * we have not received enough bytes to satisfy the condition. */ - (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || - __tcp_select_window(sk) >= tp->rcv_wnd)) || + (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || + __tcp_select_window(sk) >= tp->rcv_wnd))) || /* We ACK each frame or... */ tcp_in_quickack_mode(sk) || /* Protocol state mandates a one-time immediate ACK */ diff -ruN --no-dereference -x .git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c --- a/net/ipv4/tcp_minisocks.c 2026-10-11 15:27:50.000000000 +0200 +++ b/net/ipv4/tcp_minisocks.c 2026-10-13 14:33:30.000000000 +0200 @@ -500,6 +500,8 @@ u32 ca_key = dst_metric(dst, RTAX_CC_ALGO); bool ca_got_dst = false; + tcp_set_ecn_low_from_dst(sk, dst); + if (ca_key != TCP_CA_UNSPEC) { const struct tcp_congestion_ops *ca; diff -ruN --no-dereference -x .git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c --- a/net/ipv4/tcp_output.c 2026-10-11 15:27:50.000000000 +0200 +++ b/net/ipv4/tcp_output.c 2026-10-13 14:33:30.000000000 +0200 @@ -365,7 +365,8 @@ th->cwr = 1; skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; } - } else if (!tcp_ca_needs_ecn(sk)) { + } else if (!(tp->ecn_flags & TCP_ECN_ECT_PERMANENT) && + !tcp_ca_needs_ecn(sk)) { /* ACK or retransmitted segment: clear ECT|CE */ INET_ECN_dontxmit(sk); } @@ -1467,6 +1468,24 @@ list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue); } +static void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb) +{ + struct tcp_sock *tp = tcp_sk(sk); + u32 in_flight; + + /* Check, sanitize, and record packets in flight after skb was sent. */ + in_flight = tcp_packets_in_flight(tp) + tcp_skb_pcount(skb); + if (WARN_ONCE(in_flight > TCPCB_IN_FLIGHT_MAX, + "insane in_flight %u cc %s mss %u " + "cwnd %u pif %u %u %u %u\n", + in_flight, inet_csk(sk)->icsk_ca_ops->name, + tp->mss_cache, tp->snd_cwnd, + tp->packets_out, tp->retrans_out, + tp->sacked_out, tp->lost_out)) + in_flight = TCPCB_IN_FLIGHT_MAX; + TCP_SKB_CB(skb)->tx.in_flight = in_flight; +} + /* Snapshot the current delivery information in the skb, to generate * a rate sample later when the skb is (s)acked in tcp_rate_skb_delivered(). */ @@ -1499,7 +1518,9 @@ TCP_SKB_CB(skb)->tx.delivered_mstamp = tp->delivered_mstamp; TCP_SKB_CB(skb)->tx.delivered = tp->delivered; TCP_SKB_CB(skb)->tx.delivered_ce = tp->delivered_ce; + TCP_SKB_CB(skb)->tx.lost = tp->lost; TCP_SKB_CB(skb)->tx.is_app_limited = tp->app_limited ? 1 : 0; + tcp_set_tx_in_flight(sk, skb); } INDIRECT_CALLABLE_DECLARE(int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl)); @@ -1847,7 +1868,7 @@ { struct tcp_sock *tp = tcp_sk(sk); struct sk_buff *buff; - int old_factor; + int old_factor, inflight_prev; long limit; u16 flags; int nlen; @@ -1922,6 +1943,30 @@ if (diff) tcp_adjust_pcount(sk, skb, diff); + + inflight_prev = TCP_SKB_CB(skb)->tx.in_flight - old_factor; + if (inflight_prev < 0) { + WARN_ONCE(tcp_skb_tx_in_flight_is_suspicious( + old_factor, + TCP_SKB_CB(skb)->sacked, + TCP_SKB_CB(skb)->tx.in_flight), + "inconsistent: tx.in_flight: %u " + "old_factor: %d mss: %u sacked: %u " + "1st pcount: %d 2nd pcount: %d " + "1st len: %u 2nd len: %u ", + TCP_SKB_CB(skb)->tx.in_flight, old_factor, + mss_now, TCP_SKB_CB(skb)->sacked, + tcp_skb_pcount(skb), tcp_skb_pcount(buff), + skb->len, buff->len); + inflight_prev = 0; + } + /* Set 1st tx.in_flight as if 1st were sent by itself: */ + TCP_SKB_CB(skb)->tx.in_flight = inflight_prev + + tcp_skb_pcount(skb); + /* Set 2nd tx.in_flight with new 1st and 2nd pcounts: */ + TCP_SKB_CB(buff)->tx.in_flight = inflight_prev + + tcp_skb_pcount(skb) + + tcp_skb_pcount(buff); } /* Link BUFF into the send queue. */ @@ -2276,13 +2321,12 @@ static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now) { const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; - u32 min_tso, tso_segs; - - min_tso = ca_ops->min_tso_segs ? - ca_ops->min_tso_segs(sk) : - READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs); + u32 tso_segs; - tso_segs = tcp_tso_autosize(sk, mss_now, min_tso); + tso_segs = ca_ops->tso_segs ? + ca_ops->tso_segs(sk, mss_now) : + tcp_tso_autosize(sk, mss_now, + sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs); return min_t(u32, tso_segs, sk->sk_gso_max_segs); } @@ -3000,6 +3044,7 @@ skb_set_delivery_time(skb, tp->tcp_wstamp_ns, SKB_CLOCK_MONOTONIC); list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue); tcp_init_tso_segs(skb, mss_now); + tcp_set_tx_in_flight(sk, skb); goto repair; /* Skip network transmission */ } @@ -3212,6 +3257,7 @@ if (WARN_ON(!skb || !tcp_skb_pcount(skb))) goto rearm_timer; + tp->tlp_orig_data_app_limited = TCP_SKB_CB(skb)->tx.is_app_limited; if (__tcp_retransmit_skb(sk, skb, 1)) goto rearm_timer; diff -ruN --no-dereference -x .git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c --- a/net/ipv4/tcp_timer.c 2026-10-11 15:27:50.000000000 +0200 +++ b/net/ipv4/tcp_timer.c 2026-10-13 14:33:30.000000000 +0200 @@ -708,6 +708,7 @@ tcp_timeout_expires(sk)); return; } + tcp_rate_check_app_limited(sk); tcp_mstamp_refresh(tcp_sk(sk)); event = icsk->icsk_pending; diff -ruN --no-dereference -x .git a/scripts/package/PKGBUILD b/scripts/package/PKGBUILD --- a/scripts/package/PKGBUILD 2026-10-11 15:27:50.000000000 +0200 +++ b/scripts/package/PKGBUILD 2026-10-13 14:33:30.000000000 +0200 @@ -90,6 +90,11 @@ "${srctree}/scripts/package/install-extmod-build" "${builddir}" fi + # required when DEBUG_INFO_BTF_MODULES is enabled + if [ -f tools/bpf/resolve_btfids/resolve_btfids ]; then + install -Dt "$builddir/tools/bpf/resolve_btfids" tools/bpf/resolve_btfids/resolve_btfids + fi + echo "Installing System.map and config..." mkdir -p "${builddir}" cp System.map "${builddir}/System.map" diff -ruN --no-dereference -x .git a/sound/hda/codecs/helpers/razer_blade16_2025.c b/sound/hda/codecs/helpers/razer_blade16_2025.c --- a/sound/hda/codecs/helpers/razer_blade16_2025.c 1970-01-01 01:00:00.000000000 +0100 +++ b/sound/hda/codecs/helpers/razer_blade16_2025.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,820 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Razer Blade 16 (2025) external smart-amplifier programming and fixup, + * to be included from the codec driver. + * + * The internal speakers are driven by an external smart amplifier whose + * crossover, voicing and protection live in a DSP reached over a vendor + * coef mailbox on NID 0x20. These tables hold the programming captured + * from the Windows driver, reduced to the NID-0x20 traffic this machine + * needs (everything else the HDA parser rebuilds). + * + * Each entry is either a plain codec coef write or a mailbox write: + * strobe == 0: coef[addr] = lo + * strobe != 0: amp[addr] = (hi << 16) | lo, with the bank selected on + * port 0x89 and the write committed by the strobe opcode. + */ + +struct alc298_razer_blade16_2025_op { + u16 bank; /* mailbox bank (port 0x89); unused for a plain coef */ + u16 addr; /* mailbox register, or plain coef index */ + u16 hi; /* mailbox high word; unused for a plain coef */ + u16 lo; /* mailbox low word, or plain coef value */ + u16 strobe; /* mailbox commit opcode; 0 marks a plain coef write */ +}; + +/* program the external DSP at init; the amp is woken/parked separately */ +static const struct alc298_razer_blade16_2025_op alc298_razer_blade16_2025_amp_init[] = { + { 0x0000, 0xf102, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xf103, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xc000, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xea00, 0x0000, 0x0047, 0xb031 }, + { 0x0000, 0xf20d, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xf212, 0x0000, 0x003e, 0xb031 }, + { 0x0000, 0xc001, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xc003, 0x0000, 0x0022, 0xb031 }, + { 0x0000, 0xc004, 0x0000, 0x0044, 0xb031 }, + { 0x0000, 0xc005, 0x0000, 0x0044, 0xb031 }, + { 0x0000, 0xc007, 0x0000, 0x0064, 0xb031 }, + { 0x0000, 0xc00e, 0x0000, 0x00e7, 0xb031 }, + { 0x0000, 0xf223, 0x0000, 0x007f, 0xb031 }, + { 0x0000, 0xf224, 0x0000, 0x00db, 0xb031 }, + { 0x0000, 0xf225, 0x0000, 0x00ee, 0xb031 }, + { 0x0000, 0xf226, 0x0000, 0x003f, 0xb031 }, + { 0x0000, 0xf227, 0x0000, 0x000f, 0xb031 }, + { 0x0000, 0xf21a, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xf242, 0x0000, 0x003c, 0xb031 }, + { 0x0000, 0xc120, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xc125, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xc321, 0x0000, 0x000b, 0xb031 }, + { 0x0000, 0xc200, 0x0000, 0x00d8, 0xb031 }, + { 0x0000, 0xc201, 0x0000, 0x0027, 0xb031 }, + { 0x0000, 0xc202, 0x0000, 0x000f, 0xb031 }, + { 0x0000, 0xc400, 0x0000, 0x000e, 0xb031 }, + { 0x0000, 0xc401, 0x0000, 0x0043, 0xb031 }, + { 0x0000, 0xc402, 0x0000, 0x00e0, 0xb031 }, + { 0x0000, 0xc403, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xc404, 0x0000, 0x004c, 0xb031 }, + { 0x0000, 0xc406, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xc407, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xc408, 0x0000, 0x003f, 0xb031 }, + { 0x0000, 0xc300, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xc125, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xdf00, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xdf5f, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xdf60, 0x0000, 0x00a7, 0xb031 }, + { 0x0000, 0xc203, 0x0000, 0x0084, 0xb031 }, + { 0x0000, 0xc206, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xf10a, 0x0000, 0x000b, 0xb031 }, + { 0x0000, 0xf10b, 0x0000, 0x004c, 0xb031 }, + { 0x0000, 0xf104, 0x0000, 0x00f4, 0xb031 }, + { 0x0000, 0xf105, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xf109, 0x0000, 0x00e0, 0xb031 }, + { 0x0000, 0xf10b, 0x0000, 0x005c, 0xb031 }, + { 0x0000, 0xf104, 0x0000, 0x00f4, 0xb031 }, + { 0x0000, 0xf105, 0x0000, 0x0004, 0xb031 }, + { 0x0000, 0xf109, 0x0000, 0x0065, 0xb031 }, + { 0x0000, 0xf10b, 0x0000, 0x005c, 0xb031 }, + { 0x0000, 0xf104, 0x0000, 0x00f7, 0xb031 }, + { 0x0000, 0xf105, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xf109, 0x0000, 0x0006, 0xb031 }, + { 0x0000, 0xf10b, 0x0000, 0x005c, 0xb031 }, + { 0x0000, 0xf104, 0x0000, 0x00f7, 0xb031 }, + { 0x0000, 0xf105, 0x0000, 0x0031, 0xb031 }, + { 0x0000, 0xf109, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xf10b, 0x0000, 0x005c, 0xb031 }, + { 0x0000, 0xe706, 0x0000, 0x000f, 0xb031 }, + { 0x0000, 0xe707, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xe806, 0x0000, 0x000f, 0xb031 }, + { 0x0000, 0xe807, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xce04, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xce05, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xce06, 0x0000, 0x0031, 0xb031 }, + { 0x0000, 0xce07, 0x0000, 0x00b4, 0xb031 }, + { 0x0000, 0xcf04, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xcf05, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xcf06, 0x0000, 0x0031, 0xb031 }, + { 0x0000, 0xcf07, 0x0000, 0x00b4, 0xb031 }, + { 0x0000, 0xce60, 0x0000, 0x00e3, 0xb031 }, + { 0x0000, 0xc130, 0x0000, 0x0051, 0xb031 }, + { 0x0000, 0xe000, 0x0000, 0x00a8, 0xb031 }, + { 0x4100, 0x1888, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xc121, 0x0000, 0x000b, 0xb031 }, + { 0x0000, 0xea00, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xf800, 0x0000, 0x0020, 0xb031 }, + { 0x0000, 0xca00, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xca10, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xca02, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xca12, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xed00, 0x0000, 0x0090, 0xb031 }, + { 0x0000, 0xcc2c, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc2d, 0x0000, 0x000e, 0xb031 }, + { 0x0000, 0xcc2e, 0x0000, 0x0076, 0xb031 }, + { 0x0000, 0xcc2f, 0x0000, 0x0043, 0xb031 }, + { 0x0000, 0xcd2c, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd2d, 0x0000, 0x000e, 0xb031 }, + { 0x0000, 0xcd2e, 0x0000, 0x0076, 0xb031 }, + { 0x0000, 0xcd2f, 0x0000, 0x0043, 0xb031 }, + { 0x0000, 0xcc24, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc25, 0x0000, 0x0051, 0xb031 }, + { 0x0000, 0xcc26, 0x0000, 0x00eb, 0xb031 }, + { 0x0000, 0xcc27, 0x0000, 0x0085, 0xb031 }, + { 0x0000, 0xcd24, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd25, 0x0000, 0x0051, 0xb031 }, + { 0x0000, 0xcd26, 0x0000, 0x00eb, 0xb031 }, + { 0x0000, 0xcd27, 0x0000, 0x0085, 0xb031 }, + { 0x0000, 0xcc20, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc21, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc22, 0x0000, 0x0043, 0xb031 }, + { 0x0000, 0xcd20, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd21, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd22, 0x0000, 0x0043, 0xb031 }, + { 0x0000, 0xcc16, 0x0000, 0x000f, 0xb031 }, + { 0x0000, 0xcc17, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd16, 0x0000, 0x000f, 0xb031 }, + { 0x0000, 0xcd17, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc29, 0x0000, 0x005d, 0xb031 }, + { 0x0000, 0xcc2a, 0x0000, 0x00c0, 0xb031 }, + { 0x0000, 0xcd29, 0x0000, 0x005d, 0xb031 }, + { 0x0000, 0xcd2a, 0x0000, 0x00c0, 0xb031 }, + { 0x0000, 0xcc31, 0x0000, 0x0020, 0xb031 }, + { 0x0000, 0xcc32, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc33, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc34, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd31, 0x0000, 0x0020, 0xb031 }, + { 0x0000, 0xcd32, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd33, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd34, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc36, 0x0000, 0x0079, 0xb031 }, + { 0x0000, 0xcc37, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xcc38, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xcc39, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xcd36, 0x0000, 0x0079, 0xb031 }, + { 0x0000, 0xcd37, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xcd38, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xcd39, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xcc09, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc0a, 0x0000, 0x000a, 0xb031 }, + { 0x0000, 0xcc0b, 0x0000, 0x007c, 0xb031 }, + { 0x0000, 0xcc0c, 0x0000, 0x005b, 0xb031 }, + { 0x0000, 0xcd09, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd0a, 0x0000, 0x000a, 0xb031 }, + { 0x0000, 0xcd0b, 0x0000, 0x007c, 0xb031 }, + { 0x0000, 0xcd0c, 0x0000, 0x005b, 0xb031 }, + { 0x0000, 0xcc0e, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc0f, 0x0000, 0x0005, 0xb031 }, + { 0x0000, 0xcc10, 0x0000, 0x003e, 0xb031 }, + { 0x0000, 0xcc11, 0x0000, 0x002d, 0xb031 }, + { 0x0000, 0xcd0e, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd0f, 0x0000, 0x0005, 0xb031 }, + { 0x0000, 0xcd10, 0x0000, 0x003e, 0xb031 }, + { 0x0000, 0xcd11, 0x0000, 0x002d, 0xb031 }, + { 0x0000, 0xccd6, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xccd7, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xcdd6, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcdd7, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xccd8, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xccd9, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xcdd8, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcdd9, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xccda, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xccdb, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xcdda, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcddb, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xc320, 0x0000, 0x0020, 0xb031 }, + { 0x0000, 0xc321, 0x0000, 0x000a, 0xb031 }, + { 0x0000, 0xe604, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xdb00, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xdd00, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xdc19, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xdc1a, 0x0000, 0x006a, 0xb031 }, + { 0x0000, 0xdc1b, 0x0000, 0x00aa, 0xb031 }, + { 0x0000, 0xdc1c, 0x0000, 0x00ab, 0xb031 }, + { 0x0000, 0xdc1d, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xdc1e, 0x0000, 0x0027, 0xb031 }, + { 0x0000, 0xdc1f, 0x0000, 0x0062, 0xb031 }, + { 0x0000, 0xdc20, 0x0000, 0x0076, 0xb031 }, + { 0x0000, 0xde19, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xde1a, 0x0000, 0x006a, 0xb031 }, + { 0x0000, 0xde1b, 0x0000, 0x00aa, 0xb031 }, + { 0x0000, 0xde1c, 0x0000, 0x00ab, 0xb031 }, + { 0x0000, 0xde1d, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xde1e, 0x0000, 0x0027, 0xb031 }, + { 0x0000, 0xde1f, 0x0000, 0x0062, 0xb031 }, + { 0x0000, 0xde20, 0x0000, 0x0076, 0xb031 }, + { 0x0000, 0xdb32, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xdd32, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xdb33, 0x0000, 0x000a, 0xb031 }, + { 0x0000, 0xdd33, 0x0000, 0x000a, 0xb031 }, + { 0x0000, 0xdb34, 0x0000, 0x001a, 0xb031 }, + { 0x0000, 0xdd34, 0x0000, 0x001a, 0xb031 }, + { 0x0000, 0xdb15, 0x0000, 0x00ef, 0xb031 }, + { 0x0000, 0xdd15, 0x0000, 0x00ef, 0xb031 }, + { 0x0000, 0xdb17, 0x0000, 0x003f, 0xb031 }, + { 0x0000, 0xdd17, 0x0000, 0x003f, 0xb031 }, + { 0x0000, 0xdb94, 0x0000, 0x0070, 0xb031 }, + { 0x0000, 0xdd94, 0x0000, 0x0070, 0xb031 }, + { 0x0000, 0xdb19, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xdd19, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xc203, 0x0000, 0x001c, 0xb031 }, + { 0x0000, 0xdb00, 0x0000, 0x004c, 0xb031 }, + { 0x0000, 0xdb04, 0x0000, 0x0005, 0xb031 }, + { 0x0000, 0xdb05, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xdd04, 0x0000, 0x0005, 0xb031 }, + { 0x0000, 0xdd05, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xdbbb, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xdbbc, 0x0000, 0x004c, 0xb031 }, + { 0x0000, 0xdbbd, 0x0000, 0x00f3, 0xb031 }, + { 0x0000, 0xdbbe, 0x0000, 0x00cf, 0xb031 }, + { 0x0000, 0xddbb, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xddbc, 0x0000, 0x004c, 0xb031 }, + { 0x0000, 0xddbd, 0x0000, 0x00f3, 0xb031 }, + { 0x0000, 0xddbe, 0x0000, 0x00cf, 0xb031 }, + { 0x0000, 0xdb01, 0x0000, 0x0079, 0xb031 }, + { 0x0000, 0xdd01, 0x0000, 0x0079, 0xb031 }, + { 0x0000, 0xdb08, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xdd08, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xdc52, 0x0000, 0x00ef, 0xb031 }, + { 0x0000, 0xde52, 0x0000, 0x00ef, 0xb031 }, + { 0x0000, 0xdb00, 0x0000, 0x00cc, 0xb031 }, + { 0x0000, 0xc203, 0x0000, 0x009c, 0xb031 }, + { 0x0000, 0xdf0a, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xdf0b, 0x0000, 0x007f, 0xb031 }, + { 0x0000, 0xc851, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xc951, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xdf01, 0x0000, 0x0073, 0xb031 }, + { 0x0000, 0xc203, 0x0000, 0x009c, 0xb031 }, + { 0x0000, 0xf800, 0x0000, 0x0020, 0xb031 }, + { 0x0000, 0xebcb, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xeb10, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xeb11, 0x0000, 0x00bb, 0xb031 }, + { 0x0000, 0xeb12, 0x0000, 0x00ab, 0xb031 }, + { 0x0000, 0xeb13, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xeb14, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xeb15, 0x0000, 0x004c, 0xb031 }, + { 0x0000, 0xeb16, 0x0000, 0x0069, 0xb031 }, + { 0x0000, 0xeb21, 0x0000, 0x0011, 0xb031 }, + { 0x0000, 0xeb22, 0x0000, 0x00c2, 0xb031 }, + { 0x0000, 0xeb23, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xeb24, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xeb25, 0x0000, 0x00bd, 0xb031 }, + { 0x0000, 0xeb26, 0x0000, 0x0055, 0xb031 }, + { 0x0000, 0xeb27, 0x0000, 0x00d0, 0xb031 }, + { 0x0000, 0xeb28, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xeb29, 0x0000, 0x003c, 0xb031 }, + { 0x0000, 0xeb2a, 0x0000, 0x0051, 0xb031 }, + { 0x0000, 0xeb2b, 0x0000, 0x0018, 0xb031 }, + { 0x0000, 0xeb30, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xeb31, 0x0000, 0x004f, 0xb031 }, + { 0x0000, 0xeb32, 0x0000, 0x004a, 0xb031 }, + { 0x0000, 0xeb33, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xeb34, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xeb35, 0x0000, 0x00bb, 0xb031 }, + { 0x0000, 0xeb36, 0x0000, 0x0024, 0xb031 }, + { 0x0000, 0xeb37, 0x0000, 0x0068, 0xb031 }, + { 0x0000, 0xeb40, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xeb41, 0x0000, 0x00f5, 0xb031 }, + { 0x0000, 0xeb42, 0x0000, 0x00f1, 0xb031 }, + { 0x0000, 0xeb43, 0x0000, 0x00e8, 0xb031 }, + { 0x0000, 0xeb44, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xeb45, 0x0000, 0x004f, 0xb031 }, + { 0x0000, 0xeb46, 0x0000, 0x004a, 0xb031 }, + { 0x0000, 0xeb47, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xeb48, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xeb49, 0x0000, 0x00c5, 0xb031 }, + { 0x0000, 0xeb4a, 0x0000, 0x0032, 0xb031 }, + { 0x0000, 0xeb4b, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xeb50, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xeb51, 0x0000, 0x008a, 0xb031 }, + { 0x0000, 0xeb52, 0x0000, 0x00b4, 0xb031 }, + { 0x0000, 0xeb53, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xeb54, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xeb55, 0x0000, 0x007d, 0xb031 }, + { 0x0000, 0xeb56, 0x0000, 0x00d9, 0xb031 }, + { 0x0000, 0xeb57, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xeb60, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xeb61, 0x0000, 0x00e7, 0xb031 }, + { 0x0000, 0xeb62, 0x0000, 0x00b5, 0xb031 }, + { 0x0000, 0xeb63, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xeb64, 0x0000, 0x00fa, 0xb031 }, + { 0x0000, 0xeb65, 0x0000, 0x0086, 0xb031 }, + { 0x0000, 0xeb66, 0x0000, 0x00a8, 0xb031 }, + { 0x0000, 0xeb67, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xeb68, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xeb69, 0x0000, 0x009a, 0xb031 }, + { 0x0000, 0xeb6a, 0x0000, 0x002f, 0xb031 }, + { 0x0000, 0xeb6b, 0x0000, 0x0018, 0xb031 }, + { 0x0000, 0xebb6, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xeb70, 0x0000, 0x00f5, 0xb031 }, + { 0x0000, 0xeb71, 0x0000, 0x00e1, 0xb031 }, + { 0x0000, 0xeb72, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xeb73, 0x0000, 0x00d0, 0xb031 }, + { 0x0000, 0xeb74, 0x0000, 0x0006, 0xb031 }, + { 0x0000, 0xeb75, 0x0000, 0x0050, 0xb031 }, + { 0x0000, 0xeb76, 0x0000, 0x002c, 0xb031 }, + { 0x0000, 0xeb77, 0x0000, 0x0050, 0xb031 }, + { 0x0000, 0xeb80, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xeb81, 0x0000, 0x0028, 0xb031 }, + { 0x0000, 0xeb82, 0x0000, 0x0016, 0xb031 }, + { 0x0000, 0xeb83, 0x0000, 0x0028, 0xb031 }, + { 0x0000, 0xeb84, 0x0000, 0x00f5, 0xb031 }, + { 0x0000, 0xeb85, 0x0000, 0x00e1, 0xb031 }, + { 0x0000, 0xeb86, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xeb87, 0x0000, 0x00d0, 0xb031 }, + { 0x0000, 0xeb88, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xeb89, 0x0000, 0x0028, 0xb031 }, + { 0x0000, 0xeb8a, 0x0000, 0x0016, 0xb031 }, + { 0x0000, 0xeb8b, 0x0000, 0x0028, 0xb031 }, + { 0x0000, 0xebb8, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xeb90, 0x0000, 0x00fc, 0xb031 }, + { 0x0000, 0xeb91, 0x0000, 0x008b, 0xb031 }, + { 0x0000, 0xeb92, 0x0000, 0x00bd, 0xb031 }, + { 0x0000, 0xeb93, 0x0000, 0x0064, 0xb031 }, + { 0x0000, 0xeb94, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xeb95, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xeb96, 0x0000, 0x00e7, 0xb031 }, + { 0x0000, 0xeb97, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xeba0, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xeba1, 0x0000, 0x0091, 0xb031 }, + { 0x0000, 0xeba2, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xeba3, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xeba4, 0x0000, 0x00f6, 0xb031 }, + { 0x0000, 0xeba5, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xeba6, 0x0000, 0x0093, 0xb031 }, + { 0x0000, 0xeba7, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xeba8, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xeba9, 0x0000, 0x0087, 0xb031 }, + { 0x0000, 0xebaa, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xebab, 0x0000, 0x006c, 0xb031 }, + { 0x0000, 0xebc3, 0x0000, 0x00ff, 0xb031 }, + { 0x0000, 0xebc4, 0x0000, 0x00fc, 0xb031 }, + { 0x0000, 0xeb00, 0x0000, 0x0005, 0xb031 }, + { 0x0000, 0xeb01, 0x0000, 0x00a9, 0xb031 }, + { 0x0000, 0xeb02, 0x0000, 0x00df, 0xb031 }, + { 0x0000, 0xeb03, 0x0000, 0x007b, 0xb031 }, + { 0x0000, 0xeb05, 0x0000, 0x0079, 0xb031 }, + { 0x0000, 0xeb06, 0x0000, 0x005a, 0xb031 }, + { 0x0000, 0xeb07, 0x0000, 0x0004, 0xb031 }, + { 0x0000, 0xebc0, 0x0000, 0x00e3, 0xb031 }, + { 0x0000, 0xd0cb, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xd0b0, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xd010, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd011, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xd012, 0x0000, 0x0092, 0xb031 }, + { 0x0000, 0xd013, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd014, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xd015, 0x0000, 0x00f6, 0xb031 }, + { 0x0000, 0xd016, 0x0000, 0x0072, 0xb031 }, + { 0x0000, 0xd017, 0x0000, 0x00b0, 0xb031 }, + { 0x0000, 0xd020, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xd021, 0x0000, 0x00fb, 0xb031 }, + { 0x0000, 0xd022, 0x0000, 0x0037, 0xb031 }, + { 0x0000, 0xd023, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd024, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd025, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xd026, 0x0000, 0x0090, 0xb031 }, + { 0x0000, 0xd027, 0x0000, 0x0020, 0xb031 }, + { 0x0000, 0xd028, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xd029, 0x0000, 0x00fb, 0xb031 }, + { 0x0000, 0xd02a, 0x0000, 0x0037, 0xb031 }, + { 0x0000, 0xd02b, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd0c3, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd0c4, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xd0c0, 0x0000, 0x00e3, 0xb031 }, + { 0x0000, 0xce6a, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xce63, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xce64, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xce60, 0x0000, 0x00e3, 0xb031 }, + { 0x0000, 0xdb00, 0x0000, 0x00cc, 0xb031 }, + { 0x0000, 0xdb04, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xdb05, 0x0000, 0x0006, 0xb031 }, + { 0x0000, 0xcc00, 0x0000, 0x00c1, 0xb031 }, + { 0x0000, 0xca02, 0x0000, 0x000f, 0xb031 }, + { 0x0000, 0xca00, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xca62, 0x0000, 0x0077, 0xb031 }, + { 0x0000, 0xca65, 0x0000, 0x0082, 0xb031 }, + { 0x0000, 0xca68, 0x0000, 0x00c2, 0xb031 }, + { 0x0000, 0xca75, 0x0000, 0x003a, 0xb031 }, + { 0x0000, 0xca7b, 0x0000, 0x006f, 0xb031 }, + { 0x0000, 0xca00, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xca02, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xc860, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xc861, 0x0000, 0x00e5, 0xb031 }, + { 0x0000, 0xc862, 0x0000, 0x00eb, 0xb031 }, + { 0x0000, 0xc863, 0x0000, 0x00c6, 0xb031 }, + { 0x0000, 0xc864, 0x0000, 0x00ba, 0xb031 }, + { 0x0000, 0xc865, 0x0000, 0x0061, 0xb031 }, + { 0x0000, 0xc867, 0x0000, 0x0017, 0xb031 }, + { 0x0000, 0xc868, 0x0000, 0x00ed, 0xb031 }, + { 0x0000, 0xc869, 0x0000, 0x003c, 0xb031 }, + { 0x0000, 0xc86a, 0x0000, 0x008a, 0xb031 }, + { 0x0000, 0xc86b, 0x0000, 0x00e2, 0xb031 }, + { 0x0000, 0xc875, 0x0000, 0x003a, 0xb031 }, + { 0x0000, 0xc87b, 0x0000, 0x006f, 0xb031 }, + { 0x0000, 0xc836, 0x0000, 0x00f9, 0xb031 }, + { 0x0000, 0xc8a0, 0x0000, 0x00f1, 0xb031 }, + { 0x0000, 0xc833, 0x0000, 0x000b, 0xb031 }, + { 0x0000, 0xc834, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xc83f, 0x0000, 0x0014, 0xb031 }, + { 0x0000, 0xc83e, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xc83a, 0x0000, 0x0076, 0xb031 }, + { 0x0000, 0xc83b, 0x0000, 0x0089, 0xb031 }, + { 0x0000, 0xc830, 0x0000, 0x0013, 0xb031 }, + { 0x0000, 0xc826, 0x0000, 0x00d1, 0xb031 }, + { 0x0000, 0xc890, 0x0000, 0x00f1, 0xb031 }, + { 0x0000, 0xc823, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xc824, 0x0000, 0x0004, 0xb031 }, + { 0x0000, 0xc82f, 0x0000, 0x0018, 0xb031 }, + { 0x0000, 0xc82e, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xc820, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xc816, 0x0000, 0x00f9, 0xb031 }, + { 0x0000, 0xc880, 0x0000, 0x00e1, 0xb031 }, + { 0x0000, 0xc813, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xc814, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xc81f, 0x0000, 0x0014, 0xb031 }, + { 0x0000, 0xc81e, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xc81a, 0x0000, 0x0066, 0xb031 }, + { 0x0000, 0xc81b, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xc810, 0x0000, 0x000d, 0xb031 }, + { 0x0000, 0xc846, 0x0000, 0x00f9, 0xb031 }, + { 0x0000, 0xc8d0, 0x0000, 0x00f1, 0xb031 }, + { 0x0000, 0xc844, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xc84f, 0x0000, 0x0014, 0xb031 }, + { 0x0000, 0xc84e, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xc84a, 0x0000, 0x0026, 0xb031 }, + { 0x0000, 0xc84b, 0x0000, 0x0056, 0xb031 }, + { 0x0000, 0xc800, 0x0000, 0x00b0, 0xb031 }, + { 0x0000, 0xc851, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xc500, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xc501, 0x0000, 0x0026, 0xb031 }, + { 0x0000, 0xc502, 0x0000, 0x004a, 0xb031 }, + { 0x0000, 0xc503, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xc504, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xc505, 0x0000, 0x00da, 0xb031 }, + { 0x0000, 0xc506, 0x0000, 0x000e, 0xb031 }, + { 0x0000, 0xc507, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xc510, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xc511, 0x0000, 0x00da, 0xb031 }, + { 0x0000, 0xc512, 0x0000, 0x000e, 0xb031 }, + { 0x0000, 0xc513, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xc514, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xc515, 0x0000, 0x0026, 0xb031 }, + { 0x0000, 0xc516, 0x0000, 0x004a, 0xb031 }, + { 0x0000, 0xc517, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xc518, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xc541, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xd250, 0x0000, 0x0087, 0xb031 }, + { 0x0000, 0xd251, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xd264, 0x0000, 0x00c0, 0xb031 }, + { 0x0000, 0xc203, 0x0000, 0x009c, 0xb031 }, + { 0x0000, 0xf800, 0x0000, 0x0020, 0xb031 }, + { 0x0000, 0xecb0, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xec10, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xec11, 0x0000, 0x00bb, 0xb031 }, + { 0x0000, 0xec12, 0x0000, 0x00ab, 0xb031 }, + { 0x0000, 0xec13, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xec14, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xec15, 0x0000, 0x004c, 0xb031 }, + { 0x0000, 0xec16, 0x0000, 0x0069, 0xb031 }, + { 0x0000, 0xec21, 0x0000, 0x0011, 0xb031 }, + { 0x0000, 0xec22, 0x0000, 0x00c2, 0xb031 }, + { 0x0000, 0xec23, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xec24, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xec25, 0x0000, 0x00bd, 0xb031 }, + { 0x0000, 0xec26, 0x0000, 0x0055, 0xb031 }, + { 0x0000, 0xec27, 0x0000, 0x00d0, 0xb031 }, + { 0x0000, 0xec28, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xec29, 0x0000, 0x003c, 0xb031 }, + { 0x0000, 0xec2a, 0x0000, 0x0051, 0xb031 }, + { 0x0000, 0xec2b, 0x0000, 0x0018, 0xb031 }, + { 0x0000, 0xecb2, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xec30, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xec31, 0x0000, 0x004f, 0xb031 }, + { 0x0000, 0xec32, 0x0000, 0x004a, 0xb031 }, + { 0x0000, 0xec33, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xec34, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xec35, 0x0000, 0x00bb, 0xb031 }, + { 0x0000, 0xec36, 0x0000, 0x0024, 0xb031 }, + { 0x0000, 0xec37, 0x0000, 0x0068, 0xb031 }, + { 0x0000, 0xec40, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xec41, 0x0000, 0x00f5, 0xb031 }, + { 0x0000, 0xec42, 0x0000, 0x00f1, 0xb031 }, + { 0x0000, 0xec43, 0x0000, 0x00e8, 0xb031 }, + { 0x0000, 0xec44, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xec45, 0x0000, 0x004f, 0xb031 }, + { 0x0000, 0xec46, 0x0000, 0x004a, 0xb031 }, + { 0x0000, 0xec47, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xec48, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xec49, 0x0000, 0x00c5, 0xb031 }, + { 0x0000, 0xec4a, 0x0000, 0x0032, 0xb031 }, + { 0x0000, 0xec4b, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xecb4, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xec50, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xec51, 0x0000, 0x008a, 0xb031 }, + { 0x0000, 0xec52, 0x0000, 0x00b4, 0xb031 }, + { 0x0000, 0xec53, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xec54, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xec55, 0x0000, 0x007d, 0xb031 }, + { 0x0000, 0xec56, 0x0000, 0x00d9, 0xb031 }, + { 0x0000, 0xec57, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xec60, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xec61, 0x0000, 0x00e7, 0xb031 }, + { 0x0000, 0xec62, 0x0000, 0x00b5, 0xb031 }, + { 0x0000, 0xec63, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xec64, 0x0000, 0x00fa, 0xb031 }, + { 0x0000, 0xec65, 0x0000, 0x0086, 0xb031 }, + { 0x0000, 0xec66, 0x0000, 0x00a8, 0xb031 }, + { 0x0000, 0xec67, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xec68, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xec69, 0x0000, 0x009a, 0xb031 }, + { 0x0000, 0xec6a, 0x0000, 0x002f, 0xb031 }, + { 0x0000, 0xec6b, 0x0000, 0x0018, 0xb031 }, + { 0x0000, 0xecb6, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xec70, 0x0000, 0x00f5, 0xb031 }, + { 0x0000, 0xec71, 0x0000, 0x00e1, 0xb031 }, + { 0x0000, 0xec72, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xec73, 0x0000, 0x00d0, 0xb031 }, + { 0x0000, 0xec74, 0x0000, 0x0006, 0xb031 }, + { 0x0000, 0xec75, 0x0000, 0x0050, 0xb031 }, + { 0x0000, 0xec76, 0x0000, 0x002c, 0xb031 }, + { 0x0000, 0xec77, 0x0000, 0x0050, 0xb031 }, + { 0x0000, 0xec80, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xec81, 0x0000, 0x0028, 0xb031 }, + { 0x0000, 0xec82, 0x0000, 0x0016, 0xb031 }, + { 0x0000, 0xec83, 0x0000, 0x0028, 0xb031 }, + { 0x0000, 0xec84, 0x0000, 0x00f5, 0xb031 }, + { 0x0000, 0xec85, 0x0000, 0x00e1, 0xb031 }, + { 0x0000, 0xec86, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xec87, 0x0000, 0x00d0, 0xb031 }, + { 0x0000, 0xec88, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xec89, 0x0000, 0x0028, 0xb031 }, + { 0x0000, 0xec8a, 0x0000, 0x0016, 0xb031 }, + { 0x0000, 0xec8b, 0x0000, 0x0028, 0xb031 }, + { 0x0000, 0xecb8, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xec90, 0x0000, 0x00fc, 0xb031 }, + { 0x0000, 0xec91, 0x0000, 0x008b, 0xb031 }, + { 0x0000, 0xec92, 0x0000, 0x00bd, 0xb031 }, + { 0x0000, 0xec93, 0x0000, 0x0064, 0xb031 }, + { 0x0000, 0xec94, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xec95, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xec96, 0x0000, 0x00e7, 0xb031 }, + { 0x0000, 0xec97, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xeca0, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xeca1, 0x0000, 0x0091, 0xb031 }, + { 0x0000, 0xeca2, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xeca3, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xeca4, 0x0000, 0x00f6, 0xb031 }, + { 0x0000, 0xeca5, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xeca6, 0x0000, 0x0093, 0xb031 }, + { 0x0000, 0xeca7, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xeca8, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xeca9, 0x0000, 0x0087, 0xb031 }, + { 0x0000, 0xecaa, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xecab, 0x0000, 0x006c, 0xb031 }, + { 0x0000, 0xebc3, 0x0000, 0x00ff, 0xb031 }, + { 0x0000, 0xebc4, 0x0000, 0x00fc, 0xb031 }, + { 0x0000, 0xec00, 0x0000, 0x0005, 0xb031 }, + { 0x0000, 0xec01, 0x0000, 0x00a9, 0xb031 }, + { 0x0000, 0xec02, 0x0000, 0x00df, 0xb031 }, + { 0x0000, 0xec03, 0x0000, 0x007b, 0xb031 }, + { 0x0000, 0xec05, 0x0000, 0x0079, 0xb031 }, + { 0x0000, 0xec06, 0x0000, 0x005a, 0xb031 }, + { 0x0000, 0xec07, 0x0000, 0x0004, 0xb031 }, + { 0x0000, 0xebc0, 0x0000, 0x00e3, 0xb031 }, + { 0x0000, 0xd1b0, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xd110, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd111, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xd112, 0x0000, 0x0092, 0xb031 }, + { 0x0000, 0xd113, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd114, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xd115, 0x0000, 0x00f6, 0xb031 }, + { 0x0000, 0xd116, 0x0000, 0x0072, 0xb031 }, + { 0x0000, 0xd117, 0x0000, 0x00b0, 0xb031 }, + { 0x0000, 0xd120, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xd121, 0x0000, 0x00fb, 0xb031 }, + { 0x0000, 0xd122, 0x0000, 0x0037, 0xb031 }, + { 0x0000, 0xd123, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd124, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd125, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xd126, 0x0000, 0x0090, 0xb031 }, + { 0x0000, 0xd127, 0x0000, 0x0020, 0xb031 }, + { 0x0000, 0xd128, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xd129, 0x0000, 0x00fb, 0xb031 }, + { 0x0000, 0xd12a, 0x0000, 0x0037, 0xb031 }, + { 0x0000, 0xd12b, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd0c3, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd0c4, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xd0c0, 0x0000, 0x00e3, 0xb031 }, + { 0x0000, 0xce63, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xce64, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xce60, 0x0000, 0x00e3, 0xb031 }, + { 0x0000, 0xdb00, 0x0000, 0x00cc, 0xb031 }, + { 0x0000, 0xdd04, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xdd05, 0x0000, 0x0006, 0xb031 }, + { 0x0000, 0xca12, 0x0000, 0x000f, 0xb031 }, + { 0x0000, 0xca10, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xca82, 0x0000, 0x0077, 0xb031 }, + { 0x0000, 0xca85, 0x0000, 0x0082, 0xb031 }, + { 0x0000, 0xca88, 0x0000, 0x00c2, 0xb031 }, + { 0x0000, 0xca95, 0x0000, 0x003a, 0xb031 }, + { 0x0000, 0xca9b, 0x0000, 0x006f, 0xb031 }, + { 0x0000, 0xca10, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xca12, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xc960, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xc961, 0x0000, 0x00e5, 0xb031 }, + { 0x0000, 0xc962, 0x0000, 0x00eb, 0xb031 }, + { 0x0000, 0xc963, 0x0000, 0x00c6, 0xb031 }, + { 0x0000, 0xc964, 0x0000, 0x00ba, 0xb031 }, + { 0x0000, 0xc965, 0x0000, 0x0061, 0xb031 }, + { 0x0000, 0xc967, 0x0000, 0x0017, 0xb031 }, + { 0x0000, 0xc968, 0x0000, 0x00ed, 0xb031 }, + { 0x0000, 0xc969, 0x0000, 0x003c, 0xb031 }, + { 0x0000, 0xc96a, 0x0000, 0x008a, 0xb031 }, + { 0x0000, 0xc96b, 0x0000, 0x00e2, 0xb031 }, + { 0x0000, 0xc975, 0x0000, 0x003a, 0xb031 }, + { 0x0000, 0xc97b, 0x0000, 0x006f, 0xb031 }, + { 0x0000, 0xc936, 0x0000, 0x00f9, 0xb031 }, + { 0x0000, 0xc9a0, 0x0000, 0x00f1, 0xb031 }, + { 0x0000, 0xc933, 0x0000, 0x000b, 0xb031 }, + { 0x0000, 0xc934, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xc93f, 0x0000, 0x0014, 0xb031 }, + { 0x0000, 0xc93e, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xc93a, 0x0000, 0x0076, 0xb031 }, + { 0x0000, 0xc93b, 0x0000, 0x0089, 0xb031 }, + { 0x0000, 0xc930, 0x0000, 0x0013, 0xb031 }, + { 0x0000, 0xc926, 0x0000, 0x00d1, 0xb031 }, + { 0x0000, 0xc990, 0x0000, 0x00f1, 0xb031 }, + { 0x0000, 0xc923, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xc924, 0x0000, 0x0004, 0xb031 }, + { 0x0000, 0xc92f, 0x0000, 0x0018, 0xb031 }, + { 0x0000, 0xc92e, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xc920, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xc916, 0x0000, 0x00f9, 0xb031 }, + { 0x0000, 0xc980, 0x0000, 0x00e1, 0xb031 }, + { 0x0000, 0xc913, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xc914, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xc91f, 0x0000, 0x0014, 0xb031 }, + { 0x0000, 0xc91e, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xc91a, 0x0000, 0x0066, 0xb031 }, + { 0x0000, 0xc91b, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xc910, 0x0000, 0x000d, 0xb031 }, + { 0x0000, 0xc946, 0x0000, 0x00f9, 0xb031 }, + { 0x0000, 0xc9d0, 0x0000, 0x00f1, 0xb031 }, + { 0x0000, 0xc944, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xc94f, 0x0000, 0x0014, 0xb031 }, + { 0x0000, 0xc94e, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xc94a, 0x0000, 0x0026, 0xb031 }, + { 0x0000, 0xc94b, 0x0000, 0x0056, 0xb031 }, + { 0x0000, 0xc951, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xc600, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xc601, 0x0000, 0x0026, 0xb031 }, + { 0x0000, 0xc602, 0x0000, 0x004a, 0xb031 }, + { 0x0000, 0xc603, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xc604, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xc605, 0x0000, 0x00da, 0xb031 }, + { 0x0000, 0xc606, 0x0000, 0x000e, 0xb031 }, + { 0x0000, 0xc607, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xc610, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xc611, 0x0000, 0x00da, 0xb031 }, + { 0x0000, 0xc612, 0x0000, 0x000e, 0xb031 }, + { 0x0000, 0xc613, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xc614, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xc615, 0x0000, 0x0026, 0xb031 }, + { 0x0000, 0xc616, 0x0000, 0x004a, 0xb031 }, + { 0x0000, 0xc617, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xc618, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xc541, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xc802, 0x0000, 0x00d2, 0xb031 }, + { 0x0000, 0xc902, 0x0000, 0x00d2, 0xb031 }, + { 0x0000, 0xc800, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd250, 0x0000, 0x0087, 0xb031 }, + { 0x0000, 0xd252, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xd264, 0x0000, 0x00c0, 0xb031 }, + { 0x0000, 0xce60, 0x0000, 0x00e3, 0xb031 }, + { 0x0000, 0xdf00, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xdbb5, 0x0041, 0xd27f, 0xb037 }, + { 0x0000, 0xddb5, 0x0040, 0x6e3b, 0xb037 }, + { 0x0000, 0xdb93, 0x0000, 0x009e, 0xb033 }, + { 0x0000, 0xdd93, 0x0000, 0x009e, 0xb033 }, + { 0x0000, 0xdb12, 0x0000, 0x00c0, 0xb031 }, + { 0x0000, 0xdd12, 0x0000, 0x00c0, 0xb031 }, + { 0x0000, 0xdb08, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xdd08, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xdb00, 0x0000, 0x00cc, 0xb031 }, + { 0x0000, 0xc203, 0x0000, 0x009c, 0xb031 }, + { 0x0000, 0x0010, 0x0000, 0x0f21, 0x0000 }, +}; + +/* wake the amp (ea00=0x43, c121=0x0b, f109=0xe0) */ +static const struct alc298_razer_blade16_2025_op alc298_razer_blade16_2025_amp_wake[] = { + { 0x0000, 0xea00, 0x0000, 0x0043, 0xb031 }, + { 0x0000, 0xc121, 0x0000, 0x000b, 0xb031 }, + { 0x0000, 0xf109, 0x0000, 0x00e0, 0xb031 }, +}; + +/* park the amp (ea00=0x47, c121=0x0a, f109=0xa0) */ +static const struct alc298_razer_blade16_2025_op alc298_razer_blade16_2025_amp_sleep[] = { + { 0x0000, 0xea00, 0x0000, 0x0047, 0xb031 }, + { 0x0000, 0xc121, 0x0000, 0x000a, 0xb031 }, + { 0x0000, 0xf109, 0x0000, 0x00a0, 0xb031 }, +}; + +/* + * Razer Blade 16 (2025): Realtek ALC298 driving an external smart amp. + * Crossover, voicing and speaker protection live in an external DSP + * reached over a vendor-coef mailbox on NID 0x20; the captured DSP + * programming is in the tables above. NID 0x14, the BIOS-disabled + * tweeter pin, is re-exposed via a pin-config override so the parser + * drives it as a normal internal speaker (shared volume/mute, auto-mute + * on headphone insertion). + * + * alc298_razer_blade16_2025_apply() runs one op-table: a plain coef write + * (strobe == 0) goes through alc_write_coef_idx(), while a mailbox op + * selects a bank and streams four PROC_COEF words (addr/hi/lo/commit) + * into a single coef index (0x23) on NID 0x20 to reach the external DSP + * -- that one-index/many-words form is why it cannot be expressed as an + * alc_process_coef_fw() table (one value per index). + */ +static void alc298_razer_blade16_2025_apply(struct hda_codec *codec, + const struct alc298_razer_blade16_2025_op *seq, + int num) +{ + int i; + + for (i = 0; i < num; i++) { + if (!seq[i].strobe) { + alc_write_coef_idx(codec, seq[i].addr, seq[i].lo); + continue; + } + alc_write_coef_idx(codec, 0x89, seq[i].bank); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_COEF_INDEX, 0x23); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, seq[i].addr); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, seq[i].hi); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, seq[i].lo); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, seq[i].strobe); + } +} + +static void alc298_razer_blade16_2025_pcm_hook(struct hda_pcm_stream *hinfo, + struct hda_codec *codec, + struct snd_pcm_substream *substream, + int action) +{ + /* power the external amp only while a stream is running */ + switch (action) { + case HDA_GEN_PCM_ACT_PREPARE: + alc298_razer_blade16_2025_apply(codec, alc298_razer_blade16_2025_amp_wake, + ARRAY_SIZE(alc298_razer_blade16_2025_amp_wake)); + break; + case HDA_GEN_PCM_ACT_CLEANUP: + alc298_razer_blade16_2025_apply(codec, alc298_razer_blade16_2025_amp_sleep, + ARRAY_SIZE(alc298_razer_blade16_2025_amp_sleep)); + break; + } +} + +static void alc298_fixup_razer_blade16_2025(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + /* + * Pin every output to a fixed converter so the routing does not + * rely on the generic parser's DAC-allocation heuristics: without + * a fixed assignment the parser brings up only one speaker pin and + * leaves the other silent, and the allocation could change between + * kernel versions. Both speaker pins share DAC 0x03 through mixer + * 0x0d: that is the only converter that drives the woofer (NID + * 0x17) on this machine -- feeding it from DAC 0x02 or the digital + * converter 0x06 leaves the woofer silent. Both pins carry the + * same stereo stream and the external amp does the crossover. + * Headphone NID 0x21 keeps DAC 0x02. + */ + static const hda_nid_t preferred_pairs[] = { + 0x21, 0x02, + 0x14, 0x03, + 0x17, 0x03, + 0 + }; + struct alc_spec *spec = codec->spec; + + switch (action) { + case HDA_FIXUP_ACT_PRE_PROBE: + spec->gen.preferred_dacs = preferred_pairs; + break; + case HDA_FIXUP_ACT_PROBE: + spec->gen.pcm_playback_hook = alc298_razer_blade16_2025_pcm_hook; + break; + case HDA_FIXUP_ACT_INIT: + /* + * Wake the amp, program its DSP at boot and on resume, then + * park it. It is woken again for playback and parked + * afterwards from the pcm_playback_hook. + */ + alc298_razer_blade16_2025_apply(codec, alc298_razer_blade16_2025_amp_wake, + ARRAY_SIZE(alc298_razer_blade16_2025_amp_wake)); + alc298_razer_blade16_2025_apply(codec, alc298_razer_blade16_2025_amp_init, + ARRAY_SIZE(alc298_razer_blade16_2025_amp_init)); + alc298_razer_blade16_2025_apply(codec, alc298_razer_blade16_2025_amp_sleep, + ARRAY_SIZE(alc298_razer_blade16_2025_amp_sleep)); + break; + } +} diff -ruN --no-dereference -x .git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c --- a/sound/hda/codecs/realtek/alc269.c 2026-10-11 15:27:50.000000000 +0200 +++ b/sound/hda/codecs/realtek/alc269.c 2026-10-13 14:33:30.000000000 +0200 @@ -3262,6 +3262,36 @@ comp_generic_fixup(cdc, HDA_FIXUP_ACT_PRE_PROBE, bus, acpi_ids[i].hid, match, count); } +static void aw88399_fixup_i2c_two(struct hda_codec *cdc, const struct hda_fixup *fix, int action) +{ + comp_generic_fixup(cdc, action, "i2c", "AWDZ8399", "-%s:00-aw88399-hda.%d", 2); +} + +static void alc287_fixup_legion_16iax10h_aw88399(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + /* + * Force DAC 0x02 for the bass speaker 0x17, as the default 0x06 lacks volume controls. + */ + static const hda_nid_t conn[] = { 0x02 }; + struct alc_spec *spec = codec->spec; + + alc269_fixup_limit_int_mic_boost(codec, fix, action); + alc_fixup_headset_mode_no_hp_mic(codec, fix, action); + alc_fixup_headset_jack(codec, fix, action); + + switch (action) { + case HDA_FIXUP_ACT_PRE_PROBE: + spec->gen.suppress_auto_mic = 1; + snd_hda_override_conn_list(codec, 0x17, ARRAY_SIZE(conn), conn); + break; + case HDA_FIXUP_ACT_BUILD: + spec->gen.multiout.max_channels = 2; + spec->gen.pcm_rec[0]->stream[SNDRV_PCM_STREAM_PLAYBACK].channels_max = 2; + break; + } +} + static void cs35l41_fixup_i2c_two(struct hda_codec *cdc, const struct hda_fixup *fix, int action) { comp_generic_fixup(cdc, action, "i2c", "CSC3551", "-%s:00-cs35l41-hda.%d", 2); @@ -3287,6 +3317,11 @@ comp_generic_fixup(codec, action, "spi", "CSC3551", "-%s:00-cs35l41-hda.%d", 4); } +static void max98390_fixup_i2c_four(struct hda_codec *codec, const struct hda_fixup *fix, int action) +{ + comp_generic_fixup(codec, action, "i2c", "MAX98390", "-%s:00-max98390-hda.%d", 4); +} + static void alc287_fixup_legion_16achg6_speakers(struct hda_codec *cdc, const struct hda_fixup *fix, int action) { @@ -3357,6 +3392,9 @@ /* for alc285_fixup_ideapad_s740_coef() */ #include "../helpers/ideapad_s740.c" +/* for alc298_fixup_razer_blade16_2025() */ +#include "../helpers/razer_blade16_2025.c" + static const struct coef_fw alc256_fixup_set_coef_defaults_coefs[] = { WRITE_COEF(0x10, 0x0020), WRITE_COEF(0x24, 0x0000), WRITE_COEF(0x26, 0x0000), WRITE_COEF(0x29, 0x3000), @@ -4087,6 +4125,7 @@ ALC294_FIXUP_ASUS_GU502_VERBS, ALC294_FIXUP_ASUS_G513_PINS, ALC285_FIXUP_ASUS_G533Z_PINS, + ALC285_FIXUP_ASUS_G733Z_VERBS, ALC285_FIXUP_HP_GPIO_LED, ALC285_FIXUP_HP_MUTE_LED, ALC285_FIXUP_HP_SPECTRE_X360_MUTE_LED, @@ -4100,7 +4139,10 @@ ALC298_FIXUP_SAMSUNG_AMP, ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS, ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS, + ALC298_FIXUP_SAMSUNG_MAX98390_4_AMPS, ALC298_FIXUP_LG_GRAM_STYLE_14, + ALC298_FIXUP_RAZER_BLADE16_2025_PINS, + ALC298_FIXUP_RAZER_BLADE16_2025, ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, ALC295_FIXUP_ASUS_MIC_NO_PRESENCE, @@ -4237,6 +4279,8 @@ ALC236_FIXUP_DELL_HP_POP_NOISE, ALC274_FIXUP_HP_89E9_GPIO, ALC274_FIXUP_HP_VERBS, + ALC287_FIXUP_AW88399_I2C_2, + ALC287_FIXUP_LENOVO_LEGION_AW88399, }; /* A special fixup for Lenovo C940 and Yoga Duet 7; @@ -4281,6 +4325,20 @@ { } }, }, + [ALC298_FIXUP_RAZER_BLADE16_2025_PINS] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x14, 0x90170121 }, /* tweeter as internal speaker, seq 1 */ + { 0x17, 0x90170120 }, /* woofer as internal speaker, seq 0 */ + { } + }, + .chained = true, + .chain_id = ALC298_FIXUP_RAZER_BLADE16_2025, + }, + [ALC298_FIXUP_RAZER_BLADE16_2025] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc298_fixup_razer_blade16_2025, + }, [ALC269_FIXUP_GPIO2] = { .type = HDA_FIXUP_FUNC, .v.func = alc_fixup_gpio2, @@ -5871,6 +5929,10 @@ .type = HDA_FIXUP_FUNC, .v.func = alc298_fixup_lg_gram_style_14 }, + [ALC298_FIXUP_SAMSUNG_MAX98390_4_AMPS] = { + .type = HDA_FIXUP_FUNC, + .v.func = max98390_fixup_i2c_four + }, [ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET] = { .type = HDA_FIXUP_VERBS, .v.verbs = (const struct hda_verb[]) { @@ -6916,6 +6978,29 @@ .chained = true, .chain_id = ALC274_FIXUP_HP_89E9_GPIO, }, + [ALC285_FIXUP_ASUS_G733Z_VERBS] = { + .type = HDA_FIXUP_VERBS, + .v.verbs = (const struct hda_verb[]) { + /* Enables internal speaker */ + {0x17, AC_VERB_SET_CONNECT_SEL, 0x00}, + {0x14, AC_VERB_SET_PIN_WIDGET_CONTROL, 0x40}, + {0x14, AC_VERB_SET_AMP_GAIN_MUTE, 0xb000}, + {0x1e, AC_VERB_SET_PIN_WIDGET_CONTROL, 0x40}, + {0x16, AC_VERB_SET_PIN_WIDGET_CONTROL, 0x00}, + {0x1b, AC_VERB_SET_PIN_WIDGET_CONTROL, 0x00}, + { } + } + }, + [ALC287_FIXUP_AW88399_I2C_2] = { + .type = HDA_FIXUP_FUNC, + .v.func = aw88399_fixup_i2c_two, + }, + [ALC287_FIXUP_LENOVO_LEGION_AW88399] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc287_fixup_legion_16iax10h_aw88399, + .chained = true, + .chain_id = ALC287_FIXUP_AW88399_I2C_2, + }, }; static const struct hda_quirk alc269_fixup_tbl[] = { @@ -7553,6 +7638,7 @@ SND_PCI_QUIRK(0x1043, 0x12a3, "Asus N7691ZM", ALC269_FIXUP_ASUS_N7601ZM), SND_PCI_QUIRK(0x1043, 0x12af, "ASUS UX582ZS", ALC245_FIXUP_CS35L41_SPI_2), SND_PCI_QUIRK(0x1043, 0x12b4, "ASUS B3405CCA / P3405CCA", ALC294_FIXUP_ASUS_CS35L41_SPI_2), + SND_PCI_QUIRK(0x1043, 0x12bf, "ASUS ROG Strix G733ZW", ALC285_FIXUP_ASUS_G733Z_VERBS), SND_PCI_QUIRK(0x1043, 0x12e0, "ASUS X541SA", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x12f0, "ASUS X541UV", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x1313, "Asus K42JZ", ALC269VB_FIXUP_ASUS_MIC_NO_PRESENCE), @@ -7736,6 +7822,11 @@ SND_PCI_QUIRK(0x144d, 0xc876, "Samsung 730QED (NP730QED-KA2US)", ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), SND_PCI_QUIRK(0x144d, 0xca03, "Samsung Galaxy Book2 Pro 360 (NP930QED)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xca06, "Samsung Galaxy Book3 360 (NP730QFG)", ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), + SND_PCI_QUIRK(0x144d, 0xca07, "Samsung Galaxy Book4 Pro 14-inch (NP940XGK)", ALC298_FIXUP_SAMSUNG_MAX98390_4_AMPS), + SND_PCI_QUIRK(0x144d, 0xc890, "Samsung Galaxy Book4 Pro 16 inch (NP960XGK)", ALC298_FIXUP_SAMSUNG_MAX98390_4_AMPS), + SND_PCI_QUIRK(0x144d, 0xc892, "Samsung Galaxy Book4 Pro 360 (NP960QGK)", ALC298_FIXUP_SAMSUNG_MAX98390_4_AMPS), + SND_PCI_QUIRK(0x144d, 0xc1da, "Samsung Galaxy Book5 Pro 360 (NP960QHA)", ALC298_FIXUP_SAMSUNG_MAX98390_4_AMPS), + SND_PCI_QUIRK(0x144d, 0xc1d8, "Samsung Galaxy Book4 Ultra (NP960XGL)", ALC298_FIXUP_SAMSUNG_MAX98390_4_AMPS), SND_PCI_QUIRK(0x144d, 0xc868, "Samsung Galaxy Book2 Pro (NP930XED)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc870, "Samsung Galaxy Book2 Pro (NP950XED)", ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS), SND_PCI_QUIRK(0x144d, 0xc872, "Samsung Galaxy Book2 Pro (NP950XEE)", ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS), @@ -7983,6 +8074,11 @@ SND_PCI_QUIRK(0x17aa, 0x38b8, "Yoga S780-14.5 proX AMD YC Dual", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x17aa, 0x38b9, "Yoga S780-14.5 proX AMD LX Dual", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x17aa, 0x38ba, "Yoga S780-14.5 Air AMD quad YC", ALC287_FIXUP_TAS2781_I2C), + /* Legion R9000P ADR10 shares PCI SSID 17aa:38bb with Yoga S780-14.5 Air AMD quad AAC; + * use codec SSID to distinguish them + */ + HDA_CODEC_QUIRK(0x17aa, 0x3927, "Legion R9000P ADR10", ALC287_FIXUP_LENOVO_LEGION_AW88399), + HDA_CODEC_QUIRK(0x17aa, 0x3928, "Legion R9000P ADR10", ALC287_FIXUP_LENOVO_LEGION_AW88399), SND_PCI_QUIRK(0x17aa, 0x38bb, "Yoga S780-14.5 Air AMD quad AAC", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x17aa, 0x38be, "Yoga S980-14.5 proX YC Dual", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x17aa, 0x38bf, "Yoga S980-14.5 proX LX Dual", ALC287_FIXUP_TAS2781_I2C), @@ -8010,6 +8106,8 @@ SND_PCI_QUIRK(0x17aa, 0x38fc, "Lenovo Yoga Pro 7 15ASH11", ALC287_FIXUP_LENOVO_YOGA_PRO7), SND_PCI_QUIRK(0x17aa, 0x38fd, "ThinkBook plus Gen5 Hybrid", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x17aa, 0x3902, "Lenovo E50-80", ALC269_FIXUP_DMIC_THINKPAD_ACPI), + HDA_CODEC_QUIRK(0x17aa, 0x3906, "Legion Pro 7i 16IAX10H / Y9000P IAX10", ALC287_FIXUP_LENOVO_LEGION_AW88399), + HDA_CODEC_QUIRK(0x17aa, 0x3907, "Legion Pro 7i 16IAX10H / Y9000P IAX10", ALC287_FIXUP_LENOVO_LEGION_AW88399), SND_PCI_QUIRK(0x17aa, 0x390d, "Lenovo Yoga Pro 7 14ASP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x3911, "Lenovo Yoga Pro 7 14IAH10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x3912, "Lenovo Xiaoxin 14 GT", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), @@ -8018,6 +8116,8 @@ SND_PCI_QUIRK(0x17aa, 0x3920, "Yoga S990-16 pro Quad VECO Quad", ALC287_FIXUP_TXNW2781_I2C), SND_PCI_QUIRK(0x17aa, 0x3929, "Thinkbook 13x Gen 5", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD), SND_PCI_QUIRK(0x17aa, 0x392b, "Thinkbook 13x Gen 5", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD), + HDA_CODEC_QUIRK(0x17aa, 0x3938, "Legion Pro 7 16AFR10H", ALC287_FIXUP_LENOVO_LEGION_AW88399), + HDA_CODEC_QUIRK(0x17aa, 0x3939, "Legion Pro 7 16AFR10H", ALC287_FIXUP_LENOVO_LEGION_AW88399), HDA_CODEC_QUIRK(0x17aa, 0x394c, "Lenovo Yoga Slim 7 14AGP11", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x3977, "IdeaPad S210", ALC283_FIXUP_INT_MIC), SND_PCI_QUIRK(0x17aa, 0x3978, "Lenovo B50-70", ALC269_FIXUP_DMIC_THINKPAD_ACPI), @@ -8055,6 +8155,8 @@ SND_PCI_QUIRK(0x19e5, 0x3204, "Huawei MACH-WX9", ALC256_FIXUP_HUAWEI_MACH_WX9_PINS), SND_PCI_QUIRK(0x19e5, 0x320f, "Huawei WRT-WX9 ", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x19e5, 0x3212, "Huawei KLV-WX9 ", ALC256_FIXUP_ACER_HEADSET_MIC), + SND_PCI_QUIRK(0x1a58, 0x300e, "Razer Blade 16 (2025)", + ALC298_FIXUP_RAZER_BLADE16_2025_PINS), SND_PCI_QUIRK(0x1b35, 0x1235, "CZC B20", ALC269_FIXUP_CZC_B20), SND_PCI_QUIRK(0x1b35, 0x1236, "CZC TMI", ALC269_FIXUP_CZC_TMI), SND_PCI_QUIRK(0x1b35, 0x1237, "CZC L101", ALC269_FIXUP_CZC_L101), @@ -8302,6 +8404,7 @@ {.id = ALC298_FIXUP_SAMSUNG_AMP, .name = "alc298-samsung-amp"}, {.id = ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS, .name = "alc298-samsung-amp-v2-2-amps"}, {.id = ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS, .name = "alc298-samsung-amp-v2-4-amps"}, + {.id = ALC298_FIXUP_SAMSUNG_MAX98390_4_AMPS, .name = "alc298-samsung-max98390-4-amps"}, {.id = ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, .name = "alc256-samsung-headphone"}, {.id = ALC255_FIXUP_XIAOMI_HEADSET_MIC, .name = "alc255-xiaomi-headset"}, {.id = ALC274_FIXUP_HP_MIC, .name = "alc274-hp-mic-detect"}, @@ -8320,6 +8423,7 @@ {.id = ALC2XX_FIXUP_HEADSET_MIC, .name = "alc2xx-fixup-headset-mic"}, {.id = ALC245_FIXUP_BASS_HP_DAC, .name = "alc245-fixup-bass-hp-dac"}, {.id = ALC256_FIXUP_HONOR_MRB_XXX_M1020_AUDIO, .name = "alc256-honor-mrb-xxx-m1020-audio"}, + {.id = ALC287_FIXUP_LENOVO_LEGION_AW88399, .name = "alc287-lenovo-legion-aw88399"}, {} }; #define ALC225_STANDARD_PINS \ diff -ruN --no-dereference -x .git a/sound/hda/codecs/side-codecs/Kconfig b/sound/hda/codecs/side-codecs/Kconfig --- a/sound/hda/codecs/side-codecs/Kconfig 2026-10-11 15:27:50.000000000 +0200 +++ b/sound/hda/codecs/side-codecs/Kconfig 2026-10-13 14:33:30.000000000 +0200 @@ -13,6 +13,24 @@ Documentation/dev-tools/kunit/. If in doubt, say "N". +config SND_HDA_SCODEC_AW88399 + tristate + select SND_HDA_GENERIC + +config SND_HDA_SCODEC_AW88399_I2C + tristate "Build AW88399 HD-audio side codec support for I2C Bus" + depends on I2C + depends on ACPI + depends on SND_SOC + select SND_HDA_SCODEC_AW88399 + select SND_SOC_AW88399_LIB + help + Say Y or M here to include AW88399 I2C HD-audio side codec support + in snd-hda-intel driver, such as ALC287. + +comment "Set to Y if you want auto-loading the side codec driver" + depends on SND_HDA=y && SND_HDA_SCODEC_AW88399_I2C=m + config SND_HDA_SCODEC_CS35L41 tristate select SND_HDA_GENERIC @@ -143,3 +161,22 @@ comment "Set to Y if you want auto-loading the side codec driver" depends on SND_HDA=y && SND_HDA_SCODEC_TAS2781_SPI=m + +config SND_HDA_SCODEC_MAX98390 + tristate + select SND_HDA_GENERIC + select SND_HDA_SCODEC_COMPONENT + +config SND_HDA_SCODEC_MAX98390_I2C + tristate "Build MAX98390 HD-audio side codec support for I2C Bus" + depends on I2C + depends on ACPI + depends on SND_SOC + select SND_SOC_MAX98390 + select SND_HDA_SCODEC_MAX98390 + help + Say Y or M here to include MAX98390 I2C HD-audio side codec support + in snd-hda-intel driver, such as ALC298. + +comment "Set to Y if you want auto-loading the side codec driver" + depends on SND_HDA=y && SND_HDA_SCODEC_MAX98390_I2C=m diff -ruN --no-dereference -x .git a/sound/hda/codecs/side-codecs/Makefile b/sound/hda/codecs/side-codecs/Makefile --- a/sound/hda/codecs/side-codecs/Makefile 2026-10-11 15:27:50.000000000 +0200 +++ b/sound/hda/codecs/side-codecs/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -3,6 +3,8 @@ snd-hda-cirrus-scodec-y := cirrus_scodec.o snd-hda-cirrus-scodec-test-y := cirrus_scodec_test.o +snd-hda-scodec-aw88399-y := aw88399_hda.o +snd-hda-scodec-aw88399-i2c-y := aw88399_hda_i2c.o snd-hda-scodec-cs35l41-y := cs35l41_hda.o cs35l41_hda_property.o snd-hda-scodec-cs35l41-i2c-y := cs35l41_hda_i2c.o snd-hda-scodec-cs35l41-spi-y := cs35l41_hda_spi.o @@ -13,9 +15,13 @@ snd-hda-scodec-tas2781-y := tas2781_hda.o snd-hda-scodec-tas2781-i2c-y := tas2781_hda_i2c.o snd-hda-scodec-tas2781-spi-y := tas2781_hda_spi.o +snd-hda-scodec-max98390-y := max98390_hda.o max98390_hda_filters.o +snd-hda-scodec-max98390-i2c-y := max98390_hda_i2c.o obj-$(CONFIG_SND_HDA_CIRRUS_SCODEC) += snd-hda-cirrus-scodec.o obj-$(CONFIG_SND_HDA_CIRRUS_SCODEC_KUNIT_TEST) += snd-hda-cirrus-scodec-test.o +obj-$(CONFIG_SND_HDA_SCODEC_AW88399) += snd-hda-scodec-aw88399.o +obj-$(CONFIG_SND_HDA_SCODEC_AW88399_I2C) += snd-hda-scodec-aw88399-i2c.o obj-$(CONFIG_SND_HDA_SCODEC_CS35L41) += snd-hda-scodec-cs35l41.o obj-$(CONFIG_SND_HDA_SCODEC_CS35L41_I2C) += snd-hda-scodec-cs35l41-i2c.o obj-$(CONFIG_SND_HDA_SCODEC_CS35L41_SPI) += snd-hda-scodec-cs35l41-spi.o @@ -26,3 +32,5 @@ obj-$(CONFIG_SND_HDA_SCODEC_TAS2781) += snd-hda-scodec-tas2781.o obj-$(CONFIG_SND_HDA_SCODEC_TAS2781_I2C) += snd-hda-scodec-tas2781-i2c.o obj-$(CONFIG_SND_HDA_SCODEC_TAS2781_SPI) += snd-hda-scodec-tas2781-spi.o +obj-$(CONFIG_SND_HDA_SCODEC_MAX98390) += snd-hda-scodec-max98390.o +obj-$(CONFIG_SND_HDA_SCODEC_MAX98390_I2C) += snd-hda-scodec-max98390-i2c.o diff -ruN --no-dereference -x .git a/sound/hda/codecs/side-codecs/aw88399_hda.c b/sound/hda/codecs/side-codecs/aw88399_hda.c --- a/sound/hda/codecs/side-codecs/aw88399_hda.c 1970-01-01 01:00:00.000000000 +0100 +++ b/sound/hda/codecs/side-codecs/aw88399_hda.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,372 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// AW88399 HDA side codec driver +// +// Based on cs35l41_hda.c and aw88399.c +// + +#include +#include +#include +#include +#include +#include +#include +#include +#include "../generic.h" +#include "hda_component.h" +#include "aw88399_hda.h" + +#define AW88399_HDA_I2C_BASE_ADDR 0x34 + +static void aw88399_hda_playback_hook(struct device *dev, int action) +{ + struct aw88399_hda *aw88399 = dev_get_drvdata(dev); + struct aw88399 *core = aw88399->core; + int ret = 0; + + dev_dbg(aw88399->dev, "Playback action: %d\n", action); + + switch (action) { + case HDA_GEN_PCM_ACT_OPEN: + pm_runtime_get_sync(dev); + aw88399->playing = true; + break; + case HDA_GEN_PCM_ACT_PREPARE: + if (core) + aw88399_start(core, AW88399_SYNC_START); + break; + case HDA_GEN_PCM_ACT_CLEANUP: + if (aw88399->aw_dev) + ret = aw88399_stop(aw88399->aw_dev); + if (ret) + dev_err(aw88399->dev, "Failed to stop amplifier: %d\n", ret); + break; + case HDA_GEN_PCM_ACT_CLOSE: + if (aw88399->aw_dev) + aw88399_stop(aw88399->aw_dev); + aw88399->playing = false; + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); + break; + default: + dev_warn(aw88399->dev, "Unsupported action: %d\n", action); + break; + } +} + +static int aw88399_hda_bind(struct device *dev, struct device *master, void *master_data) +{ + struct aw88399_hda *aw88399 = dev_get_drvdata(dev); + struct hda_component_parent *parent = master_data; + struct hda_component *comp; + + comp = hda_component_from_index(parent, aw88399->index); + if (!comp) + return -EINVAL; + + if (comp->dev) + return -EBUSY; + + comp->dev = dev; + + strscpy(comp->name, dev_name(dev), sizeof(comp->name)); + + comp->playback_hook = aw88399_hda_playback_hook; + + dev_info(aw88399->dev, + "AW88399 Bound - SSID: %s, channel: %d\n", + aw88399->acpi_subsystem_id, aw88399->channel); + + return 0; +} + +static void aw88399_hda_unbind(struct device *dev, struct device *master, void *master_data) +{ + struct aw88399_hda *aw88399 = dev_get_drvdata(dev); + struct hda_component_parent *parent = master_data; + struct hda_component *comp; + + comp = hda_component_from_index(parent, aw88399->index); + if (comp && (comp->dev == dev)) + memset(comp, 0, sizeof(*comp)); + + dev_dbg(aw88399->dev, "Unbound from HDA codec\n"); +} + +static const struct component_ops aw88399_hda_comp_ops = { + .bind = aw88399_hda_bind, + .unbind = aw88399_hda_unbind, +}; + +static int aw88399_hda_index_from_i2c(struct aw88399_hda *aw88399) +{ + return to_i2c_client(aw88399->dev)->addr - AW88399_HDA_I2C_BASE_ADDR; +} + +static int aw88399_hda_init(struct aw88399_hda *aw88399) +{ + struct device *dev = aw88399->dev; + struct i2c_client *i2c = to_i2c_client(dev); + struct aw88399 *core; + int ret; + + core = devm_kzalloc(dev, sizeof(*core), GFP_KERNEL); + if (!core) + return -ENOMEM; + + mutex_init(&core->lock); + core->reset_gpio = aw88399->reset_gpio; + core->regmap = aw88399->regmap; + core->bsts_unreliable = aw88399->bsts_unreliable; + + aw88399_hw_reset(core); + + ret = aw88399_init(core, i2c, aw88399->regmap); + if (ret) + return ret; + + /* Set channel BEFORE loading firmware so ACF parser sees correct value */ + if (core->aw_pa) + aw88399_dev_set_channel(core, aw88399->channel); + + ret = aw88399_request_firmware_file(core); + if (ret) + return ret; + + aw88399->core = core; + aw88399->aw_dev = core->aw_pa; + + return 0; +} + +static int aw88399_swap_channels(struct aw88399_hda *aw88399) +{ + /* + * Certain Lenovo Legion laptops have their + * I2C wiring reversed: 0x34 is physically the right speaker, + * 0x35 is the left. Swap channels to correct L/R assignment. + * This is a model-specific hardware wiring issue, not a driver bug. + */ + aw88399->channel = 1 - aw88399->channel; + dev_dbg(aw88399->dev, + "Channel swap applied: index %d -> channel %d\n", + aw88399->index, aw88399->channel); + return 0; +} + +static int aw88399_skip_bsts_check(struct aw88399_hda *aw88399) +{ + /* + * BSTS (boost-finished) status bit does not reliably report on + * some hardware. On certain Lenovo Legion laptops, both amps + * report BSTS=0 (boost not finished) during normal playback + * despite clean audio output. Skip BSTS in the startup status + * check to avoid false init failures. + */ + aw88399->bsts_unreliable = true; + dev_dbg(aw88399->dev, "BSTS status check disabled\n"); + return 0; +} + +static int aw88399_apply_legion_quirks(struct aw88399_hda *aw88399) +{ + aw88399_swap_channels(aw88399); + aw88399_skip_bsts_check(aw88399); + return 0; +} + +struct aw88399_prop_model { + const char *ssid; + int (*apply_prop)(struct aw88399_hda *aw88399); +}; + +static const struct aw88399_prop_model aw88399_prop_model_table[] = { + { "17AA3906", aw88399_apply_legion_quirks }, + { "17AA3907", aw88399_apply_legion_quirks }, + { "17AA3927", aw88399_apply_legion_quirks }, + { "17AA3928", aw88399_apply_legion_quirks }, + { "17AA3938", aw88399_apply_legion_quirks }, + { "17AA3939", aw88399_apply_legion_quirks }, + { } +}; + +static int aw88399_hda_acpi_probe(struct aw88399_hda *aw88399) +{ + struct acpi_device *adev; + struct device *physdev; + const char *sub; + const struct aw88399_prop_model *model; + + aw88399->index = aw88399_hda_index_from_i2c(aw88399); + aw88399->channel = aw88399->index; + aw88399->acpi_subsystem_id = NULL; + + adev = acpi_dev_get_first_match_dev("AWDZ8399", NULL, -1); + if (!adev) { + dev_err(aw88399->dev, "Failed to find an ACPI device for AWDZ8399\n"); + return -ENODEV; + } + + physdev = get_device(acpi_get_first_physical_node(adev)); + acpi_dev_put(adev); + if (!physdev) + return -ENODEV; + + sub = acpi_get_subsystem_id(ACPI_HANDLE(physdev)); + put_device(physdev); + if (IS_ERR_OR_NULL(sub)) + return 0; + + aw88399->acpi_subsystem_id = devm_kstrdup(aw88399->dev, sub, GFP_KERNEL); + kfree(sub); + if (!aw88399->acpi_subsystem_id) + return -ENOMEM; + + for (model = aw88399_prop_model_table; model->ssid; model++) { + if (!strcasecmp(model->ssid, aw88399->acpi_subsystem_id)) { + dev_info(aw88399->dev, + "Applying properties for SSID %s\n", + aw88399->acpi_subsystem_id); + return model->apply_prop(aw88399); + } + } + + return 0; +} + +int aw88399_hda_probe(struct device *dev, struct regmap *regmap) +{ + struct aw88399_hda *aw88399; + int ret; + + aw88399 = devm_kzalloc(dev, sizeof(*aw88399), GFP_KERNEL); + if (!aw88399) + return -ENOMEM; + + if (IS_ERR(regmap)) + return dev_err_probe(dev, PTR_ERR(regmap), "Failed to obtain regmap\n"); + + aw88399->dev = dev; + aw88399->regmap = regmap; + dev_set_drvdata(dev, aw88399); + + aw88399->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW); + if (IS_ERR(aw88399->reset_gpio)) + return dev_err_probe(dev, PTR_ERR(aw88399->reset_gpio), + "Failed to get reset GPIO\n"); + + ret = aw88399_hda_acpi_probe(aw88399); + if (ret) + return dev_err_probe(dev, ret, "ACPI probe failed\n"); + + ret = aw88399_hda_init(aw88399); + if (ret) + return dev_err_probe(dev, ret, "Chip initialization failed\n"); + + /* Enable runtime PM */ + pm_runtime_set_autosuspend_delay(dev, 3000); + pm_runtime_use_autosuspend(dev); + pm_runtime_mark_last_busy(dev); + pm_runtime_set_active(dev); + pm_runtime_enable(dev); + + ret = component_add(dev, &aw88399_hda_comp_ops); + if (ret) { + pm_runtime_disable(dev); + return dev_err_probe(dev, ret, "Failed to register component\n"); + } + + dev_info(dev, "AW88399 HDA side codec registered successfully\n"); + + return 0; +} +EXPORT_SYMBOL_NS_GPL(aw88399_hda_probe, "SND_HDA_SCODEC_AW88399"); + +void aw88399_hda_remove(struct device *dev) +{ + struct aw88399_hda *aw88399 = dev_get_drvdata(dev); + + pm_runtime_disable(dev); + + if (aw88399->aw_dev) + aw88399_stop(aw88399->aw_dev); + + component_del(dev, &aw88399_hda_comp_ops); + + dev_dbg(aw88399->dev, "AW88399 HDA side codec removed\n"); +} +EXPORT_SYMBOL_NS_GPL(aw88399_hda_remove, "SND_HDA_SCODEC_AW88399"); + +static int aw88399_hda_runtime_suspend(struct device *dev) +{ + struct aw88399_hda *aw88399 = dev_get_drvdata(dev); + + dev_dbg(aw88399->dev, "Runtime suspend\n"); + + if (aw88399->aw_dev && aw88399->playing) + aw88399_stop(aw88399->aw_dev); + + return 0; +} + +static int aw88399_hda_runtime_resume(struct device *dev) +{ + struct aw88399_hda *aw88399 = dev_get_drvdata(dev); + + dev_dbg(aw88399->dev, "Runtime resume\n"); + + if (aw88399->core && aw88399->aw_dev && aw88399->playing) + aw88399_start(aw88399->core, AW88399_SYNC_START); + + return 0; +} + +static int aw88399_hda_system_suspend(struct device *dev) +{ + struct aw88399_hda *aw88399 = dev_get_drvdata(dev); + int ret; + + dev_dbg(aw88399->dev, "System suspend\n"); + + if (aw88399->aw_dev && aw88399->playing) + aw88399_stop(aw88399->aw_dev); + + if (aw88399->core) + aw88399->core->fw_needs_reload = true; + + ret = pm_runtime_force_suspend(dev); + if (ret) + dev_err(aw88399->dev, "Runtime force suspend failed: %d\n", ret); + + return ret; +} + +static int aw88399_hda_system_resume(struct device *dev) +{ + struct aw88399_hda *aw88399 = dev_get_drvdata(dev); + int ret; + + dev_dbg(aw88399->dev, "System resume\n"); + + if (aw88399->aw_dev) + aw88399_hw_reset(aw88399->core); + + ret = pm_runtime_force_resume(dev); + if (ret) + dev_err(aw88399->dev, "Runtime force resume failed: %d\n", ret); + + return ret; +} + +const struct dev_pm_ops aw88399_hda_pm_ops = { + RUNTIME_PM_OPS(aw88399_hda_runtime_suspend, aw88399_hda_runtime_resume, NULL) + SYSTEM_SLEEP_PM_OPS(aw88399_hda_system_suspend, aw88399_hda_system_resume) +}; +EXPORT_SYMBOL_NS_GPL(aw88399_hda_pm_ops, "SND_HDA_SCODEC_AW88399"); + +MODULE_DESCRIPTION("AW88399 HDA driver"); +MODULE_AUTHOR("Yakov Till "); +MODULE_AUTHOR("Marco Giunta "); +MODULE_LICENSE("GPL"); +MODULE_FIRMWARE("aw88399_acf.bin"); diff -ruN --no-dereference -x .git a/sound/hda/codecs/side-codecs/aw88399_hda.h b/sound/hda/codecs/side-codecs/aw88399_hda.h --- a/sound/hda/codecs/side-codecs/aw88399_hda.h 1970-01-01 01:00:00.000000000 +0100 +++ b/sound/hda/codecs/side-codecs/aw88399_hda.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * + * AW88399 HDA side codec driver + */ + +#ifndef __AW88399_HDA_H__ +#define __AW88399_HDA_H__ + +#include +#include +#include + +struct aw88399; +struct aw_device; + +struct aw88399_hda { + struct device *dev; + struct regmap *regmap; + struct gpio_desc *reset_gpio; + struct aw_device *aw_dev; + struct aw88399 *core; + bool bsts_unreliable; + + const char *acpi_subsystem_id; + int index; + int channel; + + bool playing; +}; + +int aw88399_hda_probe(struct device *dev, struct regmap *regmap); +void aw88399_hda_remove(struct device *dev); + +extern const struct dev_pm_ops aw88399_hda_pm_ops; + +#endif /* __AW88399_HDA_H__ */ diff -ruN --no-dereference -x .git a/sound/hda/codecs/side-codecs/aw88399_hda_i2c.c b/sound/hda/codecs/side-codecs/aw88399_hda_i2c.c --- a/sound/hda/codecs/side-codecs/aw88399_hda_i2c.c 1970-01-01 01:00:00.000000000 +0100 +++ b/sound/hda/codecs/side-codecs/aw88399_hda_i2c.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// AW88399 HDA I2C driver +// +// Based on cs35l41_hda_i2c.c +// + +#include +#include +#include + +#include "aw88399_hda.h" + +static int aw88399_hda_i2c_probe(struct i2c_client *clt) +{ + if (!strstr(dev_name(&clt->dev), "AWDZ8399")) + return -ENODEV; + + return aw88399_hda_probe(&clt->dev, + devm_regmap_init_i2c(clt, &aw88399_remap_config)); +} + +static void aw88399_hda_i2c_remove(struct i2c_client *clt) +{ + aw88399_hda_remove(&clt->dev); +} + +static const struct i2c_device_id aw88399_hda_i2c_id[] = { + { .name = "aw88399-hda" }, + { } +}; + +static const struct acpi_device_id aw88399_acpi_hda_match[] = { + { "AWDZ8399", 0 }, + { } +}; +MODULE_DEVICE_TABLE(acpi, aw88399_acpi_hda_match); + +static struct i2c_driver aw88399_hda_i2c_driver = { + .driver = { + .name = "aw88399-hda", + .acpi_match_table = aw88399_acpi_hda_match, + .pm = &aw88399_hda_pm_ops, + }, + .probe = aw88399_hda_i2c_probe, + .remove = aw88399_hda_i2c_remove, + .id_table = aw88399_hda_i2c_id, +}; +module_i2c_driver(aw88399_hda_i2c_driver); + +MODULE_DESCRIPTION("HDA AW88399 I2C driver"); +MODULE_IMPORT_NS("SND_HDA_SCODEC_AW88399"); +MODULE_AUTHOR("Yakov Till "); +MODULE_LICENSE("GPL"); diff -ruN --no-dereference -x .git a/sound/hda/codecs/side-codecs/max98390_hda.c b/sound/hda/codecs/side-codecs/max98390_hda.c --- a/sound/hda/codecs/side-codecs/max98390_hda.c 1970-01-01 01:00:00.000000000 +0100 +++ b/sound/hda/codecs/side-codecs/max98390_hda.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// MAX98390 HDA driver +// + +#include +#include +#include +#include +#include "hda_local.h" +#include "hda_component.h" +#include "../generic.h" +#include "max98390_hda.h" +#include "max98390_hda_filters.h" +#include "../../../soc/codecs/max98390.h" + +static void max98390_hda_playback_hook(struct device *dev, int action) +{ + struct max98390_hda_priv *priv = dev_get_drvdata(dev); + int ret; + + switch (action) { + case HDA_GEN_PCM_ACT_OPEN: + + /* Enable global and speaker amp */ + ret = regmap_write(priv->regmap, MAX98390_R23FF_GLOBAL_EN, 0x01); + if (ret < 0) + dev_err(dev, "Failed to write GLOBAL_EN: %d\n", ret); + + ret = regmap_write(priv->regmap, MAX98390_R203A_AMP_EN, 0x81); + if (ret < 0) + dev_err(dev, "Failed to write AMP_EN: %d\n", ret); + + break; + + case HDA_GEN_PCM_ACT_CLOSE: + /* Disable speaker amp and global */ + regmap_write(priv->regmap, MAX98390_R203A_AMP_EN, 0x80); + regmap_write(priv->regmap, MAX98390_R23FF_GLOBAL_EN, 0x00); + break; + + default: + break; + } +} + +static int max98390_hda_bind(struct device *dev, struct device *master, void *master_data) +{ + struct max98390_hda_priv *priv = dev_get_drvdata(dev); + struct hda_component_parent *parent = master_data; + struct hda_component *comp; + + comp = hda_component_from_index(parent, priv->index); + if (!comp) + return -EINVAL; + + comp->dev = dev; + strscpy(comp->name, dev_name(dev), sizeof(comp->name)); + comp->playback_hook = max98390_hda_playback_hook; + + dev_info(dev, "MAX98390 HDA component bound (index %d)\n", priv->index); + + return 0; +} + +static void max98390_hda_unbind(struct device *dev, struct device *master, void *master_data) +{ + struct max98390_hda_priv *priv = dev_get_drvdata(dev); + struct hda_component_parent *parent = master_data; + struct hda_component *comp; + + comp = hda_component_from_index(parent, priv->index); + if (comp && comp->dev == dev) { + comp->dev = NULL; + memset(comp->name, 0, sizeof(comp->name)); + comp->playback_hook = NULL; + } + + dev_info(dev, "MAX98390 HDA component unbound\n"); +} + +static const struct component_ops max98390_hda_comp_ops = { + .bind = max98390_hda_bind, + .unbind = max98390_hda_unbind, +}; + +static int max98390_hda_init(struct max98390_hda_priv *priv) +{ + int ret; + unsigned int reg; + + /* Check device ID */ + ret = regmap_read(priv->regmap, MAX98390_R24FF_REV_ID, ®); + if (ret < 0) { + return ret; + } + + /* Software reset */ + ret = regmap_write(priv->regmap, MAX98390_SOFTWARE_RESET, 0x01); + if (ret < 0) { + return ret; + } + msleep(20); + + /* Basic register initialization (minimal setup for HDA) */ + regmap_write(priv->regmap, MAX98390_CLK_MON, 0x6f); + regmap_write(priv->regmap, MAX98390_DAT_MON, 0x00); + regmap_write(priv->regmap, MAX98390_PWR_GATE_CTL, 0x00); + regmap_write(priv->regmap, MAX98390_PCM_RX_EN_A, 0x03); + regmap_write(priv->regmap, MAX98390_ENV_TRACK_VOUT_HEADROOM, 0x0e); + regmap_write(priv->regmap, MAX98390_BOOST_BYPASS1, 0x46); + regmap_write(priv->regmap, MAX98390_FET_SCALING3, 0x03); + + /* PCM/I2S configuration - CRITICAL for correct audio format */ + /* 0xC0 = I2S mode, 32-bit samples (standard for HDA) */ + regmap_write(priv->regmap, MAX98390_PCM_MODE_CFG, 0xc0); + regmap_write(priv->regmap, MAX98390_PCM_MASTER_MODE, 0x1c); + regmap_write(priv->regmap, MAX98390_PCM_CLK_SETUP, 0x44); + regmap_write(priv->regmap, MAX98390_PCM_SR_SETUP, 0x08); + + /* RESET EN - Write 0x00 to 0x23FF */ + regmap_write(priv->regmap, MAX98390_R23FF_GLOBAL_EN, 0x00); + + /* Wait 50ms */ + msleep(50); + + /* RESET SPK_EN - Write 0x80 to 0x203A */ + regmap_write(priv->regmap, MAX98390_R203A_AMP_EN, 0x80); + + /* RESET DSP_GLOBAL_EN - Write 0x00 to 0x23E1 */ + regmap_write(priv->regmap, MAX98390_R23E1_DSP_GLOBAL_EN, 0x00); + + /* Step 6: Write non-DSM registers (0x2000-0x2084) - done in configure_filters */ + /* Step 7: Write DSM registers (0x2100-0x23E0) - done in configure_filters */ + /* Step 8-10: Enable DSP and amp - done in configure_filters */ + max98390_configure_filters(priv); + + return 0; +} + +int max98390_hda_probe(struct device *dev, const char *device_name, + int id, int irq, struct regmap *regmap, + enum max98390_hda_bus_type bus_type, int i2c_addr) +{ + struct max98390_hda_priv *priv; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->dev = dev; + priv->regmap = regmap; + priv->bus_type = bus_type; + priv->irq = irq; + priv->index = id; + priv->i2c_addr = i2c_addr; + dev_set_drvdata(dev, priv); + + ret = max98390_hda_init(priv); + if (ret) + return ret; + + ret = component_add(dev, &max98390_hda_comp_ops); + if (ret) { + return ret; + } + + return 0; +} +EXPORT_SYMBOL_NS_GPL(max98390_hda_probe, "SND_HDA_SCODEC_MAX98390"); + +void max98390_hda_remove(struct device *dev) +{ + struct max98390_hda_priv *priv = dev_get_drvdata(dev); + + component_del(dev, &max98390_hda_comp_ops); + + if (priv && priv->regmap) { + /* Disable amp on removal */ + regmap_write(priv->regmap, MAX98390_R203A_AMP_EN, 0x80); + } +} +EXPORT_SYMBOL_NS_GPL(max98390_hda_remove, "SND_HDA_SCODEC_MAX98390"); + +static int max98390_hda_runtime_suspend(struct device *dev) +{ + struct max98390_hda_priv *priv = dev_get_drvdata(dev); + + regmap_write(priv->regmap, MAX98390_R203A_AMP_EN, 0x80); + regcache_cache_only(priv->regmap, true); + regcache_mark_dirty(priv->regmap); + + return 0; +} + +static int max98390_hda_runtime_resume(struct device *dev) +{ + struct max98390_hda_priv *priv = dev_get_drvdata(dev); + + regcache_cache_only(priv->regmap, false); + regcache_sync(priv->regmap); + + return 0; +} + +const struct dev_pm_ops max98390_hda_pm_ops = { + RUNTIME_PM_OPS(max98390_hda_runtime_suspend, max98390_hda_runtime_resume, NULL) +}; +EXPORT_SYMBOL_NS_GPL(max98390_hda_pm_ops, "SND_HDA_SCODEC_MAX98390"); + +MODULE_DESCRIPTION("HDA MAX98390 side codec library"); +MODULE_AUTHOR("Kevin Cuperus "); +MODULE_LICENSE("GPL"); diff -ruN --no-dereference -x .git a/sound/hda/codecs/side-codecs/max98390_hda.h b/sound/hda/codecs/side-codecs/max98390_hda.h --- a/sound/hda/codecs/side-codecs/max98390_hda.h 1970-01-01 01:00:00.000000000 +0100 +++ b/sound/hda/codecs/side-codecs/max98390_hda.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * MAX98390 HDA audio driver + */ + +#ifndef __MAX98390_HDA_H__ +#define __MAX98390_HDA_H__ + +#include +#include + +enum max98390_hda_bus_type { + MAX98390_HDA_I2C, +}; + +struct max98390_hda_priv { + struct device *dev; + struct regmap *regmap; + enum max98390_hda_bus_type bus_type; + int irq; + int index; + const char *acpi_subsystem_id; + int i2c_addr; /* I2C address for speaker identification */ +}; + +int max98390_hda_probe(struct device *dev, const char *device_name, + int id, int irq, struct regmap *regmap, + enum max98390_hda_bus_type bus_type, int i2c_addr); +void max98390_hda_remove(struct device *dev); + +extern const struct dev_pm_ops max98390_hda_pm_ops; + +#endif /* __MAX98390_HDA_H__ */ diff -ruN --no-dereference -x .git a/sound/hda/codecs/side-codecs/max98390_hda_filters.c b/sound/hda/codecs/side-codecs/max98390_hda_filters.c --- a/sound/hda/codecs/side-codecs/max98390_hda_filters.c 1970-01-01 01:00:00.000000000 +0100 +++ b/sound/hda/codecs/side-codecs/max98390_hda_filters.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,236 @@ +// High-pass filter configuration for MAX98390 HDA driver +// This file contains the filter configuration functions + +// Based on typical 4-speaker configurations +// Tweeters are typically at lower I2C addresses +// Woofers are typically at higher I2C addresses +// This pattern may vary by manufacturer + +#include +#include +#include "max98390_hda.h" +#include "max98390_hda_filters.h" +#include "../../../soc/codecs/max98390.h" + +void max98390_configure_filters(struct max98390_hda_priv *priv) +{ + unsigned int i2c_addr = priv->i2c_addr; + bool is_tweeter = false; + int pcm_channel = 0; + + // Identify speaker based on I2C address + // Configure PCM input channel for stereo separation + switch (i2c_addr) { + case 0x38: + is_tweeter = false; + pcm_channel = 0; // Left channel + break; + case 0x39: + is_tweeter = false; + pcm_channel = 1; // Right channel + break; + case 0x3C: + is_tweeter = true; + pcm_channel = 0; // Left channel + break; + case 0x3D: + is_tweeter = true; + pcm_channel = 1; // Right channel + break; + default: + dev_warn(priv->dev, "Unknown speaker at 0x%02x - defaulting to LEFT CH0\n", i2c_addr); + is_tweeter = false; + pcm_channel = 0; + } + + // Configure PCM input channel for stereo separation + // MAX98390_PCM_CH_SRC_1 register bits [3:0] = speaker source + // 0 = PCM Channel 0 (LEFT), 1 = PCM Channel 1 (RIGHT) + regmap_write(priv->regmap, 0x2021, pcm_channel); + + // Load appropriate DSM firmware for woofers AND tweeters + // Tweeters get high-pass filter, woofers get full-range with small boost for bass + max98390_configure_high_pass_filter(priv, 3000, is_tweeter); +} + +// Complete DSM firmware blobs from Google Redrix +// These firmwares configure ALL DSM registers (0x2050-0x23E0) +// Total: 913 bytes each +#define MAX98390_DSM_START_ADDR 0x2050 +#define MAX98390_DSM_PARAM_SIZE 913 + +// Woofer DSM firmware (for 0x38, 0x39) +static const u8 max98390_dsm_firmware_woofer[] = { + 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1F, // 0x2050 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2060 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0E, 0x80, 0x07, 0x07, 0x01, 0x00, 0x4A, 0x2B, 0x08, 0x00, // 0x2070 + 0x03, 0x03, 0x07, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2080 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2090 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20A0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20B0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20C0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20D0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20E0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20F0 + 0x00, 0xd0, 0x95, 0x0e, 0x00, 0x61, 0xd4, 0xe2, 0x00, 0xd0, 0x95, 0x0e, 0x00, 0x75, 0xf4, 0xe2, // 0x2100 + 0x00, 0xb4, 0x4b, 0x0d, 0x00, 0x0a, 0x10, 0x00, 0x00, 0x15, 0x20, 0x00, 0x00, 0x0a, 0x10, 0x00, // 0x2110 + 0x00, 0x75, 0xf4, 0xe2, 0x00, 0xb4, 0x4b, 0x0d, 0x00, 0xe5, 0xec, 0x0f, 0x00, 0x35, 0x26, 0xe0, // 0x2120 + 0x00, 0xe5, 0xec, 0x0f, 0x00, 0x4c, 0x26, 0xe0, 0x00, 0xe1, 0xd9, 0x0f, 0x00, 0x00, 0x00, 0x10, // 0x2130 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2140 + 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2150 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2160 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, // 0x2170 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x58, 0x00, // 0x2180 + 0x00, 0xc8, 0xb0, 0x00, 0x00, 0x64, 0x58, 0x00, 0x00, 0x3d, 0x5b, 0xe7, 0x00, 0x54, 0x06, 0x0a, // 0x2190 + 0x00, 0xc8, 0xa3, 0x10, 0x00, 0x29, 0x4d, 0xe5, 0x00, 0x32, 0x42, 0x0c, 0x00, 0x29, 0x4d, 0xe5, // 0x21a0 + 0x00, 0xfa, 0xe5, 0x0c, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x21b0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x56, 0xc5, 0x0f, 0x00, 0x7b, 0xd7, 0xe1, // 0x21c0 + 0x00, 0x77, 0xa7, 0x0e, 0x00, 0xe0, 0xbd, 0xe1, 0x00, 0x32, 0x53, 0x0e, 0x00, 0x00, 0x00, 0x00, // 0x21d0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x21e0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x21f0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0xfb, 0x7e, 0x0f, 0x00, 0x0b, 0x02, 0xe1, 0x00, 0xfb, 0x7e, 0x0f, // 0x2200 + 0x00, 0xd5, 0x17, 0xe1, 0x00, 0xc0, 0x13, 0x0f, 0x00, 0xe5, 0x0a, 0x00, 0x00, 0xcb, 0x15, 0x00, // 0x2210 + 0x00, 0xe5, 0x0a, 0x00, 0x00, 0xd5, 0x17, 0xe1, 0x00, 0xc0, 0x13, 0x0f, 0x00, 0xf5, 0xdb, 0x0e, // 0x2220 + 0x00, 0x17, 0x48, 0xe2, 0x00, 0xf5, 0xdb, 0x0e, 0x00, 0xef, 0x5c, 0xe2, 0x00, 0xc1, 0xcc, 0x0d, // 0x2230 + 0x00, 0xf5, 0xdb, 0x0e, 0x00, 0x17, 0x48, 0xe2, 0x00, 0xf5, 0xdb, 0x0e, 0x00, 0xef, 0x5c, 0xe2, // 0x2240 + 0x00, 0xc1, 0xcc, 0x0d, 0x00, 0xf5, 0xdb, 0x0e, 0x00, 0x17, 0x48, 0xe2, 0x00, 0xf5, 0xdb, 0x0e, // 0x2250 + 0x00, 0xef, 0x5c, 0xe2, 0x00, 0xc1, 0xcc, 0x0d, 0x00, 0xf5, 0xdb, 0x0e, 0x00, 0x17, 0x48, 0xe2, // 0x2260 + 0x00, 0xf5, 0xdb, 0x0e, 0x00, 0xef, 0x5c, 0xe2, 0x00, 0xc1, 0xcc, 0x0d, 0x00, 0xf5, 0xdb, 0x0e, // 0x2270 + 0x00, 0x17, 0x48, 0xe2, 0x00, 0xf5, 0xdb, 0x0e, 0x00, 0xef, 0x5c, 0xe2, 0x00, 0xc1, 0xcc, 0x0d, // 0x2280 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2290 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x22a0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x22b0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x8d, 0x00, // 0x22c0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x72, 0xff, 0x00, 0xce, 0x25, 0xe1, 0x00, 0x2f, 0xe4, 0x0e, // 0x22d0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x22e0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x22f0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2300 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2310 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2320 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2330 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2340 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2350 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2360 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2370 + 0x0B, 0x5E, 0x0B, 0x5E, 0x00, 0x01, 0x00, 0x0F, 0x01, 0x00, 0x00, 0x1E, 0x00, 0x00, 0x66, 0x0E, // 0x2380 + 0x94, 0x03, 0x4E, 0x43, 0x08, 0x64, 0xB6, 0x28, 0x0B, 0x03, 0x99, 0x0D, 0x00, 0x00, 0x00, 0x00, // 0x2390 + 0x00, 0x00, 0x00, 0x00, 0x2D, 0x0C, 0x1E, 0x78, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x23A0 + 0x00, 0x00, 0x01, 0x00, 0x00, 0x1E, 0x08, 0x00, 0x00, 0x00, 0xA0, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x23B0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x23C0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x23D0 + 0x21, // 0x23E0 - DSMIG_EN +}; + +// Tweeter DSM firmware (for 0x3C, 0x3D) +static const u8 max98390_dsm_firmware_tweeter[] = { + 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1F, // 0x2050 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2060 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0E, 0x80, 0x07, 0x07, 0x01, 0x00, 0x4A, 0x2B, 0x08, 0x00, // 0x2070 + 0x03, 0x03, 0x07, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2080 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2090 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20A0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20B0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20C0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20D0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20E0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20F0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2100 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2110 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x46, 0x0d, 0x00, 0xf6, 0x73, 0xe5, // 0x2120 + 0x00, 0x05, 0x46, 0x0d, 0x00, 0xc0, 0xe9, 0xe5, 0x00, 0xd5, 0x01, 0x0b, 0x00, 0x9d, 0x78, 0x09, // 0x2130 + 0x00, 0xc5, 0x0e, 0xed, 0x00, 0x9d, 0x78, 0x09, 0x00, 0xc4, 0x4e, 0xf0, 0x00, 0x3a, 0x31, 0x06, // 0x2140 + 0x00, 0xaf, 0x75, 0x0e, 0x00, 0x8c, 0x1f, 0xf1, 0x00, 0x9e, 0x36, 0x0c, 0x00, 0x8c, 0x1f, 0xf1, // 0x2150 + 0x00, 0x4d, 0xac, 0x0a, 0x00, 0x7d, 0xa0, 0x0f, 0x00, 0xe1, 0x9c, 0xe3, 0x00, 0xe8, 0x59, 0x0e, // 0x2160 + 0x00, 0xe1, 0x9c, 0xe3, 0x00, 0x66, 0xfa, 0x0d, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, // 0x2170 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, // 0x2180 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2190 + 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x21a0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x21b0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x21c0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x21d0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x21e0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x21f0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2200 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2210 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2220 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2230 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2240 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2250 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2260 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2270 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2280 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2290 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x22a0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x22b0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x22c0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x22d0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x22e0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x22f0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2300 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2310 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2320 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2330 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2340 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2350 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2360 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x2370 + 0x0B, 0x5E, 0x0B, 0x5E, 0x00, 0x01, 0x00, 0x0F, 0x01, 0x00, 0x00, 0x1E, 0x00, 0x00, 0x66, 0x0E, // 0x2380 + 0x94, 0x03, 0xAA, 0x2A, 0x09, 0x64, 0xB6, 0x28, 0x0B, 0x03, 0x99, 0x0D, 0x00, 0x00, 0x00, 0x00, // 0x2390 + 0x00, 0x00, 0x00, 0x00, 0x2D, 0x0C, 0x1E, 0x78, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x23A0 + 0x00, 0x00, 0x01, 0x00, 0x00, 0x1E, 0x08, 0x00, 0x00, 0x00, 0xA0, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x23B0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x23C0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x23D0 + 0x20, // 0x23E0 - DSMIG_EN +}; + +// Load complete DSM firmware blob (913 bytes from 0x2050-0x23E0) +// Selects appropriate firmware based on speaker type (woofer vs tweeter) +static void max98390_load_dsm_firmware(struct max98390_hda_priv *priv, bool is_tweeter) +{ + const u8 *firmware; + int i; + + // Select appropriate firmware blob + if (is_tweeter) { + firmware = max98390_dsm_firmware_tweeter; + } else { + firmware = max98390_dsm_firmware_woofer; + } + + // Write all 913 bytes sequentially starting from 0x2050 + // This matches coolstar's uploadDSMBin() implementation + for (i = 0; i < MAX98390_DSM_PARAM_SIZE; i++) { + regmap_write(priv->regmap, MAX98390_DSM_START_ADDR + i, firmware[i]); + } +} + +// Configure DSM with appropriate firmware based on speaker type +// cutoff_freq: cutoff frequency in Hz (unused - filter built into tweeter firmware) +// is_tweeter: true for tweeters (0x3C, 0x3D), false for woofers (0x38, 0x39) +void max98390_configure_high_pass_filter(struct max98390_hda_priv *priv, int cutoff_freq, bool is_tweeter) +{ + // Load complete DSM firmware (913 bytes, all registers) + // Tweeter firmware includes high-pass filter in STBASS section + // Woofer firmware is full-range + max98390_load_dsm_firmware(priv, is_tweeter); + + // Set DSM_VOL_CTRL - Tweeter more quiet than woofer + if (is_tweeter) { + regmap_write(priv->regmap, 0x23BA, 0x8d); // DSM_VOL_CTRL Tweeter + } if(!is_tweeter) { + regmap_write(priv->regmap, 0x23BA, 0xA0); // DSM_VOL_CTRL Woofer + } + + // SET DSP_GLOBAL_EN TO 1 (WRITE 0x01 TO 0x23E1) + regmap_write(priv->regmap, 0x23E1, 0x01); + + // SET SPK_EN TO 1 (WRITE 0x81 TO 0x203A) + regmap_write(priv->regmap, 0x203A, 0x81); + + // SET EN TO 1 (WRITE 0x01 TO 0x23FF) + regmap_write(priv->regmap, 0x23FF, 0x01); +} + +MODULE_DESCRIPTION("HDA MAX98390 driver"); +MODULE_AUTHOR("Kevin Cuperus "); +MODULE_LICENSE("GPL"); diff -ruN --no-dereference -x .git a/sound/hda/codecs/side-codecs/max98390_hda_filters.h b/sound/hda/codecs/side-codecs/max98390_hda_filters.h --- a/sound/hda/codecs/side-codecs/max98390_hda_filters.h 1970-01-01 01:00:00.000000000 +0100 +++ b/sound/hda/codecs/side-codecs/max98390_hda_filters.h 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +// +// MAX98390 HDA filter configuration header +// + +#ifndef __MAX98390_HDA_FILTERS_H +#define __MAX98390_HDA_FILTERS_H + +struct max98390_hda_priv; + +// Function prototypes +void max98390_configure_filters(struct max98390_hda_priv *priv); +void max98390_configure_high_pass_filter(struct max98390_hda_priv *priv, int cutoff_freq, bool is_tweeter); + +#endif /* __MAX98390_HDA_FILTERS_H */ \ No newline at end of file diff -ruN --no-dereference -x .git a/sound/hda/codecs/side-codecs/max98390_hda_i2c.c b/sound/hda/codecs/side-codecs/max98390_hda_i2c.c --- a/sound/hda/codecs/side-codecs/max98390_hda_i2c.c 1970-01-01 01:00:00.000000000 +0100 +++ b/sound/hda/codecs/side-codecs/max98390_hda_i2c.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// MAX98390 HDA I2C driver +// + +#include +#include +#include + +#include "max98390_hda.h" +#include "../../../soc/codecs/max98390.h" + +static int max98390_hda_i2c_probe(struct i2c_client *clt) +{ + const char *device_name; + const char *name_suffix; + int index = 0; + + /* Match device name from serial-multi-instantiate */ + if (strstr(dev_name(&clt->dev), "MAX98390") || + strstr(dev_name(&clt->dev), "max98390")) + device_name = "MAX98390"; + else + return -ENODEV; + + /* Parse index from device name (e.g., "i2c-MAX98390:00-max98390-hda.0" -> index 0) */ + name_suffix = strrchr(dev_name(&clt->dev), '.'); + if (name_suffix) { + if (kstrtoint(name_suffix + 1, 10, &index) < 0) { + dev_err(&clt->dev, "Failed to parse device index from name: %s\n", + dev_name(&clt->dev)); + return -EINVAL; + } + } + + return max98390_hda_probe(&clt->dev, device_name, index, clt->irq, + devm_regmap_init_i2c(clt, &max98390_regmap), + MAX98390_HDA_I2C, clt->addr); +} + +static void max98390_hda_i2c_remove(struct i2c_client *clt) +{ + max98390_hda_remove(&clt->dev); +} + +static const struct i2c_device_id max98390_hda_i2c_id[] = { + { "max98390-hda", 0 }, + {} +}; +MODULE_DEVICE_TABLE(i2c, max98390_hda_i2c_id); + +static const struct acpi_device_id max98390_acpi_hda_match[] = { + { "MAX98390", 0 }, + { "MX98390", 0 }, + {} +}; +MODULE_DEVICE_TABLE(acpi, max98390_acpi_hda_match); + +static struct i2c_driver max98390_hda_i2c_driver = { + .driver = { + .name = "max98390-hda", + .acpi_match_table = max98390_acpi_hda_match, + .pm = &max98390_hda_pm_ops, + }, + .id_table = max98390_hda_i2c_id, + .probe = max98390_hda_i2c_probe, + .remove = max98390_hda_i2c_remove, +}; +module_i2c_driver(max98390_hda_i2c_driver); + +MODULE_DESCRIPTION("HDA MAX98390 driver"); +MODULE_IMPORT_NS("SND_HDA_SCODEC_MAX98390"); +MODULE_AUTHOR("Kevin Cuperus "); +MODULE_LICENSE("GPL"); diff -ruN --no-dereference -x .git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c --- a/sound/soc/amd/yc/acp6x-mach.c 2026-10-11 15:27:50.000000000 +0200 +++ b/sound/soc/amd/yc/acp6x-mach.c 2026-10-13 14:33:30.000000000 +0200 @@ -615,6 +615,13 @@ { .driver_data = &acp6x_card, .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "MDU"), + DMI_MATCH(DMI_PRODUCT_NAME, "Aspire A314-23P"), + } + }, + { + .driver_data = &acp6x_card, + .matches = { DMI_MATCH(DMI_BOARD_VENDOR, "IRBIS"), DMI_MATCH(DMI_PRODUCT_NAME, "15NBC1011"), } diff -ruN --no-dereference -x .git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig --- a/sound/soc/codecs/Kconfig 2026-10-11 15:27:50.000000000 +0200 +++ b/sound/soc/codecs/Kconfig 2026-10-13 14:33:30.000000000 +0200 @@ -726,6 +726,10 @@ sound quality, which is a new high efficiency, low noise, constant large volume, 6th Smart K audio amplifier. +config SND_SOC_AW88399_LIB + tristate + select SND_SOC_AW88395_LIB + config SND_SOC_AW88399 tristate "Soc Audio for awinic aw88399" depends on I2C @@ -733,6 +737,7 @@ select REGMAP_I2C select GPIOLIB select SND_SOC_AW88395_LIB + select SND_SOC_AW88399_LIB help This option enables support for aw88399 Smart PA. The awinic AW88399 is an I2S/TDM input, high efficiency diff -ruN --no-dereference -x .git a/sound/soc/codecs/Makefile b/sound/soc/codecs/Makefile --- a/sound/soc/codecs/Makefile 2026-10-11 15:27:50.000000000 +0200 +++ b/sound/soc/codecs/Makefile 2026-10-13 14:33:30.000000000 +0200 @@ -55,6 +55,7 @@ snd-soc-aw88166-y := aw88166.o snd-soc-aw88261-y := aw88261.o snd-soc-aw88399-y := aw88399.o +snd-soc-aw88399-lib-y := aw88399-lib.o snd-soc-bd28623-y := bd28623.o snd-soc-bt-sco-y := bt-sco.o snd-soc-chv3-codec-y := chv3-codec.o @@ -496,6 +497,7 @@ obj-$(CONFIG_SND_SOC_AW88166) +=snd-soc-aw88166.o obj-$(CONFIG_SND_SOC_AW88261) +=snd-soc-aw88261.o obj-$(CONFIG_SND_SOC_AW88399) += snd-soc-aw88399.o +obj-$(CONFIG_SND_SOC_AW88399_LIB) += snd-soc-aw88399-lib.o obj-$(CONFIG_SND_SOC_BD28623) += snd-soc-bd28623.o obj-$(CONFIG_SND_SOC_BT_SCO) += snd-soc-bt-sco.o obj-$(CONFIG_SND_SOC_CHV3_CODEC) += snd-soc-chv3-codec.o diff -ruN --no-dereference -x .git a/sound/soc/codecs/aw88399-lib.c b/sound/soc/codecs/aw88399-lib.c --- a/sound/soc/codecs/aw88399-lib.c 1970-01-01 01:00:00.000000000 +0100 +++ b/sound/soc/codecs/aw88399-lib.c 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,1411 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// aw88399-lib.c -- AW88399 Common functions for ASoC and HDA audio drivers +// +// Copyright (c) 2023 AWINIC Technology CO., LTD +// +// Author: Weidong Wang +// + +#include +#include +#include +#include +#include +#include +#include +#include "aw88395/aw88395_device.h" + +const struct regmap_config aw88399_remap_config = { + .val_bits = 16, + .reg_bits = 8, + .max_register = AW88399_REG_MAX, + .reg_format_endian = REGMAP_ENDIAN_LITTLE, + .val_format_endian = REGMAP_ENDIAN_BIG, +}; +EXPORT_SYMBOL_GPL(aw88399_remap_config); + +static void aw_dev_pwd(struct aw_device *aw_dev, bool pwd) +{ + int ret; + + if (pwd) + ret = regmap_update_bits(aw_dev->regmap, AW88399_SYSCTRL_REG, + ~AW88399_PWDN_MASK, AW88399_PWDN_POWER_DOWN_VALUE); + else + ret = regmap_update_bits(aw_dev->regmap, AW88399_SYSCTRL_REG, + ~AW88399_PWDN_MASK, AW88399_PWDN_WORKING_VALUE); + + if (ret) + dev_dbg(aw_dev->dev, "%s failed", __func__); +} + +static void aw_dev_get_int_status(struct aw_device *aw_dev, unsigned short *int_status) +{ + unsigned int reg_val; + int ret; + + ret = regmap_read(aw_dev->regmap, AW88399_SYSINT_REG, ®_val); + if (ret) + dev_err(aw_dev->dev, "read interrupt reg fail, ret=%d", ret); + else + *int_status = reg_val; + + dev_dbg(aw_dev->dev, "read interrupt reg=0x%04x", *int_status); +} + +static void aw_dev_clear_int_status(struct aw_device *aw_dev) +{ + u16 int_status; + + /* read int status and clear */ + aw_dev_get_int_status(aw_dev, &int_status); + /* make sure int status is clear */ + aw_dev_get_int_status(aw_dev, &int_status); + if (int_status) + dev_dbg(aw_dev->dev, "int status(%d) is not cleaned.\n", int_status); +} + +static int aw_dev_get_iis_status(struct aw_device *aw_dev) +{ + unsigned int reg_val; + int ret; + + ret = regmap_read(aw_dev->regmap, AW88399_SYSST_REG, ®_val); + if (ret) + return ret; + if ((reg_val & AW88399_BIT_PLL_CHECK) != AW88399_BIT_PLL_CHECK) { + dev_err(aw_dev->dev, "check pll lock fail, reg_val:0x%04x", reg_val); + return -EINVAL; + } + + return 0; +} + +static int aw_dev_check_mode1_pll(struct aw_device *aw_dev) +{ + int ret, i; + + for (i = 0; i < AW88399_DEV_SYSST_CHECK_MAX; i++) { + ret = aw_dev_get_iis_status(aw_dev); + if (ret) { + dev_err(aw_dev->dev, "mode1 iis signal check error"); + usleep_range(AW88399_2000_US, AW88399_2000_US + 10); + } else { + return 0; + } + } + + return -EPERM; +} + +static int aw_dev_check_mode2_pll(struct aw_device *aw_dev) +{ + unsigned int reg_val; + int ret, i; + + ret = regmap_read(aw_dev->regmap, AW88399_PLLCTRL2_REG, ®_val); + if (ret) + return ret; + + reg_val &= (~AW88399_CCO_MUX_MASK); + if (reg_val == AW88399_CCO_MUX_DIVIDED_VALUE) { + dev_dbg(aw_dev->dev, "CCO_MUX is already divider"); + return -EPERM; + } + + /* change mode2 */ + ret = regmap_update_bits(aw_dev->regmap, AW88399_PLLCTRL2_REG, + ~AW88399_CCO_MUX_MASK, AW88399_CCO_MUX_DIVIDED_VALUE); + if (ret) + return ret; + + for (i = 0; i < AW88399_DEV_SYSST_CHECK_MAX; i++) { + ret = aw_dev_get_iis_status(aw_dev); + if (ret) { + dev_err(aw_dev->dev, "mode2 iis signal check error"); + usleep_range(AW88399_2000_US, AW88399_2000_US + 10); + } else { + break; + } + } + + /* change mode1 */ + regmap_update_bits(aw_dev->regmap, AW88399_PLLCTRL2_REG, + ~AW88399_CCO_MUX_MASK, AW88399_CCO_MUX_BYPASS_VALUE); + if (ret == 0) { + usleep_range(AW88399_2000_US, AW88399_2000_US + 10); + for (i = 0; i < AW88399_DEV_SYSST_CHECK_MAX; i++) { + ret = aw_dev_get_iis_status(aw_dev); + if (ret) { + dev_err(aw_dev->dev, "mode2 switch to mode1, iis signal check error"); + usleep_range(AW88399_2000_US, AW88399_2000_US + 10); + } else { + break; + } + } + } + + return ret; +} + +int aw_dev_check_syspll(struct aw_device *aw_dev) +{ + int ret; + + ret = aw_dev_check_mode1_pll(aw_dev); + if (ret) { + dev_dbg(aw_dev->dev, "mode1 check iis failed try switch to mode2 check"); + ret = aw_dev_check_mode2_pll(aw_dev); + if (ret) { + dev_err(aw_dev->dev, "mode2 check iis failed"); + return ret; + } + } + + return 0; +} +EXPORT_SYMBOL_GPL(aw_dev_check_syspll); + +static int aw_dev_check_sysst(struct aw88399 *aw88399) +{ + struct aw_device *aw_dev = aw88399->aw_pa; + unsigned int check_val; + unsigned int reg_val; + int ret, i; + + ret = regmap_read(aw_dev->regmap, AW88399_PWMCTRL3_REG, ®_val); + if (ret) + return ret; + + if (reg_val & (~AW88399_NOISE_GATE_EN_MASK)) + check_val = AW88399_BIT_SYSST_NOSWS_CHECK; + else + check_val = AW88399_BIT_SYSST_SWS_CHECK; + + /* + * On some hardware the BSTS (boost-finished) status bit does not + * reliably assert even when audio output is working normally. + * Allow per-instance bypass when flagged by the side-codec driver. + */ + if (aw88399->bsts_unreliable) + check_val &= ~AW88399_BSTS_FINISHED_VALUE; + + for (i = 0; i < AW88399_DEV_SYSST_CHECK_MAX; i++) { + ret = regmap_read(aw_dev->regmap, AW88399_SYSST_REG, ®_val); + if (ret) + return ret; + + if ((reg_val & (~AW88399_BIT_SYSST_CHECK_MASK) & check_val) != check_val) { + dev_err(aw_dev->dev, "check sysst fail, cnt=%d, reg_val=0x%04x, check:0x%x", + i, reg_val, AW88399_BIT_SYSST_NOSWS_CHECK); + usleep_range(AW88399_2000_US, AW88399_2000_US + 10); + } else { + return 0; + } + } + + return -EPERM; +} + +static void aw_dev_amppd(struct aw_device *aw_dev, bool amppd) +{ + int ret; + + if (amppd) + ret = regmap_update_bits(aw_dev->regmap, AW88399_SYSCTRL_REG, + ~AW88399_AMPPD_MASK, AW88399_AMPPD_POWER_DOWN_VALUE); + else + ret = regmap_update_bits(aw_dev->regmap, AW88399_SYSCTRL_REG, + ~AW88399_AMPPD_MASK, AW88399_AMPPD_WORKING_VALUE); + + if (ret) + dev_dbg(aw_dev->dev, "%s failed", __func__); +} + +void aw_dev_dsp_enable(struct aw_device *aw_dev, bool is_enable) +{ + int ret; + + if (is_enable) + ret = regmap_update_bits(aw_dev->regmap, AW88399_SYSCTRL_REG, + ~AW88399_DSPBY_MASK, AW88399_DSPBY_WORKING_VALUE); + else + ret = regmap_update_bits(aw_dev->regmap, AW88399_SYSCTRL_REG, + ~AW88399_DSPBY_MASK, AW88399_DSPBY_BYPASS_VALUE); + + if (ret) + dev_dbg(aw_dev->dev, "%s failed\n", __func__); +} +EXPORT_SYMBOL_GPL(aw_dev_dsp_enable); + +static int aw88399_dev_get_icalk(struct aw88399 *aw88399, int16_t *icalk) +{ + uint16_t icalkh_val, icalkl_val, icalk_val; + struct aw_device *aw_dev = aw88399->aw_pa; + unsigned int reg_val; + int ret; + + ret = regmap_read(aw_dev->regmap, AW88399_EFRH4_REG, ®_val); + if (ret) + return ret; + icalkh_val = reg_val & (~AW88399_EF_ISN_GESLP_H_MASK); + + ret = regmap_read(aw_dev->regmap, AW88399_EFRL4_REG, ®_val); + if (ret) + return ret; + icalkl_val = reg_val & (~AW88399_EF_ISN_GESLP_L_MASK); + + if (aw88399->check_val == AW_EF_AND_CHECK) + icalk_val = icalkh_val & icalkl_val; + else + icalk_val = icalkh_val | icalkl_val; + + if (icalk_val & (~AW88399_EF_ISN_GESLP_SIGN_MASK)) + icalk_val = icalk_val | AW88399_EF_ISN_GESLP_SIGN_NEG; + *icalk = (int16_t)icalk_val; + + return 0; +} + +static int aw88399_dev_get_vcalk(struct aw88399 *aw88399, int16_t *vcalk) +{ + uint16_t vcalkh_val, vcalkl_val, vcalk_val; + struct aw_device *aw_dev = aw88399->aw_pa; + unsigned int reg_val; + int ret; + + ret = regmap_read(aw_dev->regmap, AW88399_EFRH3_REG, ®_val); + if (ret) + return ret; + + vcalkh_val = reg_val & (~AW88399_EF_VSN_GESLP_H_MASK); + + ret = regmap_read(aw_dev->regmap, AW88399_EFRL3_REG, ®_val); + if (ret) + return ret; + + vcalkl_val = reg_val & (~AW88399_EF_VSN_GESLP_L_MASK); + + if (aw88399->check_val == AW_EF_AND_CHECK) + vcalk_val = vcalkh_val & vcalkl_val; + else + vcalk_val = vcalkh_val | vcalkl_val; + + if (vcalk_val & AW88399_EF_VSN_GESLP_SIGN_MASK) + vcalk_val = vcalk_val | AW88399_EF_VSN_GESLP_SIGN_NEG; + *vcalk = (int16_t)vcalk_val; + + return 0; +} + +static int aw88399_dev_get_internal_vcalk(struct aw88399 *aw88399, int16_t *vcalk) +{ + uint16_t vcalkh_val, vcalkl_val, vcalk_val; + struct aw_device *aw_dev = aw88399->aw_pa; + unsigned int reg_val; + int ret; + + ret = regmap_read(aw_dev->regmap, AW88399_EFRH2_REG, ®_val); + if (ret) + return ret; + vcalkh_val = reg_val & (~AW88399_INTERNAL_VSN_TRIM_H_MASK); + + ret = regmap_read(aw_dev->regmap, AW88399_EFRL2_REG, ®_val); + if (ret) + return ret; + vcalkl_val = reg_val & (~AW88399_INTERNAL_VSN_TRIM_L_MASK); + + if (aw88399->check_val == AW_EF_AND_CHECK) + vcalk_val = (vcalkh_val >> AW88399_INTERNAL_VSN_TRIM_H_START_BIT) & + (vcalkl_val >> AW88399_INTERNAL_VSN_TRIM_L_START_BIT); + else + vcalk_val = (vcalkh_val >> AW88399_INTERNAL_VSN_TRIM_H_START_BIT) | + (vcalkl_val >> AW88399_INTERNAL_VSN_TRIM_L_START_BIT); + + if (vcalk_val & (~AW88399_TEM4_SIGN_MASK)) + vcalk_val = vcalk_val | AW88399_TEM4_SIGN_NEG; + + *vcalk = (int16_t)vcalk_val; + + return 0; +} + +static int aw_dev_set_vcalb(struct aw88399 *aw88399) +{ + struct aw_device *aw_dev = aw88399->aw_pa; + unsigned int vsense_select, vsense_value; + int32_t ical_k, vcal_k, vcalb; + int16_t icalk, vcalk; + uint16_t reg_val; + int ret; + + ret = regmap_read(aw_dev->regmap, AW88399_VSNCTRL1_REG, &vsense_value); + if (ret) + return ret; + + vsense_select = vsense_value & (~AW88399_VDSEL_MASK); + + ret = aw88399_dev_get_icalk(aw88399, &icalk); + if (ret) { + dev_err(aw_dev->dev, "get icalk failed\n"); + return ret; + } + + ical_k = icalk * AW88399_ICABLK_FACTOR + AW88399_CABL_BASE_VALUE; + + switch (vsense_select) { + case AW88399_DEV_VDSEL_VSENSE: + ret = aw88399_dev_get_vcalk(aw88399, &vcalk); + vcal_k = vcalk * AW88399_VCABLK_FACTOR + AW88399_CABL_BASE_VALUE; + vcalb = AW88399_VCALB_ACCURACY * AW88399_VSCAL_FACTOR / AW88399_ISCAL_FACTOR * + ical_k / vcal_k * aw88399->vcalb_init_val; + break; + case AW88399_DEV_VDSEL_DAC: + ret = aw88399_dev_get_internal_vcalk(aw88399, &vcalk); + vcal_k = vcalk * AW88399_VCABLK_DAC_FACTOR + AW88399_CABL_BASE_VALUE; + vcalb = AW88399_VCALB_ACCURACY * AW88399_VSCAL_DAC_FACTOR / + AW88399_ISCAL_DAC_FACTOR * ical_k / + vcal_k * aw88399->vcalb_init_val; + break; + default: + dev_err(aw_dev->dev, "%s: unsupported vsense\n", __func__); + ret = -EINVAL; + break; + } + if (ret) + return ret; + + vcalb = vcalb >> AW88399_VCALB_ADJ_FACTOR; + reg_val = (uint32_t)vcalb; + + regmap_write(aw_dev->regmap, AW88399_DSPVCALB_REG, reg_val); + + return 0; +} + +int aw_dev_update_cali_re(struct aw_cali_desc *cali_desc) +{ + struct aw_device *aw_dev = + container_of(cali_desc, struct aw_device, cali_desc); + uint16_t re_lbits, re_hbits; + u32 cali_re; + int ret; + + if ((aw_dev->cali_desc.cali_re >= AW88399_CALI_RE_MAX) || + (aw_dev->cali_desc.cali_re <= AW88399_CALI_RE_MIN)) + return -EINVAL; + + cali_re = AW88399_SHOW_RE_TO_DSP_RE((aw_dev->cali_desc.cali_re + + aw_dev->cali_desc.ra), AW88399_DSP_RE_SHIFT); + + re_hbits = (cali_re & (~AW88399_CALI_RE_HBITS_MASK)) >> AW88399_CALI_RE_HBITS_SHIFT; + re_lbits = (cali_re & (~AW88399_CALI_RE_LBITS_MASK)) >> AW88399_CALI_RE_LBITS_SHIFT; + + ret = regmap_write(aw_dev->regmap, AW88399_ACR1_REG, re_hbits); + if (ret) { + dev_err(aw_dev->dev, "set cali re error"); + return ret; + } + + ret = regmap_write(aw_dev->regmap, AW88399_ACR2_REG, re_lbits); + if (ret) + dev_err(aw_dev->dev, "set cali re error"); + + return ret; +} +EXPORT_SYMBOL_GPL(aw_dev_update_cali_re); + +static int aw_dev_fw_crc_check(struct aw_device *aw_dev) +{ + uint16_t check_val, fw_len_val; + unsigned int reg_val; + int ret; + + /* calculate fw_end_addr */ + fw_len_val = ((aw_dev->dsp_fw_len / AW_FW_ADDR_LEN) - 1) + AW88399_CRC_FW_BASE_ADDR; + + /* write fw_end_addr to crc_end_addr */ + ret = regmap_update_bits(aw_dev->regmap, AW88399_CRCCTRL_REG, + ~AW88399_CRC_END_ADDR_MASK, fw_len_val); + if (ret) + return ret; + /* enable fw crc check */ + ret = regmap_update_bits(aw_dev->regmap, AW88399_CRCCTRL_REG, + ~AW88399_CRC_CODE_EN_MASK, AW88399_CRC_CODE_EN_ENABLE_VALUE); + + usleep_range(AW88399_2000_US, AW88399_2000_US + 10); + + /* read crc check result */ + regmap_read(aw_dev->regmap, AW88399_HAGCST_REG, ®_val); + if (ret) + return ret; + + check_val = (reg_val & (~AW88399_CRC_CHECK_BITS_MASK)) >> AW88399_CRC_CHECK_START_BIT; + + /* disable fw crc check */ + ret = regmap_update_bits(aw_dev->regmap, AW88399_CRCCTRL_REG, + ~AW88399_CRC_CODE_EN_MASK, AW88399_CRC_CODE_EN_DISABLE_VALUE); + if (ret) + return ret; + + if (check_val != AW88399_CRC_CHECK_PASS_VAL) { + dev_err(aw_dev->dev, "%s failed, check_val 0x%x != 0x%x", + __func__, check_val, AW88399_CRC_CHECK_PASS_VAL); + ret = -EINVAL; + } + + return ret; +} + +static int aw_dev_cfg_crc_check(struct aw_device *aw_dev) +{ + uint16_t check_val, cfg_len_val; + unsigned int reg_val; + int ret; + + /* calculate cfg end addr */ + cfg_len_val = ((aw_dev->dsp_cfg_len / AW_FW_ADDR_LEN) - 1) + AW88399_CRC_CFG_BASE_ADDR; + + /* write cfg_end_addr to crc_end_addr */ + ret = regmap_update_bits(aw_dev->regmap, AW88399_CRCCTRL_REG, + ~AW88399_CRC_END_ADDR_MASK, cfg_len_val); + if (ret) + return ret; + + /* enable cfg crc check */ + ret = regmap_update_bits(aw_dev->regmap, AW88399_CRCCTRL_REG, + ~AW88399_CRC_CFG_EN_MASK, AW88399_CRC_CFG_EN_ENABLE_VALUE); + if (ret) + return ret; + + usleep_range(AW88399_1000_US, AW88399_1000_US + 10); + + /* read crc check result */ + ret = regmap_read(aw_dev->regmap, AW88399_HAGCST_REG, ®_val); + if (ret) + return ret; + + check_val = (reg_val & (~AW88399_CRC_CHECK_BITS_MASK)) >> AW88399_CRC_CHECK_START_BIT; + + /* disable cfg crc check */ + ret = regmap_update_bits(aw_dev->regmap, AW88399_CRCCTRL_REG, + ~AW88399_CRC_CFG_EN_MASK, AW88399_CRC_CFG_EN_DISABLE_VALUE); + if (ret) + return ret; + + if (check_val != AW88399_CRC_CHECK_PASS_VAL) { + dev_err(aw_dev->dev, "crc_check failed, check val 0x%x != 0x%x", + check_val, AW88399_CRC_CHECK_PASS_VAL); + ret = -EINVAL; + } + + return ret; +} + +static int aw_dev_hw_crc_check(struct aw88399 *aw88399) +{ + struct aw_device *aw_dev = aw88399->aw_pa; + int ret; + + ret = regmap_update_bits(aw_dev->regmap, AW88399_I2SCFG1_REG, + ~AW88399_RAM_CG_BYP_MASK, AW88399_RAM_CG_BYP_BYPASS_VALUE); + if (ret) + return ret; + + ret = aw_dev_fw_crc_check(aw_dev); + if (ret) { + dev_err(aw_dev->dev, "fw_crc_check failed\n"); + goto crc_check_failed; + } + + ret = aw_dev_cfg_crc_check(aw_dev); + if (ret) { + dev_err(aw_dev->dev, "cfg_crc_check failed\n"); + goto crc_check_failed; + } + + ret = regmap_write(aw_dev->regmap, AW88399_CRCCTRL_REG, aw88399->crc_init_val); + if (ret) + return ret; + + ret = regmap_update_bits(aw_dev->regmap, AW88399_I2SCFG1_REG, + ~AW88399_RAM_CG_BYP_MASK, AW88399_RAM_CG_BYP_WORK_VALUE); + + return ret; + +crc_check_failed: + regmap_update_bits(aw_dev->regmap, AW88399_I2SCFG1_REG, + ~AW88399_RAM_CG_BYP_MASK, AW88399_RAM_CG_BYP_WORK_VALUE); + return ret; +} + +static void aw_dev_i2s_tx_enable(struct aw_device *aw_dev, bool flag) +{ + int ret; + + if (flag) + ret = regmap_update_bits(aw_dev->regmap, AW88399_I2SCTRL3_REG, + ~AW88399_I2STXEN_MASK, AW88399_I2STXEN_ENABLE_VALUE); + else + ret = regmap_update_bits(aw_dev->regmap, AW88399_I2SCFG1_REG, + ~AW88399_I2STXEN_MASK, AW88399_I2STXEN_DISABLE_VALUE); + + if (ret) + dev_dbg(aw_dev->dev, "%s failed", __func__); +} + +int aw_dev_get_dsp_status(struct aw_device *aw_dev) +{ + unsigned int reg_val; + int ret; + + ret = regmap_read(aw_dev->regmap, AW88399_WDT_REG, ®_val); + if (ret) + return ret; + if (!(reg_val & (~AW88399_WDT_CNT_MASK))) + return -EPERM; + + return 0; +} +EXPORT_SYMBOL_GPL(aw_dev_get_dsp_status); + +static int aw_dev_dsp_check(struct aw_device *aw_dev) +{ + int ret, i; + + switch (aw_dev->dsp_cfg) { + case AW88399_DEV_DSP_BYPASS: + dev_dbg(aw_dev->dev, "dsp bypass"); + ret = 0; + break; + case AW88399_DEV_DSP_WORK: + aw_dev_dsp_enable(aw_dev, false); + aw_dev_dsp_enable(aw_dev, true); + usleep_range(AW88399_1000_US, AW88399_1000_US + 10); + for (i = 0; i < AW88399_DEV_DSP_CHECK_MAX; i++) { + ret = aw_dev_get_dsp_status(aw_dev); + if (ret) { + dev_err(aw_dev->dev, "dsp wdt status error=%d", ret); + usleep_range(AW88399_2000_US, AW88399_2000_US + 10); + } + } + break; + default: + dev_err(aw_dev->dev, "unknown dsp cfg=%d", aw_dev->dsp_cfg); + ret = -EINVAL; + break; + } + + return ret; +} + +int aw_dev_set_volume(struct aw_device *aw_dev, unsigned int value) +{ + struct aw_volume_desc *vol_desc = &aw_dev->volume_desc; + unsigned int reg_value; + u16 real_value; + int ret; + + real_value = min((value + vol_desc->init_volume), (unsigned int)AW88399_MUTE_VOL); + + ret = regmap_read(aw_dev->regmap, AW88399_SYSCTRL2_REG, ®_value); + if (ret) + return ret; + + dev_dbg(aw_dev->dev, "value 0x%x , reg:0x%x", value, real_value); + + real_value = (real_value << AW88399_VOL_START_BIT) | (reg_value & AW88399_VOL_MASK); + + ret = regmap_write(aw_dev->regmap, AW88399_SYSCTRL2_REG, real_value); + + return ret; +} +EXPORT_SYMBOL_GPL(aw_dev_set_volume); + +static void aw_dev_fade_in(struct aw_device *aw_dev) +{ + struct aw_volume_desc *desc = &aw_dev->volume_desc; + u16 fade_in_vol = desc->ctl_volume; + int fade_step = aw_dev->fade_step; + int i; + + if (fade_step == 0 || aw_dev->fade_in_time == 0) { + aw_dev_set_volume(aw_dev, fade_in_vol); + return; + } + + for (i = AW88399_MUTE_VOL; i >= fade_in_vol; i -= fade_step) { + aw_dev_set_volume(aw_dev, i); + usleep_range(aw_dev->fade_in_time, aw_dev->fade_in_time + 10); + } + + if (i != fade_in_vol) + aw_dev_set_volume(aw_dev, fade_in_vol); +} + +static void aw_dev_fade_out(struct aw_device *aw_dev) +{ + struct aw_volume_desc *desc = &aw_dev->volume_desc; + int fade_step = aw_dev->fade_step; + int i; + + if (fade_step == 0 || aw_dev->fade_out_time == 0) { + aw_dev_set_volume(aw_dev, AW88399_MUTE_VOL); + return; + } + + for (i = desc->ctl_volume; i <= AW88399_MUTE_VOL; i += fade_step) { + aw_dev_set_volume(aw_dev, i); + usleep_range(aw_dev->fade_out_time, aw_dev->fade_out_time + 10); + } + + if (i != AW88399_MUTE_VOL) { + aw_dev_set_volume(aw_dev, AW88399_MUTE_VOL); + usleep_range(aw_dev->fade_out_time, aw_dev->fade_out_time + 10); + } +} + +void aw88399_dev_mute(struct aw_device *aw_dev, bool is_mute) +{ + if (is_mute) { + aw_dev_fade_out(aw_dev); + regmap_update_bits(aw_dev->regmap, AW88399_SYSCTRL_REG, + ~AW88399_HMUTE_MASK, AW88399_HMUTE_ENABLE_VALUE); + } else { + regmap_update_bits(aw_dev->regmap, AW88399_SYSCTRL_REG, + ~AW88399_HMUTE_MASK, AW88399_HMUTE_DISABLE_VALUE); + aw_dev_fade_in(aw_dev); + } +} +EXPORT_SYMBOL_GPL(aw88399_dev_mute); + +static void aw88399_dev_set_dither(struct aw88399 *aw88399, bool dither) +{ + struct aw_device *aw_dev = aw88399->aw_pa; + + if (dither) + regmap_update_bits(aw_dev->regmap, AW88399_DBGCTRL_REG, + ~AW88399_DITHER_EN_MASK, AW88399_DITHER_EN_ENABLE_VALUE); + else + regmap_update_bits(aw_dev->regmap, AW88399_DBGCTRL_REG, + ~AW88399_DITHER_EN_MASK, AW88399_DITHER_EN_DISABLE_VALUE); +} + +static int aw88399_dev_start(struct aw88399 *aw88399) +{ + struct aw_device *aw_dev = aw88399->aw_pa; + int ret; + + if (aw_dev->status == AW88399_DEV_PW_ON) { + dev_dbg(aw_dev->dev, "already power on"); + return 0; + } + + aw88399_dev_set_dither(aw88399, false); + + /* power on */ + aw_dev_pwd(aw_dev, false); + usleep_range(AW88399_2000_US, AW88399_2000_US + 10); + + ret = aw_dev_check_syspll(aw_dev); + if (ret) { + dev_err(aw_dev->dev, "pll check failed cannot start"); + goto pll_check_fail; + } + + /* amppd on */ + aw_dev_amppd(aw_dev, false); + usleep_range(AW88399_1000_US, AW88399_1000_US + 50); + + /* check i2s status */ + ret = aw_dev_check_sysst(aw88399); + if (ret) { + dev_err(aw_dev->dev, "sysst check failed"); + goto sysst_check_fail; + } + + if (aw_dev->dsp_cfg == AW88399_DEV_DSP_WORK) { + ret = aw_dev_hw_crc_check(aw88399); + if (ret) { + dev_err(aw_dev->dev, "dsp crc check failed"); + goto crc_check_fail; + } + aw_dev_dsp_enable(aw_dev, false); + aw_dev_set_vcalb(aw88399); + aw_dev_update_cali_re(&aw_dev->cali_desc); + + ret = aw_dev_dsp_check(aw_dev); + if (ret) { + dev_err(aw_dev->dev, "dsp status check failed"); + goto dsp_check_fail; + } + } else { + dev_dbg(aw_dev->dev, "start pa with dsp bypass"); + } + + /* enable tx feedback */ + aw_dev_i2s_tx_enable(aw_dev, true); + + if (aw88399->dither_st == AW88399_DITHER_EN_ENABLE_VALUE) + aw88399_dev_set_dither(aw88399, true); + + /* close mute */ + aw88399_dev_mute(aw_dev, false); + /* clear inturrupt */ + aw_dev_clear_int_status(aw_dev); + aw_dev->status = AW88399_DEV_PW_ON; + + return 0; + +dsp_check_fail: +crc_check_fail: + aw_dev_dsp_enable(aw_dev, false); +sysst_check_fail: + aw_dev_clear_int_status(aw_dev); + aw_dev_amppd(aw_dev, true); +pll_check_fail: + aw_dev_pwd(aw_dev, true); + aw_dev->status = AW88399_DEV_PW_OFF; + + return ret; +} + +static int aw_dev_dsp_update_container(struct aw_device *aw_dev, + unsigned char *data, unsigned int len, unsigned short base) +{ + u32 tmp_len; + int i, ret; + + ret = regmap_write(aw_dev->regmap, AW88399_DSPMADD_REG, base); + if (ret) + return ret; + + for (i = 0; i < len; i += AW88399_MAX_RAM_WRITE_BYTE_SIZE) { + tmp_len = min(len - i, AW88399_MAX_RAM_WRITE_BYTE_SIZE); + ret = regmap_raw_write(aw_dev->regmap, AW88399_DSPMDAT_REG, + &data[i], tmp_len); + if (ret) + return ret; + } + + return 0; +} + +static int aw_dev_get_ra(struct aw_cali_desc *cali_desc) +{ + struct aw_device *aw_dev = + container_of(cali_desc, struct aw_device, cali_desc); + u32 dsp_ra; + int ret; + + ret = aw_dev_dsp_read(aw_dev, AW88399_DSP_REG_CFG_ADPZ_RA, + &dsp_ra, AW_DSP_32_DATA); + if (ret) { + dev_err(aw_dev->dev, "read ra error"); + return ret; + } + + cali_desc->ra = AW88399_DSP_RE_TO_SHOW_RE(dsp_ra, + AW88399_DSP_RE_SHIFT); + + return 0; +} + +static int aw_dev_dsp_update_cfg(struct aw_device *aw_dev, + unsigned char *data, unsigned int len) +{ + int ret; + + dev_dbg(aw_dev->dev, "dsp config len:%d", len); + + if (!len || !data) { + dev_err(aw_dev->dev, "dsp config data is null or len is 0"); + return -EINVAL; + } + + ret = aw_dev_dsp_update_container(aw_dev, data, len, AW88399_DSP_CFG_ADDR); + if (ret) + return ret; + + aw_dev->dsp_cfg_len = len; + + ret = aw_dev_get_ra(&aw_dev->cali_desc); + + return ret; +} + +static int aw_dev_dsp_update_fw(struct aw_device *aw_dev, + unsigned char *data, unsigned int len) +{ + int ret; + + dev_dbg(aw_dev->dev, "dsp firmware len:%d", len); + + if (!len || !data) { + dev_err(aw_dev->dev, "dsp firmware data is null or len is 0"); + return -EINVAL; + } + + aw_dev->dsp_fw_len = len; + ret = aw_dev_dsp_update_container(aw_dev, data, len, AW88399_DSP_FW_ADDR); + + return ret; +} + +static int aw_dev_check_sram(struct aw_device *aw_dev) +{ + unsigned int reg_val; + + /* read dsp_rom_check_reg */ + aw_dev_dsp_read(aw_dev, AW88399_DSP_ROM_CHECK_ADDR, ®_val, AW_DSP_16_DATA); + if (reg_val != AW88399_DSP_ROM_CHECK_DATA) { + dev_err(aw_dev->dev, "check dsp rom failed, read[0x%x] != check[0x%x]", + reg_val, AW88399_DSP_ROM_CHECK_DATA); + return -EPERM; + } + + /* check dsp_cfg_base_addr */ + aw_dev_dsp_write(aw_dev, AW88399_DSP_CFG_ADDR, + AW88399_DSP_ODD_NUM_BIT_TEST, AW_DSP_16_DATA); + aw_dev_dsp_read(aw_dev, AW88399_DSP_CFG_ADDR, ®_val, AW_DSP_16_DATA); + if (reg_val != AW88399_DSP_ODD_NUM_BIT_TEST) { + dev_err(aw_dev->dev, "check dsp cfg failed, read[0x%x] != write[0x%x]", + reg_val, AW88399_DSP_ODD_NUM_BIT_TEST); + return -EPERM; + } + + return 0; +} + +static void aw_dev_select_memclk(struct aw_device *aw_dev, unsigned char flag) +{ + int ret; + + switch (flag) { + case AW88399_DEV_MEMCLK_PLL: + ret = regmap_update_bits(aw_dev->regmap, AW88399_DBGCTRL_REG, + ~AW88399_MEM_CLKSEL_MASK, + AW88399_MEM_CLKSEL_DAPHCLK_VALUE); + if (ret) + dev_err(aw_dev->dev, "memclk select pll failed"); + break; + case AW88399_DEV_MEMCLK_OSC: + ret = regmap_update_bits(aw_dev->regmap, AW88399_DBGCTRL_REG, + ~AW88399_MEM_CLKSEL_MASK, + AW88399_MEM_CLKSEL_OSCCLK_VALUE); + if (ret) + dev_err(aw_dev->dev, "memclk select OSC failed"); + break; + default: + dev_err(aw_dev->dev, "unknown memclk config, flag=0x%x", flag); + break; + } +} + +static void aw_dev_get_cur_mode_st(struct aw_device *aw_dev) +{ + struct aw_profctrl_desc *profctrl_desc = &aw_dev->profctrl_desc; + unsigned int reg_val; + int ret; + + ret = regmap_read(aw_dev->regmap, AW88399_SYSCTRL_REG, ®_val); + if (ret) { + dev_dbg(aw_dev->dev, "%s failed", __func__); + return; + } + if ((reg_val & (~AW88399_RCV_MODE_MASK)) == AW88399_RCV_MODE_RECEIVER_VALUE) + profctrl_desc->cur_mode = AW88399_RCV_MODE; + else + profctrl_desc->cur_mode = AW88399_NOT_RCV_MODE; +} + +static int aw_dev_update_reg_container(struct aw88399 *aw88399, + unsigned char *data, unsigned int len) +{ + struct aw_device *aw_dev = aw88399->aw_pa; + struct aw_volume_desc *vol_desc = &aw_dev->volume_desc; + u16 read_vol, reg_val; + int data_len, i, ret; + int16_t *reg_data; + u8 reg_addr; + + reg_data = (int16_t *)data; + data_len = len >> 1; + + if (data_len & 0x1) { + dev_err(aw_dev->dev, "data len:%d unsupported", data_len); + return -EINVAL; + } + + for (i = 0; i < data_len; i += 2) { + reg_addr = reg_data[i]; + reg_val = reg_data[i + 1]; + + if (reg_addr == AW88399_DSPVCALB_REG) { + aw88399->vcalb_init_val = reg_val; + continue; + } + + if (reg_addr == AW88399_SYSCTRL_REG) { + if (reg_val & (~AW88399_DSPBY_MASK)) + aw_dev->dsp_cfg = AW88399_DEV_DSP_BYPASS; + else + aw_dev->dsp_cfg = AW88399_DEV_DSP_WORK; + + reg_val &= (AW88399_HMUTE_MASK | AW88399_PWDN_MASK | + AW88399_DSPBY_MASK); + reg_val |= (AW88399_HMUTE_ENABLE_VALUE | AW88399_PWDN_POWER_DOWN_VALUE | + AW88399_DSPBY_BYPASS_VALUE); + } + + if (reg_addr == AW88399_I2SCTRL3_REG) { + reg_val &= AW88399_I2STXEN_MASK; + reg_val |= AW88399_I2STXEN_DISABLE_VALUE; + } + + if (reg_addr == AW88399_SYSCTRL2_REG) { + read_vol = (reg_val & (~AW88399_VOL_MASK)) >> + AW88399_VOL_START_BIT; + aw_dev->volume_desc.init_volume = read_vol; + } + + if (reg_addr == AW88399_DBGCTRL_REG) { + if ((reg_val & (~AW88399_EF_DBMD_MASK)) == AW88399_EF_DBMD_OR_VALUE) + aw88399->check_val = AW_EF_OR_CHECK; + else + aw88399->check_val = AW_EF_AND_CHECK; + + aw88399->dither_st = reg_val & (~AW88399_DITHER_EN_MASK); + } + + if (reg_addr == AW88399_CRCCTRL_REG) + aw88399->crc_init_val = reg_val; + + ret = regmap_write(aw_dev->regmap, reg_addr, reg_val); + if (ret) + return ret; + } + + aw_dev_pwd(aw_dev, false); + usleep_range(AW88399_1000_US, AW88399_1000_US + 10); + + aw_dev_get_cur_mode_st(aw_dev); + + if (aw_dev->prof_cur != aw_dev->prof_index) + vol_desc->ctl_volume = 0; + else + aw_dev_set_volume(aw_dev, vol_desc->ctl_volume); + + return 0; +} + +static int aw_dev_reg_update(struct aw88399 *aw88399, + unsigned char *data, unsigned int len) +{ + int ret; + + if (!len || !data) { + dev_err(aw88399->aw_pa->dev, "reg data is null or len is 0"); + return -EINVAL; + } + + ret = aw_dev_update_reg_container(aw88399, data, len); + if (ret) + dev_err(aw88399->aw_pa->dev, "reg update failed"); + + return ret; +} + +int aw88399_dev_get_prof_name(struct aw_device *aw_dev, int index, char **prof_name) +{ + struct aw_prof_info *prof_info = &aw_dev->prof_info; + struct aw_prof_desc *prof_desc; + + if ((index >= aw_dev->prof_info.count) || (index < 0)) { + dev_err(aw_dev->dev, "index[%d] overflow count[%d]", + index, aw_dev->prof_info.count); + return -EINVAL; + } + + prof_desc = &aw_dev->prof_info.prof_desc[index]; + + *prof_name = prof_info->prof_name_list[prof_desc->id]; + + return 0; +} +EXPORT_SYMBOL_GPL(aw88399_dev_get_prof_name); + +static int aw88399_dev_get_prof_data(struct aw_device *aw_dev, int index, + struct aw_prof_desc **prof_desc) +{ + if ((index >= aw_dev->prof_info.count) || (index < 0)) { + dev_err(aw_dev->dev, "%s: index[%d] overflow count[%d]\n", + __func__, index, aw_dev->prof_info.count); + return -EINVAL; + } + + *prof_desc = &aw_dev->prof_info.prof_desc[index]; + + return 0; +} + +static int aw88399_dev_fw_update(struct aw88399 *aw88399, bool up_dsp_fw_en, bool force_up_en) +{ + struct aw_device *aw_dev = aw88399->aw_pa; + struct aw_prof_desc *prof_index_desc; + struct aw_sec_data_desc *sec_desc; + char *prof_name; + int ret; + + if ((aw_dev->prof_cur == aw_dev->prof_index) && + (force_up_en == AW88399_FORCE_UPDATE_OFF)) { + dev_dbg(aw_dev->dev, "scene no change, not update"); + return 0; + } + + if (aw_dev->fw_status == AW88399_DEV_FW_FAILED) { + dev_err(aw_dev->dev, "fw status[%d] error", aw_dev->fw_status); + return -EPERM; + } + + ret = aw88399_dev_get_prof_name(aw_dev, aw_dev->prof_index, &prof_name); + if (ret) + return ret; + + dev_dbg(aw_dev->dev, "start update %s", prof_name); + + ret = aw88399_dev_get_prof_data(aw_dev, aw_dev->prof_index, &prof_index_desc); + if (ret) + return ret; + + /* update reg */ + sec_desc = prof_index_desc->sec_desc; + ret = aw_dev_reg_update(aw88399, sec_desc[AW88395_DATA_TYPE_REG].data, + sec_desc[AW88395_DATA_TYPE_REG].len); + if (ret) { + dev_err(aw_dev->dev, "update reg failed"); + return ret; + } + + aw88399_dev_mute(aw_dev, true); + + if (aw_dev->dsp_cfg == AW88399_DEV_DSP_WORK) + aw_dev_dsp_enable(aw_dev, false); + + aw_dev_select_memclk(aw_dev, AW88399_DEV_MEMCLK_OSC); + + ret = aw_dev_check_sram(aw_dev); + if (ret) { + dev_err(aw_dev->dev, "check sram failed"); + goto error; + } + + if (up_dsp_fw_en) { + dev_dbg(aw_dev->dev, "fw_ver: [%x]", prof_index_desc->fw_ver); + ret = aw_dev_dsp_update_fw(aw_dev, sec_desc[AW88395_DATA_TYPE_DSP_FW].data, + sec_desc[AW88395_DATA_TYPE_DSP_FW].len); + if (ret) { + dev_err(aw_dev->dev, "update dsp fw failed"); + goto error; + } + } + + /* update dsp config */ + ret = aw_dev_dsp_update_cfg(aw_dev, sec_desc[AW88395_DATA_TYPE_DSP_CFG].data, + sec_desc[AW88395_DATA_TYPE_DSP_CFG].len); + if (ret) { + dev_err(aw_dev->dev, "update dsp cfg failed"); + goto error; + } + + aw_dev_select_memclk(aw_dev, AW88399_DEV_MEMCLK_PLL); + + aw_dev->prof_cur = aw_dev->prof_index; + + return 0; + +error: + aw_dev_select_memclk(aw_dev, AW88399_DEV_MEMCLK_PLL); + return ret; +} + +static void aw88399_start_pa(struct aw88399 *aw88399) +{ + int ret, i; + + for (i = 0; i < AW88399_START_RETRIES; i++) { + ret = aw88399_dev_start(aw88399); + if (ret) { + dev_err(aw88399->aw_pa->dev, "aw88399 device start failed. retry = %d", i); + ret = aw88399_dev_fw_update(aw88399, AW88399_DSP_FW_UPDATE_ON, true); + if (ret) { + dev_err(aw88399->aw_pa->dev, "fw update failed"); + continue; + } + } else { + dev_dbg(aw88399->aw_pa->dev, "start success\n"); + break; + } + } +} + +void aw88399_startup_work(struct work_struct *work) +{ + struct aw88399 *aw88399 = + container_of(work, struct aw88399, start_work.work); + + mutex_lock(&aw88399->lock); + aw88399_start_pa(aw88399); + mutex_unlock(&aw88399->lock); +} +EXPORT_SYMBOL_GPL(aw88399_startup_work); + +void aw88399_start(struct aw88399 *aw88399, bool sync_start) +{ + int ret; + + if (aw88399->aw_pa->fw_status != AW88399_DEV_FW_OK) + return; + + if (aw88399->aw_pa->status == AW88399_DEV_PW_ON) + return; + + ret = aw88399_dev_fw_update(aw88399, aw88399->fw_needs_reload ? + AW88399_DSP_FW_UPDATE_ON : AW88399_DSP_FW_UPDATE_OFF, true); + if (ret) { + dev_err(aw88399->aw_pa->dev, "fw update failed."); + return; + } + + aw88399->fw_needs_reload = false; + + if (sync_start == AW88399_SYNC_START) + aw88399_start_pa(aw88399); + else + queue_delayed_work(system_dfl_wq, + &aw88399->start_work, + AW88399_START_WORK_DELAY_MS); +} +EXPORT_SYMBOL_GPL(aw88399_start); + +static int aw_dev_check_sysint(struct aw_device *aw_dev) +{ + u16 reg_val; + + aw_dev_get_int_status(aw_dev, ®_val); + if (reg_val & AW88399_BIT_SYSINT_CHECK) { + dev_err(aw_dev->dev, "pa stop check fail:0x%04x", reg_val); + return -EINVAL; + } + + return 0; +} + +int aw88399_stop(struct aw_device *aw_dev) +{ + struct aw_sec_data_desc *dsp_cfg = + &aw_dev->prof_info.prof_desc[aw_dev->prof_cur].sec_desc[AW88395_DATA_TYPE_DSP_CFG]; + struct aw_sec_data_desc *dsp_fw = + &aw_dev->prof_info.prof_desc[aw_dev->prof_cur].sec_desc[AW88395_DATA_TYPE_DSP_FW]; + int int_st; + + if (aw_dev->status == AW88399_DEV_PW_OFF) { + dev_dbg(aw_dev->dev, "already power off"); + return 0; + } + + aw_dev->status = AW88399_DEV_PW_OFF; + + aw88399_dev_mute(aw_dev, true); + usleep_range(AW88399_4000_US, AW88399_4000_US + 100); + + aw_dev_i2s_tx_enable(aw_dev, false); + usleep_range(AW88399_1000_US, AW88399_1000_US + 100); + + int_st = aw_dev_check_sysint(aw_dev); + + aw_dev_dsp_enable(aw_dev, false); + + aw_dev_amppd(aw_dev, true); + + if (int_st) { + aw_dev_select_memclk(aw_dev, AW88399_DEV_MEMCLK_OSC); + aw_dev_dsp_update_fw(aw_dev, dsp_fw->data, dsp_fw->len); + aw_dev_dsp_update_cfg(aw_dev, dsp_cfg->data, dsp_cfg->len); + aw_dev_select_memclk(aw_dev, AW88399_DEV_MEMCLK_PLL); + } + + aw_dev_pwd(aw_dev, true); + + return 0; +} +EXPORT_SYMBOL_GPL(aw88399_stop); + +static int aw88399_dev_init(struct aw88399 *aw88399, struct aw_container *aw_cfg) +{ + struct aw_device *aw_dev = aw88399->aw_pa; + int ret; + + ret = aw88395_dev_cfg_load(aw_dev, aw_cfg); + if (ret) { + dev_err(aw_dev->dev, "aw_dev acf parse failed"); + return -EINVAL; + } + aw_dev->fade_in_time = AW88399_1000_US / 10; + aw_dev->fade_out_time = AW88399_1000_US >> 1; + aw_dev->prof_cur = aw_dev->prof_info.prof_desc[0].id; + aw_dev->prof_index = aw_dev->prof_info.prof_desc[0].id; + + ret = aw88399_dev_fw_update(aw88399, AW88399_FORCE_UPDATE_ON, AW88399_DSP_FW_UPDATE_ON); + if (ret) { + dev_err(aw_dev->dev, "fw update failed ret = %d\n", ret); + return ret; + } + + aw88399_dev_mute(aw_dev, true); + + /* close tx feedback */ + aw_dev_i2s_tx_enable(aw_dev, false); + usleep_range(AW88399_1000_US, AW88399_1000_US + 100); + + /* enable amppd */ + aw_dev_amppd(aw_dev, true); + + /* close dsp */ + aw_dev_dsp_enable(aw_dev, false); + /* set power down */ + aw_dev_pwd(aw_dev, true); + + return 0; +} + +int aw88399_request_firmware_file(struct aw88399 *aw88399) +{ + const struct firmware *cont = NULL; + int ret; + + aw88399->aw_pa->fw_status = AW88399_DEV_FW_FAILED; + + ret = request_firmware(&cont, AW88399_ACF_FILE, aw88399->aw_pa->dev); + if (ret) { + dev_err(aw88399->aw_pa->dev, "request [%s] failed!", AW88399_ACF_FILE); + return ret; + } + + dev_dbg(aw88399->aw_pa->dev, "loaded %s - size: %zu\n", + AW88399_ACF_FILE, cont ? cont->size : 0); + + aw88399->aw_cfg = devm_kzalloc(aw88399->aw_pa->dev, + struct_size(aw88399->aw_cfg, data, cont->size), GFP_KERNEL); + if (!aw88399->aw_cfg) { + release_firmware(cont); + return -ENOMEM; + } + aw88399->aw_cfg->len = (int)cont->size; + memcpy(aw88399->aw_cfg->data, cont->data, cont->size); + release_firmware(cont); + + ret = aw88395_dev_load_acf_check(aw88399->aw_pa, aw88399->aw_cfg); + if (ret) { + dev_err(aw88399->aw_pa->dev, "load [%s] failed!", AW88399_ACF_FILE); + return ret; + } + + mutex_lock(&aw88399->lock); + /* aw device init */ + ret = aw88399_dev_init(aw88399, aw88399->aw_cfg); + if (ret) + dev_err(aw88399->aw_pa->dev, "dev init failed"); + mutex_unlock(&aw88399->lock); + + return ret; +} +EXPORT_SYMBOL_GPL(aw88399_request_firmware_file); + +void aw88399_hw_reset(struct aw88399 *aw88399) +{ + if (aw88399->reset_gpio) { + gpiod_set_value_cansleep(aw88399->reset_gpio, 1); + usleep_range(AW88399_1000_US, AW88399_1000_US + 10); + gpiod_set_value_cansleep(aw88399->reset_gpio, 0); + usleep_range(AW88399_1000_US, AW88399_1000_US + 10); + gpiod_set_value_cansleep(aw88399->reset_gpio, 1); + usleep_range(AW88399_1000_US, AW88399_1000_US + 10); + } +} +EXPORT_SYMBOL_GPL(aw88399_hw_reset); + +static void aw88399_parse_channel_dt(struct aw_device *aw_dev) +{ + struct device_node *np = aw_dev->dev->of_node; + u32 channel_value; + int ret; + + ret = of_property_read_u32(np, "awinic,audio-channel", &channel_value); + if (ret) { + /* + * On ACPI systems, DT properties don't exist. Derive channel + * from I2C address: 0x34 -> channel 0 (left), 0x35 -> channel 1 (right) + */ + aw_dev->channel = aw_dev->i2c->addr - 0x34; + dev_dbg(aw_dev->dev, + "DT channel property not found, using I2C address-based channel %d (addr 0x%02x)\n", + aw_dev->channel, aw_dev->i2c->addr); + return; + } + aw_dev->channel = channel_value; +} + +int aw88399_init(struct aw88399 *aw88399, struct i2c_client *i2c, struct regmap *regmap) +{ + struct aw_device *aw_dev; + unsigned int chip_id; + int ret; + + ret = regmap_read(regmap, AW88399_ID_REG, &chip_id); + if (ret) { + dev_err(&i2c->dev, "%s read chipid error. ret = %d", __func__, ret); + return ret; + } + if (chip_id != AW88399_CHIP_ID) { + dev_err(&i2c->dev, "unsupported device"); + return -ENXIO; + } + dev_dbg(&i2c->dev, "chip id = %x\n", chip_id); + + aw_dev = devm_kzalloc(&i2c->dev, sizeof(*aw_dev), GFP_KERNEL); + if (!aw_dev) + return -ENOMEM; + aw88399->aw_pa = aw_dev; + + aw_dev->i2c = i2c; + aw_dev->dev = &i2c->dev; + aw_dev->regmap = regmap; + mutex_init(&aw_dev->dsp_lock); + + aw_dev->chip_id = chip_id; + aw_dev->acf = NULL; + aw_dev->prof_info.prof_desc = NULL; + aw_dev->prof_info.count = 0; + aw_dev->prof_info.prof_type = AW88395_DEV_NONE_TYPE_ID; + aw_dev->channel = AW88399_DEV_DEFAULT_CH; + aw_dev->fw_status = AW88399_DEV_FW_FAILED; + + aw_dev->fade_step = AW88399_VOLUME_STEP_DB; + aw_dev->volume_desc.ctl_volume = AW88399_VOL_DEFAULT_VALUE; + + aw88399_parse_channel_dt(aw_dev); + + return 0; +} +EXPORT_SYMBOL_GPL(aw88399_init); + +void aw88399_dev_set_channel(struct aw88399 *aw88399, int channel) +{ + aw88399->aw_pa->channel = channel; +} +EXPORT_SYMBOL_GPL(aw88399_dev_set_channel); + +MODULE_DESCRIPTION("AW88399 common device library"); +MODULE_LICENSE("GPL"); diff -ruN --no-dereference -x .git a/sound/soc/codecs/aw88399.c b/sound/soc/codecs/aw88399.c --- a/sound/soc/codecs/aw88399.c 2026-10-11 15:27:50.000000000 +0200 +++ b/sound/soc/codecs/aw88399.c 2026-10-13 14:33:30.000000000 +0200 @@ -10,1217 +10,12 @@ #include #include #include -#include -#include #include #include #include #include "aw88399.h" #include "aw88395/aw88395_device.h" -static const struct regmap_config aw88399_remap_config = { - .val_bits = 16, - .reg_bits = 8, - .max_register = AW88399_REG_MAX, - .reg_format_endian = REGMAP_ENDIAN_LITTLE, - .val_format_endian = REGMAP_ENDIAN_BIG, -}; - -static void aw_dev_pwd(struct aw_device *aw_dev, bool pwd) -{ - int ret; - - if (pwd) - ret = regmap_update_bits(aw_dev->regmap, AW88399_SYSCTRL_REG, - ~AW88399_PWDN_MASK, AW88399_PWDN_POWER_DOWN_VALUE); - else - ret = regmap_update_bits(aw_dev->regmap, AW88399_SYSCTRL_REG, - ~AW88399_PWDN_MASK, AW88399_PWDN_WORKING_VALUE); - - if (ret) - dev_dbg(aw_dev->dev, "%s failed", __func__); -} - -static void aw_dev_get_int_status(struct aw_device *aw_dev, unsigned short *int_status) -{ - unsigned int reg_val; - int ret; - - ret = regmap_read(aw_dev->regmap, AW88399_SYSINT_REG, ®_val); - if (ret) - dev_err(aw_dev->dev, "read interrupt reg fail, ret=%d", ret); - else - *int_status = reg_val; - - dev_dbg(aw_dev->dev, "read interrupt reg=0x%04x", *int_status); -} - -static void aw_dev_clear_int_status(struct aw_device *aw_dev) -{ - u16 int_status; - - /* read int status and clear */ - aw_dev_get_int_status(aw_dev, &int_status); - /* make sure int status is clear */ - aw_dev_get_int_status(aw_dev, &int_status); - if (int_status) - dev_dbg(aw_dev->dev, "int status(%d) is not cleaned.\n", int_status); -} - -static int aw_dev_get_iis_status(struct aw_device *aw_dev) -{ - unsigned int reg_val; - int ret; - - ret = regmap_read(aw_dev->regmap, AW88399_SYSST_REG, ®_val); - if (ret) - return ret; - if ((reg_val & AW88399_BIT_PLL_CHECK) != AW88399_BIT_PLL_CHECK) { - dev_err(aw_dev->dev, "check pll lock fail, reg_val:0x%04x", reg_val); - return -EINVAL; - } - - return 0; -} - -static int aw_dev_check_mode1_pll(struct aw_device *aw_dev) -{ - int ret, i; - - for (i = 0; i < AW88399_DEV_SYSST_CHECK_MAX; i++) { - ret = aw_dev_get_iis_status(aw_dev); - if (ret) { - dev_err(aw_dev->dev, "mode1 iis signal check error"); - usleep_range(AW88399_2000_US, AW88399_2000_US + 10); - } else { - return 0; - } - } - - return -EPERM; -} - -static int aw_dev_check_mode2_pll(struct aw_device *aw_dev) -{ - unsigned int reg_val; - int ret, i; - - ret = regmap_read(aw_dev->regmap, AW88399_PLLCTRL2_REG, ®_val); - if (ret) - return ret; - - reg_val &= (~AW88399_CCO_MUX_MASK); - if (reg_val == AW88399_CCO_MUX_DIVIDED_VALUE) { - dev_dbg(aw_dev->dev, "CCO_MUX is already divider"); - return -EPERM; - } - - /* change mode2 */ - ret = regmap_update_bits(aw_dev->regmap, AW88399_PLLCTRL2_REG, - ~AW88399_CCO_MUX_MASK, AW88399_CCO_MUX_DIVIDED_VALUE); - if (ret) - return ret; - - for (i = 0; i < AW88399_DEV_SYSST_CHECK_MAX; i++) { - ret = aw_dev_get_iis_status(aw_dev); - if (ret) { - dev_err(aw_dev->dev, "mode2 iis signal check error"); - usleep_range(AW88399_2000_US, AW88399_2000_US + 10); - } else { - break; - } - } - - /* change mode1 */ - regmap_update_bits(aw_dev->regmap, AW88399_PLLCTRL2_REG, - ~AW88399_CCO_MUX_MASK, AW88399_CCO_MUX_BYPASS_VALUE); - if (ret == 0) { - usleep_range(AW88399_2000_US, AW88399_2000_US + 10); - for (i = 0; i < AW88399_DEV_SYSST_CHECK_MAX; i++) { - ret = aw_dev_get_iis_status(aw_dev); - if (ret) { - dev_err(aw_dev->dev, "mode2 switch to mode1, iis signal check error"); - usleep_range(AW88399_2000_US, AW88399_2000_US + 10); - } else { - break; - } - } - } - - return ret; -} - -static int aw_dev_check_syspll(struct aw_device *aw_dev) -{ - int ret; - - ret = aw_dev_check_mode1_pll(aw_dev); - if (ret) { - dev_dbg(aw_dev->dev, "mode1 check iis failed try switch to mode2 check"); - ret = aw_dev_check_mode2_pll(aw_dev); - if (ret) { - dev_err(aw_dev->dev, "mode2 check iis failed"); - return ret; - } - } - - return 0; -} - -static int aw_dev_check_sysst(struct aw_device *aw_dev) -{ - unsigned int check_val; - unsigned int reg_val; - int ret, i; - - ret = regmap_read(aw_dev->regmap, AW88399_PWMCTRL3_REG, ®_val); - if (ret) - return ret; - - if (reg_val & (~AW88399_NOISE_GATE_EN_MASK)) - check_val = AW88399_BIT_SYSST_NOSWS_CHECK; - else - check_val = AW88399_BIT_SYSST_SWS_CHECK; - - for (i = 0; i < AW88399_DEV_SYSST_CHECK_MAX; i++) { - ret = regmap_read(aw_dev->regmap, AW88399_SYSST_REG, ®_val); - if (ret) - return ret; - - if ((reg_val & (~AW88399_BIT_SYSST_CHECK_MASK) & check_val) != check_val) { - dev_err(aw_dev->dev, "check sysst fail, cnt=%d, reg_val=0x%04x, check:0x%x", - i, reg_val, AW88399_BIT_SYSST_NOSWS_CHECK); - usleep_range(AW88399_2000_US, AW88399_2000_US + 10); - } else { - return 0; - } - } - - return -EPERM; -} - -static void aw_dev_amppd(struct aw_device *aw_dev, bool amppd) -{ - int ret; - - if (amppd) - ret = regmap_update_bits(aw_dev->regmap, AW88399_SYSCTRL_REG, - ~AW88399_AMPPD_MASK, AW88399_AMPPD_POWER_DOWN_VALUE); - else - ret = regmap_update_bits(aw_dev->regmap, AW88399_SYSCTRL_REG, - ~AW88399_AMPPD_MASK, AW88399_AMPPD_WORKING_VALUE); - - if (ret) - dev_dbg(aw_dev->dev, "%s failed", __func__); -} - -static void aw_dev_dsp_enable(struct aw_device *aw_dev, bool is_enable) -{ - int ret; - - if (is_enable) - ret = regmap_update_bits(aw_dev->regmap, AW88399_SYSCTRL_REG, - ~AW88399_DSPBY_MASK, AW88399_DSPBY_WORKING_VALUE); - else - ret = regmap_update_bits(aw_dev->regmap, AW88399_SYSCTRL_REG, - ~AW88399_DSPBY_MASK, AW88399_DSPBY_BYPASS_VALUE); - - if (ret) - dev_dbg(aw_dev->dev, "%s failed\n", __func__); -} - -static int aw88399_dev_get_icalk(struct aw88399 *aw88399, int16_t *icalk) -{ - uint16_t icalkh_val, icalkl_val, icalk_val; - struct aw_device *aw_dev = aw88399->aw_pa; - unsigned int reg_val; - int ret; - - ret = regmap_read(aw_dev->regmap, AW88399_EFRH4_REG, ®_val); - if (ret) - return ret; - icalkh_val = reg_val & (~AW88399_EF_ISN_GESLP_H_MASK); - - ret = regmap_read(aw_dev->regmap, AW88399_EFRL4_REG, ®_val); - if (ret) - return ret; - icalkl_val = reg_val & (~AW88399_EF_ISN_GESLP_L_MASK); - - if (aw88399->check_val == AW_EF_AND_CHECK) - icalk_val = icalkh_val & icalkl_val; - else - icalk_val = icalkh_val | icalkl_val; - - if (icalk_val & (~AW88399_EF_ISN_GESLP_SIGN_MASK)) - icalk_val = icalk_val | AW88399_EF_ISN_GESLP_SIGN_NEG; - *icalk = (int16_t)icalk_val; - - return 0; -} - -static int aw88399_dev_get_vcalk(struct aw88399 *aw88399, int16_t *vcalk) -{ - uint16_t vcalkh_val, vcalkl_val, vcalk_val; - struct aw_device *aw_dev = aw88399->aw_pa; - unsigned int reg_val; - int ret; - - ret = regmap_read(aw_dev->regmap, AW88399_EFRH3_REG, ®_val); - if (ret) - return ret; - - vcalkh_val = reg_val & (~AW88399_EF_VSN_GESLP_H_MASK); - - ret = regmap_read(aw_dev->regmap, AW88399_EFRL3_REG, ®_val); - if (ret) - return ret; - - vcalkl_val = reg_val & (~AW88399_EF_VSN_GESLP_L_MASK); - - if (aw88399->check_val == AW_EF_AND_CHECK) - vcalk_val = vcalkh_val & vcalkl_val; - else - vcalk_val = vcalkh_val | vcalkl_val; - - if (vcalk_val & AW88399_EF_VSN_GESLP_SIGN_MASK) - vcalk_val = vcalk_val | AW88399_EF_VSN_GESLP_SIGN_NEG; - *vcalk = (int16_t)vcalk_val; - - return 0; -} - -static int aw88399_dev_get_internal_vcalk(struct aw88399 *aw88399, int16_t *vcalk) -{ - uint16_t vcalkh_val, vcalkl_val, vcalk_val; - struct aw_device *aw_dev = aw88399->aw_pa; - unsigned int reg_val; - int ret; - - ret = regmap_read(aw_dev->regmap, AW88399_EFRH2_REG, ®_val); - if (ret) - return ret; - vcalkh_val = reg_val & (~AW88399_INTERNAL_VSN_TRIM_H_MASK); - - ret = regmap_read(aw_dev->regmap, AW88399_EFRL2_REG, ®_val); - if (ret) - return ret; - vcalkl_val = reg_val & (~AW88399_INTERNAL_VSN_TRIM_L_MASK); - - if (aw88399->check_val == AW_EF_AND_CHECK) - vcalk_val = (vcalkh_val >> AW88399_INTERNAL_VSN_TRIM_H_START_BIT) & - (vcalkl_val >> AW88399_INTERNAL_VSN_TRIM_L_START_BIT); - else - vcalk_val = (vcalkh_val >> AW88399_INTERNAL_VSN_TRIM_H_START_BIT) | - (vcalkl_val >> AW88399_INTERNAL_VSN_TRIM_L_START_BIT); - - if (vcalk_val & (~AW88399_TEM4_SIGN_MASK)) - vcalk_val = vcalk_val | AW88399_TEM4_SIGN_NEG; - - *vcalk = (int16_t)vcalk_val; - - return 0; -} - -static int aw_dev_set_vcalb(struct aw88399 *aw88399) -{ - struct aw_device *aw_dev = aw88399->aw_pa; - unsigned int vsense_select, vsense_value; - int32_t ical_k, vcal_k, vcalb; - int16_t icalk, vcalk; - uint16_t reg_val; - int ret; - - ret = regmap_read(aw_dev->regmap, AW88399_VSNCTRL1_REG, &vsense_value); - if (ret) - return ret; - - vsense_select = vsense_value & (~AW88399_VDSEL_MASK); - - ret = aw88399_dev_get_icalk(aw88399, &icalk); - if (ret) { - dev_err(aw_dev->dev, "get icalk failed\n"); - return ret; - } - - ical_k = icalk * AW88399_ICABLK_FACTOR + AW88399_CABL_BASE_VALUE; - - switch (vsense_select) { - case AW88399_DEV_VDSEL_VSENSE: - ret = aw88399_dev_get_vcalk(aw88399, &vcalk); - vcal_k = vcalk * AW88399_VCABLK_FACTOR + AW88399_CABL_BASE_VALUE; - vcalb = AW88399_VCALB_ACCURACY * AW88399_VSCAL_FACTOR / AW88399_ISCAL_FACTOR * - ical_k / vcal_k * aw88399->vcalb_init_val; - break; - case AW88399_DEV_VDSEL_DAC: - ret = aw88399_dev_get_internal_vcalk(aw88399, &vcalk); - vcal_k = vcalk * AW88399_VCABLK_DAC_FACTOR + AW88399_CABL_BASE_VALUE; - vcalb = AW88399_VCALB_ACCURACY * AW88399_VSCAL_DAC_FACTOR / - AW88399_ISCAL_DAC_FACTOR * ical_k / - vcal_k * aw88399->vcalb_init_val; - break; - default: - dev_err(aw_dev->dev, "%s: unsupported vsense\n", __func__); - ret = -EINVAL; - break; - } - if (ret) - return ret; - - vcalb = vcalb >> AW88399_VCALB_ADJ_FACTOR; - reg_val = (uint32_t)vcalb; - - regmap_write(aw_dev->regmap, AW88399_DSPVCALB_REG, reg_val); - - return 0; -} - -static int aw_dev_update_cali_re(struct aw_cali_desc *cali_desc) -{ - struct aw_device *aw_dev = - container_of(cali_desc, struct aw_device, cali_desc); - uint16_t re_lbits, re_hbits; - u32 cali_re; - int ret; - - if ((aw_dev->cali_desc.cali_re >= AW88399_CALI_RE_MAX) || - (aw_dev->cali_desc.cali_re <= AW88399_CALI_RE_MIN)) - return -EINVAL; - - cali_re = AW88399_SHOW_RE_TO_DSP_RE((aw_dev->cali_desc.cali_re + - aw_dev->cali_desc.ra), AW88399_DSP_RE_SHIFT); - - re_hbits = (cali_re & (~AW88399_CALI_RE_HBITS_MASK)) >> AW88399_CALI_RE_HBITS_SHIFT; - re_lbits = (cali_re & (~AW88399_CALI_RE_LBITS_MASK)) >> AW88399_CALI_RE_LBITS_SHIFT; - - ret = regmap_write(aw_dev->regmap, AW88399_ACR1_REG, re_hbits); - if (ret) { - dev_err(aw_dev->dev, "set cali re error"); - return ret; - } - - ret = regmap_write(aw_dev->regmap, AW88399_ACR2_REG, re_lbits); - if (ret) - dev_err(aw_dev->dev, "set cali re error"); - - return ret; -} - -static int aw_dev_fw_crc_check(struct aw_device *aw_dev) -{ - uint16_t check_val, fw_len_val; - unsigned int reg_val; - int ret; - - /* calculate fw_end_addr */ - fw_len_val = ((aw_dev->dsp_fw_len / AW_FW_ADDR_LEN) - 1) + AW88399_CRC_FW_BASE_ADDR; - - /* write fw_end_addr to crc_end_addr */ - ret = regmap_update_bits(aw_dev->regmap, AW88399_CRCCTRL_REG, - ~AW88399_CRC_END_ADDR_MASK, fw_len_val); - if (ret) - return ret; - /* enable fw crc check */ - ret = regmap_update_bits(aw_dev->regmap, AW88399_CRCCTRL_REG, - ~AW88399_CRC_CODE_EN_MASK, AW88399_CRC_CODE_EN_ENABLE_VALUE); - - usleep_range(AW88399_2000_US, AW88399_2000_US + 10); - - /* read crc check result */ - regmap_read(aw_dev->regmap, AW88399_HAGCST_REG, ®_val); - if (ret) - return ret; - - check_val = (reg_val & (~AW88399_CRC_CHECK_BITS_MASK)) >> AW88399_CRC_CHECK_START_BIT; - - /* disable fw crc check */ - ret = regmap_update_bits(aw_dev->regmap, AW88399_CRCCTRL_REG, - ~AW88399_CRC_CODE_EN_MASK, AW88399_CRC_CODE_EN_DISABLE_VALUE); - if (ret) - return ret; - - if (check_val != AW88399_CRC_CHECK_PASS_VAL) { - dev_err(aw_dev->dev, "%s failed, check_val 0x%x != 0x%x", - __func__, check_val, AW88399_CRC_CHECK_PASS_VAL); - ret = -EINVAL; - } - - return ret; -} - -static int aw_dev_cfg_crc_check(struct aw_device *aw_dev) -{ - uint16_t check_val, cfg_len_val; - unsigned int reg_val; - int ret; - - /* calculate cfg end addr */ - cfg_len_val = ((aw_dev->dsp_cfg_len / AW_FW_ADDR_LEN) - 1) + AW88399_CRC_CFG_BASE_ADDR; - - /* write cfg_end_addr to crc_end_addr */ - ret = regmap_update_bits(aw_dev->regmap, AW88399_CRCCTRL_REG, - ~AW88399_CRC_END_ADDR_MASK, cfg_len_val); - if (ret) - return ret; - - /* enable cfg crc check */ - ret = regmap_update_bits(aw_dev->regmap, AW88399_CRCCTRL_REG, - ~AW88399_CRC_CFG_EN_MASK, AW88399_CRC_CFG_EN_ENABLE_VALUE); - if (ret) - return ret; - - usleep_range(AW88399_1000_US, AW88399_1000_US + 10); - - /* read crc check result */ - ret = regmap_read(aw_dev->regmap, AW88399_HAGCST_REG, ®_val); - if (ret) - return ret; - - check_val = (reg_val & (~AW88399_CRC_CHECK_BITS_MASK)) >> AW88399_CRC_CHECK_START_BIT; - - /* disable cfg crc check */ - ret = regmap_update_bits(aw_dev->regmap, AW88399_CRCCTRL_REG, - ~AW88399_CRC_CFG_EN_MASK, AW88399_CRC_CFG_EN_DISABLE_VALUE); - if (ret) - return ret; - - if (check_val != AW88399_CRC_CHECK_PASS_VAL) { - dev_err(aw_dev->dev, "crc_check failed, check val 0x%x != 0x%x", - check_val, AW88399_CRC_CHECK_PASS_VAL); - ret = -EINVAL; - } - - return ret; -} - -static int aw_dev_hw_crc_check(struct aw88399 *aw88399) -{ - struct aw_device *aw_dev = aw88399->aw_pa; - int ret; - - ret = regmap_update_bits(aw_dev->regmap, AW88399_I2SCFG1_REG, - ~AW88399_RAM_CG_BYP_MASK, AW88399_RAM_CG_BYP_BYPASS_VALUE); - if (ret) - return ret; - - ret = aw_dev_fw_crc_check(aw_dev); - if (ret) { - dev_err(aw_dev->dev, "fw_crc_check failed\n"); - goto crc_check_failed; - } - - ret = aw_dev_cfg_crc_check(aw_dev); - if (ret) { - dev_err(aw_dev->dev, "cfg_crc_check failed\n"); - goto crc_check_failed; - } - - ret = regmap_write(aw_dev->regmap, AW88399_CRCCTRL_REG, aw88399->crc_init_val); - if (ret) - return ret; - - ret = regmap_update_bits(aw_dev->regmap, AW88399_I2SCFG1_REG, - ~AW88399_RAM_CG_BYP_MASK, AW88399_RAM_CG_BYP_WORK_VALUE); - - return ret; - -crc_check_failed: - regmap_update_bits(aw_dev->regmap, AW88399_I2SCFG1_REG, - ~AW88399_RAM_CG_BYP_MASK, AW88399_RAM_CG_BYP_WORK_VALUE); - return ret; -} - -static void aw_dev_i2s_tx_enable(struct aw_device *aw_dev, bool flag) -{ - int ret; - - if (flag) - ret = regmap_update_bits(aw_dev->regmap, AW88399_I2SCTRL3_REG, - ~AW88399_I2STXEN_MASK, AW88399_I2STXEN_ENABLE_VALUE); - else - ret = regmap_update_bits(aw_dev->regmap, AW88399_I2SCFG1_REG, - ~AW88399_I2STXEN_MASK, AW88399_I2STXEN_DISABLE_VALUE); - - if (ret) - dev_dbg(aw_dev->dev, "%s failed", __func__); -} - -static int aw_dev_get_dsp_status(struct aw_device *aw_dev) -{ - unsigned int reg_val; - int ret; - - ret = regmap_read(aw_dev->regmap, AW88399_WDT_REG, ®_val); - if (ret) - return ret; - if (!(reg_val & (~AW88399_WDT_CNT_MASK))) - return -EPERM; - - return 0; -} - -static int aw_dev_dsp_check(struct aw_device *aw_dev) -{ - int ret, i; - - switch (aw_dev->dsp_cfg) { - case AW88399_DEV_DSP_BYPASS: - dev_dbg(aw_dev->dev, "dsp bypass"); - ret = 0; - break; - case AW88399_DEV_DSP_WORK: - aw_dev_dsp_enable(aw_dev, false); - aw_dev_dsp_enable(aw_dev, true); - usleep_range(AW88399_1000_US, AW88399_1000_US + 10); - for (i = 0; i < AW88399_DEV_DSP_CHECK_MAX; i++) { - ret = aw_dev_get_dsp_status(aw_dev); - if (ret) { - dev_err(aw_dev->dev, "dsp wdt status error=%d", ret); - usleep_range(AW88399_2000_US, AW88399_2000_US + 10); - } - } - break; - default: - dev_err(aw_dev->dev, "unknown dsp cfg=%d", aw_dev->dsp_cfg); - ret = -EINVAL; - break; - } - - return ret; -} - -static int aw_dev_set_volume(struct aw_device *aw_dev, unsigned int value) -{ - struct aw_volume_desc *vol_desc = &aw_dev->volume_desc; - unsigned int reg_value; - u16 real_value; - int ret; - - real_value = min((value + vol_desc->init_volume), (unsigned int)AW88399_MUTE_VOL); - - ret = regmap_read(aw_dev->regmap, AW88399_SYSCTRL2_REG, ®_value); - if (ret) - return ret; - - dev_dbg(aw_dev->dev, "value 0x%x , reg:0x%x", value, real_value); - - real_value = (real_value << AW88399_VOL_START_BIT) | (reg_value & AW88399_VOL_MASK); - - ret = regmap_write(aw_dev->regmap, AW88399_SYSCTRL2_REG, real_value); - - return ret; -} - -static void aw_dev_fade_in(struct aw_device *aw_dev) -{ - struct aw_volume_desc *desc = &aw_dev->volume_desc; - u16 fade_in_vol = desc->ctl_volume; - int fade_step = aw_dev->fade_step; - int i; - - if (fade_step == 0 || aw_dev->fade_in_time == 0) { - aw_dev_set_volume(aw_dev, fade_in_vol); - return; - } - - for (i = AW88399_MUTE_VOL; i >= fade_in_vol; i -= fade_step) { - aw_dev_set_volume(aw_dev, i); - usleep_range(aw_dev->fade_in_time, aw_dev->fade_in_time + 10); - } - - if (i != fade_in_vol) - aw_dev_set_volume(aw_dev, fade_in_vol); -} - -static void aw_dev_fade_out(struct aw_device *aw_dev) -{ - struct aw_volume_desc *desc = &aw_dev->volume_desc; - int fade_step = aw_dev->fade_step; - int i; - - if (fade_step == 0 || aw_dev->fade_out_time == 0) { - aw_dev_set_volume(aw_dev, AW88399_MUTE_VOL); - return; - } - - for (i = desc->ctl_volume; i <= AW88399_MUTE_VOL; i += fade_step) { - aw_dev_set_volume(aw_dev, i); - usleep_range(aw_dev->fade_out_time, aw_dev->fade_out_time + 10); - } - - if (i != AW88399_MUTE_VOL) { - aw_dev_set_volume(aw_dev, AW88399_MUTE_VOL); - usleep_range(aw_dev->fade_out_time, aw_dev->fade_out_time + 10); - } -} - -static void aw88399_dev_mute(struct aw_device *aw_dev, bool is_mute) -{ - if (is_mute) { - aw_dev_fade_out(aw_dev); - regmap_update_bits(aw_dev->regmap, AW88399_SYSCTRL_REG, - ~AW88399_HMUTE_MASK, AW88399_HMUTE_ENABLE_VALUE); - } else { - regmap_update_bits(aw_dev->regmap, AW88399_SYSCTRL_REG, - ~AW88399_HMUTE_MASK, AW88399_HMUTE_DISABLE_VALUE); - aw_dev_fade_in(aw_dev); - } -} - -static void aw88399_dev_set_dither(struct aw88399 *aw88399, bool dither) -{ - struct aw_device *aw_dev = aw88399->aw_pa; - - if (dither) - regmap_update_bits(aw_dev->regmap, AW88399_DBGCTRL_REG, - ~AW88399_DITHER_EN_MASK, AW88399_DITHER_EN_ENABLE_VALUE); - else - regmap_update_bits(aw_dev->regmap, AW88399_DBGCTRL_REG, - ~AW88399_DITHER_EN_MASK, AW88399_DITHER_EN_DISABLE_VALUE); -} - -static int aw88399_dev_start(struct aw88399 *aw88399) -{ - struct aw_device *aw_dev = aw88399->aw_pa; - int ret; - - if (aw_dev->status == AW88399_DEV_PW_ON) { - dev_dbg(aw_dev->dev, "already power on"); - return 0; - } - - aw88399_dev_set_dither(aw88399, false); - - /* power on */ - aw_dev_pwd(aw_dev, false); - usleep_range(AW88399_2000_US, AW88399_2000_US + 10); - - ret = aw_dev_check_syspll(aw_dev); - if (ret) { - dev_err(aw_dev->dev, "pll check failed cannot start"); - goto pll_check_fail; - } - - /* amppd on */ - aw_dev_amppd(aw_dev, false); - usleep_range(AW88399_1000_US, AW88399_1000_US + 50); - - /* check i2s status */ - ret = aw_dev_check_sysst(aw_dev); - if (ret) { - dev_err(aw_dev->dev, "sysst check failed"); - goto sysst_check_fail; - } - - if (aw_dev->dsp_cfg == AW88399_DEV_DSP_WORK) { - ret = aw_dev_hw_crc_check(aw88399); - if (ret) { - dev_err(aw_dev->dev, "dsp crc check failed"); - goto crc_check_fail; - } - aw_dev_dsp_enable(aw_dev, false); - aw_dev_set_vcalb(aw88399); - aw_dev_update_cali_re(&aw_dev->cali_desc); - - ret = aw_dev_dsp_check(aw_dev); - if (ret) { - dev_err(aw_dev->dev, "dsp status check failed"); - goto dsp_check_fail; - } - } else { - dev_dbg(aw_dev->dev, "start pa with dsp bypass"); - } - - /* enable tx feedback */ - aw_dev_i2s_tx_enable(aw_dev, true); - - if (aw88399->dither_st == AW88399_DITHER_EN_ENABLE_VALUE) - aw88399_dev_set_dither(aw88399, true); - - /* close mute */ - aw88399_dev_mute(aw_dev, false); - /* clear inturrupt */ - aw_dev_clear_int_status(aw_dev); - aw_dev->status = AW88399_DEV_PW_ON; - - return 0; - -dsp_check_fail: -crc_check_fail: - aw_dev_dsp_enable(aw_dev, false); -sysst_check_fail: - aw_dev_clear_int_status(aw_dev); - aw_dev_amppd(aw_dev, true); -pll_check_fail: - aw_dev_pwd(aw_dev, true); - aw_dev->status = AW88399_DEV_PW_OFF; - - return ret; -} - -static int aw_dev_dsp_update_container(struct aw_device *aw_dev, - unsigned char *data, unsigned int len, unsigned short base) -{ - u32 tmp_len; - int i, ret; - - ret = regmap_write(aw_dev->regmap, AW88399_DSPMADD_REG, base); - if (ret) - return ret; - - for (i = 0; i < len; i += AW88399_MAX_RAM_WRITE_BYTE_SIZE) { - tmp_len = min(len - i, AW88399_MAX_RAM_WRITE_BYTE_SIZE); - ret = regmap_raw_write(aw_dev->regmap, AW88399_DSPMDAT_REG, - &data[i], tmp_len); - if (ret) - return ret; - } - - return 0; -} - -static int aw_dev_get_ra(struct aw_cali_desc *cali_desc) -{ - struct aw_device *aw_dev = - container_of(cali_desc, struct aw_device, cali_desc); - u32 dsp_ra; - int ret; - - ret = aw_dev_dsp_read(aw_dev, AW88399_DSP_REG_CFG_ADPZ_RA, - &dsp_ra, AW_DSP_32_DATA); - if (ret) { - dev_err(aw_dev->dev, "read ra error"); - return ret; - } - - cali_desc->ra = AW88399_DSP_RE_TO_SHOW_RE(dsp_ra, - AW88399_DSP_RE_SHIFT); - - return 0; -} - -static int aw_dev_dsp_update_cfg(struct aw_device *aw_dev, - unsigned char *data, unsigned int len) -{ - int ret; - - dev_dbg(aw_dev->dev, "dsp config len:%d", len); - - if (!len || !data) { - dev_err(aw_dev->dev, "dsp config data is null or len is 0"); - return -EINVAL; - } - - ret = aw_dev_dsp_update_container(aw_dev, data, len, AW88399_DSP_CFG_ADDR); - if (ret) - return ret; - - aw_dev->dsp_cfg_len = len; - - ret = aw_dev_get_ra(&aw_dev->cali_desc); - - return ret; -} - -static int aw_dev_dsp_update_fw(struct aw_device *aw_dev, - unsigned char *data, unsigned int len) -{ - int ret; - - dev_dbg(aw_dev->dev, "dsp firmware len:%d", len); - - if (!len || !data) { - dev_err(aw_dev->dev, "dsp firmware data is null or len is 0"); - return -EINVAL; - } - - aw_dev->dsp_fw_len = len; - ret = aw_dev_dsp_update_container(aw_dev, data, len, AW88399_DSP_FW_ADDR); - - return ret; -} - -static int aw_dev_check_sram(struct aw_device *aw_dev) -{ - unsigned int reg_val; - - /* read dsp_rom_check_reg */ - aw_dev_dsp_read(aw_dev, AW88399_DSP_ROM_CHECK_ADDR, ®_val, AW_DSP_16_DATA); - if (reg_val != AW88399_DSP_ROM_CHECK_DATA) { - dev_err(aw_dev->dev, "check dsp rom failed, read[0x%x] != check[0x%x]", - reg_val, AW88399_DSP_ROM_CHECK_DATA); - return -EPERM; - } - - /* check dsp_cfg_base_addr */ - aw_dev_dsp_write(aw_dev, AW88399_DSP_CFG_ADDR, - AW88399_DSP_ODD_NUM_BIT_TEST, AW_DSP_16_DATA); - aw_dev_dsp_read(aw_dev, AW88399_DSP_CFG_ADDR, ®_val, AW_DSP_16_DATA); - if (reg_val != AW88399_DSP_ODD_NUM_BIT_TEST) { - dev_err(aw_dev->dev, "check dsp cfg failed, read[0x%x] != write[0x%x]", - reg_val, AW88399_DSP_ODD_NUM_BIT_TEST); - return -EPERM; - } - - return 0; -} - -static void aw_dev_select_memclk(struct aw_device *aw_dev, unsigned char flag) -{ - int ret; - - switch (flag) { - case AW88399_DEV_MEMCLK_PLL: - ret = regmap_update_bits(aw_dev->regmap, AW88399_DBGCTRL_REG, - ~AW88399_MEM_CLKSEL_MASK, - AW88399_MEM_CLKSEL_DAPHCLK_VALUE); - if (ret) - dev_err(aw_dev->dev, "memclk select pll failed"); - break; - case AW88399_DEV_MEMCLK_OSC: - ret = regmap_update_bits(aw_dev->regmap, AW88399_DBGCTRL_REG, - ~AW88399_MEM_CLKSEL_MASK, - AW88399_MEM_CLKSEL_OSCCLK_VALUE); - if (ret) - dev_err(aw_dev->dev, "memclk select OSC failed"); - break; - default: - dev_err(aw_dev->dev, "unknown memclk config, flag=0x%x", flag); - break; - } -} - -static void aw_dev_get_cur_mode_st(struct aw_device *aw_dev) -{ - struct aw_profctrl_desc *profctrl_desc = &aw_dev->profctrl_desc; - unsigned int reg_val; - int ret; - - ret = regmap_read(aw_dev->regmap, AW88399_SYSCTRL_REG, ®_val); - if (ret) { - dev_dbg(aw_dev->dev, "%s failed", __func__); - return; - } - if ((reg_val & (~AW88399_RCV_MODE_MASK)) == AW88399_RCV_MODE_RECEIVER_VALUE) - profctrl_desc->cur_mode = AW88399_RCV_MODE; - else - profctrl_desc->cur_mode = AW88399_NOT_RCV_MODE; -} - -static int aw_dev_update_reg_container(struct aw88399 *aw88399, - unsigned char *data, unsigned int len) -{ - struct aw_device *aw_dev = aw88399->aw_pa; - struct aw_volume_desc *vol_desc = &aw_dev->volume_desc; - u16 read_vol, reg_val; - int data_len, i, ret; - int16_t *reg_data; - u8 reg_addr; - - reg_data = (int16_t *)data; - data_len = len >> 1; - - if (data_len & 0x1) { - dev_err(aw_dev->dev, "data len:%d unsupported", data_len); - return -EINVAL; - } - - for (i = 0; i < data_len; i += 2) { - reg_addr = reg_data[i]; - reg_val = reg_data[i + 1]; - - if (reg_addr == AW88399_DSPVCALB_REG) { - aw88399->vcalb_init_val = reg_val; - continue; - } - - if (reg_addr == AW88399_SYSCTRL_REG) { - if (reg_val & (~AW88399_DSPBY_MASK)) - aw_dev->dsp_cfg = AW88399_DEV_DSP_BYPASS; - else - aw_dev->dsp_cfg = AW88399_DEV_DSP_WORK; - - reg_val &= (AW88399_HMUTE_MASK | AW88399_PWDN_MASK | - AW88399_DSPBY_MASK); - reg_val |= (AW88399_HMUTE_ENABLE_VALUE | AW88399_PWDN_POWER_DOWN_VALUE | - AW88399_DSPBY_BYPASS_VALUE); - } - - if (reg_addr == AW88399_I2SCTRL3_REG) { - reg_val &= AW88399_I2STXEN_MASK; - reg_val |= AW88399_I2STXEN_DISABLE_VALUE; - } - - if (reg_addr == AW88399_SYSCTRL2_REG) { - read_vol = (reg_val & (~AW88399_VOL_MASK)) >> - AW88399_VOL_START_BIT; - aw_dev->volume_desc.init_volume = read_vol; - } - - if (reg_addr == AW88399_DBGCTRL_REG) { - if ((reg_val & (~AW88399_EF_DBMD_MASK)) == AW88399_EF_DBMD_OR_VALUE) - aw88399->check_val = AW_EF_OR_CHECK; - else - aw88399->check_val = AW_EF_AND_CHECK; - - aw88399->dither_st = reg_val & (~AW88399_DITHER_EN_MASK); - } - - if (reg_addr == AW88399_CRCCTRL_REG) - aw88399->crc_init_val = reg_val; - - ret = regmap_write(aw_dev->regmap, reg_addr, reg_val); - if (ret) - return ret; - } - - aw_dev_pwd(aw_dev, false); - usleep_range(AW88399_1000_US, AW88399_1000_US + 10); - - aw_dev_get_cur_mode_st(aw_dev); - - if (aw_dev->prof_cur != aw_dev->prof_index) - vol_desc->ctl_volume = 0; - else - aw_dev_set_volume(aw_dev, vol_desc->ctl_volume); - - return 0; -} - -static int aw_dev_reg_update(struct aw88399 *aw88399, - unsigned char *data, unsigned int len) -{ - int ret; - - if (!len || !data) { - dev_err(aw88399->aw_pa->dev, "reg data is null or len is 0"); - return -EINVAL; - } - - ret = aw_dev_update_reg_container(aw88399, data, len); - if (ret) - dev_err(aw88399->aw_pa->dev, "reg update failed"); - - return ret; -} - -static int aw88399_dev_get_prof_name(struct aw_device *aw_dev, int index, char **prof_name) -{ - struct aw_prof_info *prof_info = &aw_dev->prof_info; - struct aw_prof_desc *prof_desc; - - if ((index >= aw_dev->prof_info.count) || (index < 0)) { - dev_err(aw_dev->dev, "index[%d] overflow count[%d]", - index, aw_dev->prof_info.count); - return -EINVAL; - } - - prof_desc = &aw_dev->prof_info.prof_desc[index]; - - *prof_name = prof_info->prof_name_list[prof_desc->id]; - - return 0; -} - -static int aw88399_dev_get_prof_data(struct aw_device *aw_dev, int index, - struct aw_prof_desc **prof_desc) -{ - if ((index >= aw_dev->prof_info.count) || (index < 0)) { - dev_err(aw_dev->dev, "%s: index[%d] overflow count[%d]\n", - __func__, index, aw_dev->prof_info.count); - return -EINVAL; - } - - *prof_desc = &aw_dev->prof_info.prof_desc[index]; - - return 0; -} - -static int aw88399_dev_fw_update(struct aw88399 *aw88399, bool up_dsp_fw_en, bool force_up_en) -{ - struct aw_device *aw_dev = aw88399->aw_pa; - struct aw_prof_desc *prof_index_desc; - struct aw_sec_data_desc *sec_desc; - char *prof_name; - int ret; - - if ((aw_dev->prof_cur == aw_dev->prof_index) && - (force_up_en == AW88399_FORCE_UPDATE_OFF)) { - dev_dbg(aw_dev->dev, "scene no change, not update"); - return 0; - } - - if (aw_dev->fw_status == AW88399_DEV_FW_FAILED) { - dev_err(aw_dev->dev, "fw status[%d] error", aw_dev->fw_status); - return -EPERM; - } - - ret = aw88399_dev_get_prof_name(aw_dev, aw_dev->prof_index, &prof_name); - if (ret) - return ret; - - dev_dbg(aw_dev->dev, "start update %s", prof_name); - - ret = aw88399_dev_get_prof_data(aw_dev, aw_dev->prof_index, &prof_index_desc); - if (ret) - return ret; - - /* update reg */ - sec_desc = prof_index_desc->sec_desc; - ret = aw_dev_reg_update(aw88399, sec_desc[AW88395_DATA_TYPE_REG].data, - sec_desc[AW88395_DATA_TYPE_REG].len); - if (ret) { - dev_err(aw_dev->dev, "update reg failed"); - return ret; - } - - aw88399_dev_mute(aw_dev, true); - - if (aw_dev->dsp_cfg == AW88399_DEV_DSP_WORK) - aw_dev_dsp_enable(aw_dev, false); - - aw_dev_select_memclk(aw_dev, AW88399_DEV_MEMCLK_OSC); - - ret = aw_dev_check_sram(aw_dev); - if (ret) { - dev_err(aw_dev->dev, "check sram failed"); - goto error; - } - - if (up_dsp_fw_en) { - dev_dbg(aw_dev->dev, "fw_ver: [%x]", prof_index_desc->fw_ver); - ret = aw_dev_dsp_update_fw(aw_dev, sec_desc[AW88395_DATA_TYPE_DSP_FW].data, - sec_desc[AW88395_DATA_TYPE_DSP_FW].len); - if (ret) { - dev_err(aw_dev->dev, "update dsp fw failed"); - goto error; - } - } - - /* update dsp config */ - ret = aw_dev_dsp_update_cfg(aw_dev, sec_desc[AW88395_DATA_TYPE_DSP_CFG].data, - sec_desc[AW88395_DATA_TYPE_DSP_CFG].len); - if (ret) { - dev_err(aw_dev->dev, "update dsp cfg failed"); - goto error; - } - - aw_dev_select_memclk(aw_dev, AW88399_DEV_MEMCLK_PLL); - - aw_dev->prof_cur = aw_dev->prof_index; - - return 0; - -error: - aw_dev_select_memclk(aw_dev, AW88399_DEV_MEMCLK_PLL); - return ret; -} - -static void aw88399_start_pa(struct aw88399 *aw88399) -{ - int ret, i; - - for (i = 0; i < AW88399_START_RETRIES; i++) { - ret = aw88399_dev_start(aw88399); - if (ret) { - dev_err(aw88399->aw_pa->dev, "aw88399 device start failed. retry = %d", i); - ret = aw88399_dev_fw_update(aw88399, AW88399_DSP_FW_UPDATE_ON, true); - if (ret) { - dev_err(aw88399->aw_pa->dev, "fw update failed"); - continue; - } - } else { - dev_dbg(aw88399->aw_pa->dev, "start success\n"); - break; - } - } -} - -static void aw88399_startup_work(struct work_struct *work) -{ - struct aw88399 *aw88399 = - container_of(work, struct aw88399, start_work.work); - - mutex_lock(&aw88399->lock); - aw88399_start_pa(aw88399); - mutex_unlock(&aw88399->lock); -} - -static void aw88399_start(struct aw88399 *aw88399, bool sync_start) -{ - int ret; - - if (aw88399->aw_pa->fw_status != AW88399_DEV_FW_OK) - return; - - if (aw88399->aw_pa->status == AW88399_DEV_PW_ON) - return; - - ret = aw88399_dev_fw_update(aw88399, AW88399_DSP_FW_UPDATE_OFF, true); - if (ret) { - dev_err(aw88399->aw_pa->dev, "fw update failed."); - return; - } - - if (sync_start == AW88399_SYNC_START) - aw88399_start_pa(aw88399); - else - queue_delayed_work(system_dfl_wq, - &aw88399->start_work, - AW88399_START_WORK_DELAY_MS); -} - -static int aw_dev_check_sysint(struct aw_device *aw_dev) -{ - u16 reg_val; - - aw_dev_get_int_status(aw_dev, ®_val); - if (reg_val & AW88399_BIT_SYSINT_CHECK) { - dev_err(aw_dev->dev, "pa stop check fail:0x%04x", reg_val); - return -EINVAL; - } - - return 0; -} - -static int aw88399_stop(struct aw_device *aw_dev) -{ - struct aw_sec_data_desc *dsp_cfg = - &aw_dev->prof_info.prof_desc[aw_dev->prof_cur].sec_desc[AW88395_DATA_TYPE_DSP_CFG]; - struct aw_sec_data_desc *dsp_fw = - &aw_dev->prof_info.prof_desc[aw_dev->prof_cur].sec_desc[AW88395_DATA_TYPE_DSP_FW]; - int int_st; - - if (aw_dev->status == AW88399_DEV_PW_OFF) { - dev_dbg(aw_dev->dev, "already power off"); - return 0; - } - - aw_dev->status = AW88399_DEV_PW_OFF; - - aw88399_dev_mute(aw_dev, true); - usleep_range(AW88399_4000_US, AW88399_4000_US + 100); - - aw_dev_i2s_tx_enable(aw_dev, false); - usleep_range(AW88399_1000_US, AW88399_1000_US + 100); - - int_st = aw_dev_check_sysint(aw_dev); - - aw_dev_dsp_enable(aw_dev, false); - - aw_dev_amppd(aw_dev, true); - - if (int_st) { - aw_dev_select_memclk(aw_dev, AW88399_DEV_MEMCLK_OSC); - aw_dev_dsp_update_fw(aw_dev, dsp_fw->data, dsp_fw->len); - aw_dev_dsp_update_cfg(aw_dev, dsp_cfg->data, dsp_cfg->len); - aw_dev_select_memclk(aw_dev, AW88399_DEV_MEMCLK_PLL); - } - - aw_dev_pwd(aw_dev, true); - - return 0; -} - static struct snd_soc_dai_driver aw88399_dai[] = { { .name = "aw88399-aif", @@ -1869,86 +664,6 @@ return 0; } -static int aw88399_dev_init(struct aw88399 *aw88399, struct aw_container *aw_cfg) -{ - struct aw_device *aw_dev = aw88399->aw_pa; - int ret; - - ret = aw88395_dev_cfg_load(aw_dev, aw_cfg); - if (ret) { - dev_err(aw_dev->dev, "aw_dev acf parse failed"); - return -EINVAL; - } - aw_dev->fade_in_time = AW88399_1000_US / 10; - aw_dev->fade_out_time = AW88399_1000_US >> 1; - aw_dev->prof_cur = aw_dev->prof_info.prof_desc[0].id; - aw_dev->prof_index = aw_dev->prof_info.prof_desc[0].id; - - ret = aw88399_dev_fw_update(aw88399, AW88399_FORCE_UPDATE_ON, AW88399_DSP_FW_UPDATE_ON); - if (ret) { - dev_err(aw_dev->dev, "fw update failed ret = %d\n", ret); - return ret; - } - - aw88399_dev_mute(aw_dev, true); - - /* close tx feedback */ - aw_dev_i2s_tx_enable(aw_dev, false); - usleep_range(AW88399_1000_US, AW88399_1000_US + 100); - - /* enable amppd */ - aw_dev_amppd(aw_dev, true); - - /* close dsp */ - aw_dev_dsp_enable(aw_dev, false); - /* set power down */ - aw_dev_pwd(aw_dev, true); - - return 0; -} - -static int aw88399_request_firmware_file(struct aw88399 *aw88399) -{ - const struct firmware *cont = NULL; - int ret; - - aw88399->aw_pa->fw_status = AW88399_DEV_FW_FAILED; - - ret = request_firmware(&cont, AW88399_ACF_FILE, aw88399->aw_pa->dev); - if (ret) { - dev_err(aw88399->aw_pa->dev, "request [%s] failed!", AW88399_ACF_FILE); - return ret; - } - - dev_dbg(aw88399->aw_pa->dev, "loaded %s - size: %zu\n", - AW88399_ACF_FILE, cont ? cont->size : 0); - - aw88399->aw_cfg = devm_kzalloc(aw88399->aw_pa->dev, - struct_size(aw88399->aw_cfg, data, cont->size), GFP_KERNEL); - if (!aw88399->aw_cfg) { - release_firmware(cont); - return -ENOMEM; - } - aw88399->aw_cfg->len = (int)cont->size; - memcpy(aw88399->aw_cfg->data, cont->data, cont->size); - release_firmware(cont); - - ret = aw88395_dev_load_acf_check(aw88399->aw_pa, aw88399->aw_cfg); - if (ret) { - dev_err(aw88399->aw_pa->dev, "load [%s] failed!", AW88399_ACF_FILE); - return ret; - } - - mutex_lock(&aw88399->lock); - /* aw device init */ - ret = aw88399_dev_init(aw88399, aw88399->aw_cfg); - if (ret) - dev_err(aw88399->aw_pa->dev, "dev init failed"); - mutex_unlock(&aw88399->lock); - - return ret; -} - static const struct snd_kcontrol_new aw88399_controls[] = { SOC_SINGLE_EXT("PCM Playback Volume", AW88399_SYSCTRL2_REG, 6, AW88399_MUTE_VOL, 0, aw88399_volume_get, @@ -2040,70 +755,6 @@ .num_controls = ARRAY_SIZE(aw88399_controls), }; -static void aw88399_hw_reset(struct aw88399 *aw88399) -{ - if (aw88399->reset_gpio) { - gpiod_set_value_cansleep(aw88399->reset_gpio, 1); - usleep_range(AW88399_1000_US, AW88399_1000_US + 10); - gpiod_set_value_cansleep(aw88399->reset_gpio, 0); - usleep_range(AW88399_1000_US, AW88399_1000_US + 10); - gpiod_set_value_cansleep(aw88399->reset_gpio, 1); - usleep_range(AW88399_1000_US, AW88399_1000_US + 10); - } -} - -static void aw88399_parse_channel_dt(struct aw_device *aw_dev) -{ - struct device_node *np = aw_dev->dev->of_node; - u32 channel_value; - - of_property_read_u32(np, "awinic,audio-channel", &channel_value); - aw_dev->channel = channel_value; -} - -static int aw88399_init(struct aw88399 *aw88399, struct i2c_client *i2c, struct regmap *regmap) -{ - struct aw_device *aw_dev; - unsigned int chip_id; - int ret; - - ret = regmap_read(regmap, AW88399_ID_REG, &chip_id); - if (ret) { - dev_err(&i2c->dev, "%s read chipid error. ret = %d", __func__, ret); - return ret; - } - if (chip_id != AW88399_CHIP_ID) { - dev_err(&i2c->dev, "unsupported device"); - return -ENXIO; - } - dev_dbg(&i2c->dev, "chip id = %x\n", chip_id); - - aw_dev = devm_kzalloc(&i2c->dev, sizeof(*aw_dev), GFP_KERNEL); - if (!aw_dev) - return -ENOMEM; - aw88399->aw_pa = aw_dev; - - aw_dev->i2c = i2c; - aw_dev->dev = &i2c->dev; - aw_dev->regmap = regmap; - mutex_init(&aw_dev->dsp_lock); - - aw_dev->chip_id = chip_id; - aw_dev->acf = NULL; - aw_dev->prof_info.prof_desc = NULL; - aw_dev->prof_info.count = 0; - aw_dev->prof_info.prof_type = AW88395_DEV_NONE_TYPE_ID; - aw_dev->channel = AW88399_DEV_DEFAULT_CH; - aw_dev->fw_status = AW88399_DEV_FW_FAILED; - - aw_dev->fade_step = AW88399_VOLUME_STEP_DB; - aw_dev->volume_desc.ctl_volume = AW88399_VOL_DEFAULT_VALUE; - - aw88399_parse_channel_dt(aw_dev); - - return 0; -} - static int aw88399_i2c_probe(struct i2c_client *i2c) { struct aw88399 *aw88399; diff -ruN --no-dereference -x .git a/sound/soc/codecs/aw88399.h b/sound/soc/codecs/aw88399.h --- a/sound/soc/codecs/aw88399.h 2026-10-11 15:27:50.000000000 +0200 +++ b/sound/soc/codecs/aw88399.h 2026-10-13 14:33:30.000000000 +0200 @@ -10,512 +10,10 @@ #ifndef __AW88399_H__ #define __AW88399_H__ -/* registers list */ -#define AW88399_ID_REG (0x00) -#define AW88399_SYSST_REG (0x01) -#define AW88399_SYSINT_REG (0x02) -#define AW88399_SYSINTM_REG (0x03) -#define AW88399_SYSCTRL_REG (0x04) -#define AW88399_SYSCTRL2_REG (0x05) -#define AW88399_I2SCTRL1_REG (0x06) -#define AW88399_I2SCTRL2_REG (0x07) -#define AW88399_I2SCTRL3_REG (0x08) -#define AW88399_DACCFG1_REG (0x09) -#define AW88399_DACCFG2_REG (0x0A) -#define AW88399_DACCFG3_REG (0x0B) -#define AW88399_DACCFG4_REG (0x0C) -#define AW88399_DACCFG5_REG (0x0D) -#define AW88399_DACCFG6_REG (0x0E) -#define AW88399_DACCFG7_REG (0x0F) -#define AW88399_MPDCFG1_REG (0x10) -#define AW88399_MPDCFG2_REG (0x11) -#define AW88399_MPDCFG3_REG (0x12) -#define AW88399_MPDCFG4_REG (0x13) -#define AW88399_PWMCTRL1_REG (0x14) -#define AW88399_PWMCTRL2_REG (0x15) -#define AW88399_PWMCTRL3_REG (0x16) -#define AW88399_I2SCFG1_REG (0x17) -#define AW88399_DBGCTRL_REG (0x18) -#define AW88399_HAGCST_REG (0x20) -#define AW88399_VBAT_REG (0x21) -#define AW88399_TEMP_REG (0x22) -#define AW88399_PVDD_REG (0x23) -#define AW88399_ISNDAT_REG (0x24) -#define AW88399_VSNDAT_REG (0x25) -#define AW88399_I2SINT_REG (0x26) -#define AW88399_I2SCAPCNT_REG (0x27) -#define AW88399_ANASTA1_REG (0x28) -#define AW88399_ANASTA2_REG (0x29) -#define AW88399_ANASTA3_REG (0x2A) -#define AW88399_TESTDET_REG (0x2B) -#define AW88399_DSMCFG1_REG (0x30) -#define AW88399_DSMCFG2_REG (0x31) -#define AW88399_DSMCFG3_REG (0x32) -#define AW88399_DSMCFG4_REG (0x33) -#define AW88399_DSMCFG5_REG (0x34) -#define AW88399_DSMCFG6_REG (0x35) -#define AW88399_DSMCFG7_REG (0x36) -#define AW88399_DSMCFG8_REG (0x37) -#define AW88399_TESTIN_REG (0x38) -#define AW88399_TESTOUT_REG (0x39) -#define AW88399_MEMTEST_REG (0x3A) -#define AW88399_VSNCTRL1_REG (0x3B) -#define AW88399_ISNCTRL1_REG (0x3C) -#define AW88399_ISNCTRL2_REG (0x3D) -#define AW88399_DSPMADD_REG (0x40) -#define AW88399_DSPMDAT_REG (0x41) -#define AW88399_WDT_REG (0x42) -#define AW88399_ACR1_REG (0x43) -#define AW88399_ACR2_REG (0x44) -#define AW88399_ASR1_REG (0x45) -#define AW88399_ASR2_REG (0x46) -#define AW88399_DSPCFG_REG (0x47) -#define AW88399_ASR3_REG (0x48) -#define AW88399_ASR4_REG (0x49) -#define AW88399_DSPVCALB_REG (0x4A) -#define AW88399_CRCCTRL_REG (0x4B) -#define AW88399_DSPDBG1_REG (0x4C) -#define AW88399_DSPDBG2_REG (0x4D) -#define AW88399_DSPDBG3_REG (0x4E) -#define AW88399_PLLCTRL1_REG (0x50) -#define AW88399_PLLCTRL2_REG (0x51) -#define AW88399_PLLCTRL3_REG (0x52) -#define AW88399_CDACTRL1_REG (0x53) -#define AW88399_CDACTRL2_REG (0x54) -#define AW88399_CDACTRL3_REG (0x55) -#define AW88399_SADCCTRL1_REG (0x56) -#define AW88399_SADCCTRL2_REG (0x57) -#define AW88399_BOPCTRL1_REG (0x58) -#define AW88399_BOPCTRL2_REG (0x5A) -#define AW88399_BOPCTRL3_REG (0x5B) -#define AW88399_BOPCTRL4_REG (0x5C) -#define AW88399_BOPCTRL5_REG (0x5D) -#define AW88399_BOPCTRL6_REG (0x5E) -#define AW88399_BOPCTRL7_REG (0x5F) -#define AW88399_BSTCTRL1_REG (0x60) -#define AW88399_BSTCTRL2_REG (0x61) -#define AW88399_BSTCTRL3_REG (0x62) -#define AW88399_BSTCTRL4_REG (0x63) -#define AW88399_BSTCTRL5_REG (0x64) -#define AW88399_BSTCTRL6_REG (0x65) -#define AW88399_BSTCTRL7_REG (0x66) -#define AW88399_BSTCTRL8_REG (0x67) -#define AW88399_BSTCTRL9_REG (0x68) -#define AW88399_BSTCTRL10_REG (0x69) -#define AW88399_CPCTRL_REG (0x6A) -#define AW88399_EFWH_REG (0x6C) -#define AW88399_EFWM2_REG (0x6D) -#define AW88399_EFWM1_REG (0x6E) -#define AW88399_EFWL_REG (0x6F) -#define AW88399_TESTCTRL1_REG (0x70) -#define AW88399_TESTCTRL2_REG (0x71) -#define AW88399_EFCTRL1_REG (0x72) -#define AW88399_EFCTRL2_REG (0x73) -#define AW88399_EFRH4_REG (0x74) -#define AW88399_EFRH3_REG (0x75) -#define AW88399_EFRH2_REG (0x76) -#define AW88399_EFRH1_REG (0x77) -#define AW88399_EFRL4_REG (0x78) -#define AW88399_EFRL3_REG (0x79) -#define AW88399_EFRL2_REG (0x7A) -#define AW88399_EFRL1_REG (0x7B) -#define AW88399_TM_REG (0x7C) -#define AW88399_TM2_REG (0x7D) - -#define AW88399_REG_MAX (0x7E) -#define AW88399_MUTE_VOL (1023) - -#define AW88399_DSP_CFG_ADDR (0x9B00) -#define AW88399_DSP_REG_CFG_ADPZ_RA (0x9B68) -#define AW88399_DSP_FW_ADDR (0x8980) -#define AW88399_DSP_ROM_CHECK_ADDR (0x1F40) -#define AW88399_DSP_ROM_CHECK_DATA (0x4638) - -#define AW88399_CALI_RE_HBITS_MASK (~(0xFFFF0000)) -#define AW88399_CALI_RE_HBITS_SHIFT (16) - -#define AW88399_CALI_RE_LBITS_MASK (~(0xFFFF)) -#define AW88399_CALI_RE_LBITS_SHIFT (0) - -#define AW88399_I2STXEN_START_BIT (9) -#define AW88399_I2STXEN_BITS_LEN (1) -#define AW88399_I2STXEN_MASK \ - (~(((1<> (shift)) -#define AW88399_SHOW_RE_TO_DSP_RE(re, shift) (((re) << shift) / (1000)) -#define AW88399_CRC_CHECK_PASS_VAL (0x4) - -#define AW88399_CRC_CFG_BASE_ADDR (0xD80) -#define AW88399_CRC_FW_BASE_ADDR (0x4C0) -#define AW88399_ACF_FILE "aw88399_acf.bin" -#define AW88399_DEV_SYSST_CHECK_MAX (10) -#define AW88399_CHIP_ID 0x2183 +#include #define AW88399_I2C_NAME "aw88399" -#define AW88399_START_RETRIES (5) -#define AW88399_START_WORK_DELAY_MS (0) - #define AW88399_RATES (SNDRV_PCM_RATE_8000_48000 | \ SNDRV_PCM_RATE_96000) #define AW88399_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | \ @@ -550,80 +48,4 @@ .put = profile_set, \ } -enum { - AW_EF_AND_CHECK = 0, - AW_EF_OR_CHECK, -}; - -enum { - AW88399_DEV_VDSEL_DAC = 0, - AW88399_DEV_VDSEL_VSENSE = 32, -}; - -enum { - AW88399_DSP_CRC_NA = 0, - AW88399_DSP_CRC_OK = 1, -}; - -enum { - AW88399_DSP_FW_UPDATE_OFF = 0, - AW88399_DSP_FW_UPDATE_ON = 1, -}; - -enum { - AW88399_FORCE_UPDATE_OFF = 0, - AW88399_FORCE_UPDATE_ON = 1, -}; - -enum { - AW88399_1000_US = 1000, - AW88399_2000_US = 2000, - AW88399_3000_US = 3000, - AW88399_4000_US = 4000, -}; - -enum AW88399_DEV_STATUS { - AW88399_DEV_PW_OFF = 0, - AW88399_DEV_PW_ON, -}; - -enum AW88399_DEV_FW_STATUS { - AW88399_DEV_FW_FAILED = 0, - AW88399_DEV_FW_OK, -}; - -enum AW88399_DEV_MEMCLK { - AW88399_DEV_MEMCLK_OSC = 0, - AW88399_DEV_MEMCLK_PLL = 1, -}; - -enum AW88399_DEV_DSP_CFG { - AW88399_DEV_DSP_WORK = 0, - AW88399_DEV_DSP_BYPASS = 1, -}; - -enum { - AW88399_NOT_RCV_MODE = 0, - AW88399_RCV_MODE = 1, -}; - -enum { - AW88399_SYNC_START = 0, - AW88399_ASYNC_START, -}; - -struct aw88399 { - struct aw_device *aw_pa; - struct mutex lock; - struct gpio_desc *reset_gpio; - struct delayed_work start_work; - struct regmap *regmap; - struct aw_container *aw_cfg; - - unsigned int check_val; - unsigned int crc_init_val; - unsigned int vcalb_init_val; - unsigned int dither_st; -}; - -#endif +#endif /* __AW88399_H__ */ diff -ruN --no-dereference -x .git a/sound/soc/codecs/max98390.c b/sound/soc/codecs/max98390.c --- a/sound/soc/codecs/max98390.c 2026-10-11 15:27:50.000000000 +0200 +++ b/sound/soc/codecs/max98390.c 2026-10-13 14:33:30.000000000 +0200 @@ -978,7 +978,7 @@ .endianness = 1, }; -static const struct regmap_config max98390_regmap = { +const struct regmap_config max98390_regmap = { .reg_bits = 16, .val_bits = 8, .max_register = MAX98390_R24FF_REV_ID, @@ -988,6 +988,7 @@ .volatile_reg = max98390_volatile_reg, .cache_type = REGCACHE_RBTREE, }; +EXPORT_SYMBOL_GPL(max98390_regmap); static void max98390_slot_config(struct i2c_client *i2c, struct max98390_priv *max98390) @@ -1112,6 +1113,7 @@ #ifdef CONFIG_ACPI static const struct acpi_device_id max98390_acpi_match[] = { + { "MAX98390", 0 }, { "MX98390", 0 }, {}, }; diff -ruN --no-dereference -x .git a/sound/soc/codecs/max98390.h b/sound/soc/codecs/max98390.h --- a/sound/soc/codecs/max98390.h 2026-10-11 15:27:50.000000000 +0200 +++ b/sound/soc/codecs/max98390.h 2026-10-13 14:33:30.000000000 +0200 @@ -664,4 +664,8 @@ unsigned int ambient_temp_value; const char *dsm_param_name; }; + +/* Exported for HDA side-codec driver */ +extern const struct regmap_config max98390_regmap; + #endif diff -ruN --no-dereference -x .git a/sound/soc/intel/boards/sof_sdw.c b/sound/soc/intel/boards/sof_sdw.c --- a/sound/soc/intel/boards/sof_sdw.c 2026-10-11 15:27:50.000000000 +0200 +++ b/sound/soc/intel/boards/sof_sdw.c 2026-10-13 14:33:30.000000000 +0200 @@ -845,6 +845,15 @@ }, .driver_data = (void *)(SOC_SDW_PCH_DMIC), }, + { + /* Dell XPS 13 DX13260 (Wildcat Lake), CS42L43 + 2x CS35L63 sidecar amps */ + .callback = sof_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc"), + DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "0E53") + }, + .driver_data = (void *)(SOC_SDW_SIDECAR_AMPS), + }, {} }; diff -ruN --no-dereference -x .git a/tag-release.sh b/tag-release.sh --- a/tag-release.sh 1970-01-01 01:00:00.000000000 +0100 +++ b/tag-release.sh 2026-10-13 14:33:30.000000000 +0200 @@ -0,0 +1,271 @@ +#!/bin/bash +# tag-release.sh - Create signed CachyOS kernel tags in cachyos/linux +# Usage: ./tag-release.sh [pkgrel] +# Example: ./tag-release.sh 6.19.3 1 + +set -euo pipefail + +usage() { + cat < [pkgrel] + +Create a signed CachyOS release tag on the current branch. +Tags the HEAD commit of whatever branch is currently checked out. + +Tag scheme: cachyos-- + + Kernel variant Tag example + ───────────────────────────────────────────── + Stable initial cachyos-6.19.0-1 + Stable point release cachyos-6.19.3-1 + Stable re-spin cachyos-6.19.3-2 + LTS cachyos-6.18.9-1 + RC cachyos-6.19-rc8-1 + +Arguments: + kernel-version Kernel version (required) + Accepts X.Y.Z (e.g. 6.19.3) or X.Y-rcN (e.g. 6.19-rc8) + pkgrel Package release number (default: 1) + +Examples: + $(basename "$0") 6.19.3 + $(basename "$0") 6.19.3 2 + $(basename "$0") 6.19-rc8 + +Notes: + - Requires a GPG signing key configured via 'git config user.signingkey' + or signing enabled globally with 'git config tag.gpgSign true' + - The release tarball is signed with the same GPG key (produces .tar.gz.asc) + - Run this script from a checkout of github.com/CachyOS/linux + - Topic branches must be merged (not squashed) into the release branch. + The changelog is built from first-parent merge commits on top of the + upstream base commit ("Linux X.Y.Z"). +EOF +} + +if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then + usage + exit 0 +fi + +VERSION="${1:-}" +PKGREL="${2:-1}" + +if [[ -z "$VERSION" ]]; then + usage + exit 1 +fi + +if ! git rev-parse --is-inside-work-tree &>/dev/null; then + echo "Error: not inside a git repository" + exit 1 +fi + +if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+(\.[0-9]+|-rc[0-9]+)$ ]]; then + echo "Error: version must be X.Y.Z or X.Y-rcN (got: $VERSION)" + exit 1 +fi + +if ! [[ "$PKGREL" =~ ^[0-9]+$ ]]; then + echo "Error: pkgrel must be a positive integer (got: $PKGREL)" + exit 1 +fi + +TAG="cachyos-${VERSION}-${PKGREL}" + +# Extract major.minor ("6.19" from "6.19.3" or "6.19-rc8") +[[ "$VERSION" =~ ^([0-9]+\.[0-9]+) ]] +MAJOR_MINOR="${BASH_REMATCH[1]}" + +if git tag -l "$TAG" | grep -q .; then + echo "Error: tag '$TAG' already exists" + exit 1 +fi + +# --------------------------------------------------------------------------- +# Changelog helpers +# --------------------------------------------------------------------------- + +# Hash of the "Linux X.Y.Z" commit reachable from . +# Upstream tags the first release of a series as "Linux X.Y" (no ".0"), +# so fall back to the stripped form when "Linux X.Y.0" isn't found. +find_upstream_commit() { + local version="$1" ref="$2" hash + hash=$(git log --format="%H %s" "$ref" 2>/dev/null \ + | awk -v tgt="Linux $version" '$0 ~ "^[0-9a-f]+ "tgt"$" {print $1; exit}') + if [[ -z "$hash" && "$version" == *.0 ]]; then + hash=$(git log --format="%H %s" "$ref" 2>/dev/null \ + | awk -v tgt="Linux ${version%.0}" '$0 ~ "^[0-9a-f]+ "tgt"$" {print $1; exit}') + fi + printf '%s' "$hash" +} + +# Most recent cachyos--* tag. Works across stable rebases +# because it sorts by version name, not by ancestry. +find_prev_tag() { + local major_minor="$1" + git tag -l "cachyos-${major_minor}*" --sort=-v:refname 2>/dev/null | head -n1 +} + +# Extract the kernel version out of a "cachyos--" tag. +tag_to_version() { + sed -E 's/^cachyos-(.+)-[0-9]+$/\1/' <<< "$1" +} + +# Enumerate first-parent merge commits of topic branches on top of +# reachable from . Emits "\t" per line. +list_branch_merges() { + local upstream="$1" ref="$2" + [[ -z "$upstream" ]] && return 0 + git log --first-parent --merges --format="%H%x09%s" \ + "${upstream}..${ref}" 2>/dev/null \ + | sed -n "s/^\\([0-9a-f]\\+\\)\\t[Mm]erge branch '\\([^']\\+\\)'.*/\\1\\t\\2/p" +} + +# Commits introduced by a merge (branch side), oldest first: " ". +merge_side_commits() { + git log --format="%h %s" --no-merges --reverse "${1}^1..${1}^2" 2>/dev/null +} + +# Snapshot of branch composition at : "\t" lines. +# Used to diff two releases without relying on hashes (survives rebases). +snapshot_branches() { + local upstream="$1" ref="$2" + list_branch_merges "$upstream" "$ref" | while IFS=$'\t' read -r mhash branch; do + merge_side_commits "$mhash" | while read -r _ subject; do + printf '%s\t%s\n' "$branch" "$subject" + done + done +} + +generate_changelog() { + local version="$1" pkgrel="$2" major_minor="$3" prev_tag="$4" + + local upstream_hash + upstream_hash=$(find_upstream_commit "$version" "HEAD") + + echo "## CachyOS Linux ${version}-${pkgrel}" + echo "" + echo "Based on Linux ${version}" + [[ -n "$prev_tag" ]] && echo "Previous release: \`${prev_tag}\`" + echo "" + + # --- Diff vs. previous release ----------------------------------------- + if [[ -n "$prev_tag" ]]; then + local prev_version prev_upstream prev_snap cur_snap + prev_version=$(tag_to_version "$prev_tag") + prev_upstream=$(find_upstream_commit "$prev_version" "$prev_tag") + prev_snap=$(snapshot_branches "$prev_upstream" "$prev_tag") + cur_snap=$(snapshot_branches "$upstream_hash" "HEAD") + + local prev_branches cur_branches + prev_branches=$(cut -f1 <<< "$prev_snap" | sort -u | sed '/^$/d') + cur_branches=$(cut -f1 <<< "$cur_snap" | sort -u | sed '/^$/d') + + echo "### Changes since \`${prev_tag}\`" + echo "" + + if [[ "$prev_version" != "$version" ]]; then + echo "- Rebased to upstream: \`${prev_version}\` → \`${version}\`" + fi + + comm -13 <(echo "$prev_branches") <(echo "$cur_branches") \ + | sed '/^$/d;s/^/- Added branch: **/;s/$/**/' + comm -23 <(echo "$prev_branches") <(echo "$cur_branches") \ + | sed '/^$/d;s/^/- Removed branch: **/;s/$/**/' + + while read -r b; do + [[ -z "$b" ]] && continue + local prev_set cur_set added dropped + prev_set=$(awk -F'\t' -v B="$b" '$1==B {print $2}' <<< "$prev_snap" | sort) + cur_set=$(awk -F'\t' -v B="$b" '$1==B {print $2}' <<< "$cur_snap" | sort) + added=$(comm -13 <(echo "$prev_set") <(echo "$cur_set") | sed '/^$/d') + dropped=$(comm -23 <(echo "$prev_set") <(echo "$cur_set") | sed '/^$/d') + if [[ -n "$added" || -n "$dropped" ]]; then + echo "- Updated branch: **${b}**" + [[ -n "$added" ]] && sed 's/^/ - `+` /' <<< "$added" + [[ -n "$dropped" ]] && sed 's/^/ - `-` /' <<< "$dropped" + fi + done < <(comm -12 <(echo "$prev_branches") <(echo "$cur_branches")) + + echo "" + fi + + # --- Full branch listing ----------------------------------------------- + echo "### Applied branches" + echo "" + list_branch_merges "$upstream_hash" "HEAD" | while IFS=$'\t' read -r mhash branch; do + echo "#### ${branch}" + merge_side_commits "$mhash" | while read -r chash subject; do + echo "- \`${chash}\` ${subject}" + done + echo "" + done +} + +# --------------------------------------------------------------------------- +# Main flow +# --------------------------------------------------------------------------- + +PREV_TAG=$(find_prev_tag "$MAJOR_MINOR") + +echo "" +echo "Tag: $TAG" +echo "Branch: $(git branch --show-current)" +echo "Commit: $(git log -1 --oneline)" +[[ -n "$PREV_TAG" ]] && echo "Prev: $PREV_TAG" +echo "" + +echo "Generating changelog..." +CHANGELOG=$(generate_changelog "$VERSION" "$PKGREL" "$MAJOR_MINOR" "$PREV_TAG") + +echo "" +echo "=== Changelog Preview ===" +echo "$CHANGELOG" +echo "=========================" +echo "" + +read -rp "Create signed tag? [y/N] " confirm +[[ "$confirm" == [yY] ]] || { echo "Aborted."; exit 0; } + +git tag -s "$TAG" -m "$CHANGELOG" + +echo "" +echo "Tag '$TAG' created." + +read -rp "Push tag and create GitHub release? [y/N] " push_confirm +if [[ "$push_confirm" == [yY] ]]; then + git push origin "$TAG" + + echo "Generating source tarball..." + TARBALL="${TAG}.tar.gz" + git archive --format=tar.gz --prefix="${TAG}/" "$TAG" > "$TARBALL" + + SIGNING_KEY=$(git config --get user.signingkey || true) + echo "Signing tarball with GPG..." + if [[ -n "$SIGNING_KEY" ]]; then + gpg --local-user "$SIGNING_KEY" --armor --detach-sign -o "${TARBALL}.asc" "$TARBALL" + else + gpg --armor --detach-sign -o "${TARBALL}.asc" "$TARBALL" + fi + + NOTES_FILE=$(mktemp --suffix=.md) + echo "$CHANGELOG" > "$NOTES_FILE" + + echo "Creating GitHub release and uploading tarball + signature..." + REPO_URL=$(git remote get-url origin) + REPO_SLUG=$(echo "$REPO_URL" | sed -E 's#(https://github\.com/|git@github\.com:)##;s#\.git$##') + gh release create "$TAG" "$TARBALL" "${TARBALL}.asc" \ + --repo "$REPO_SLUG" \ + --title "CachyOS Linux ${VERSION}-${PKGREL}" \ + --notes-file "$NOTES_FILE" \ + --verify-tag + + rm -f "$TARBALL" "${TARBALL}.asc" "$NOTES_FILE" + echo "" + echo "Release created with uploaded tarball and GPG signature." + echo "Download URL: https://github.com/CachyOS/linux/releases/download/${TAG}/${TARBALL}" + echo "Signature: https://github.com/CachyOS/linux/releases/download/${TAG}/${TARBALL}.asc" +else + echo "Push with: git push origin $TAG" +fi diff -ruN --no-dereference -x .git a/tools/testing/selftests/bpf/progs/tcp_ca_kfunc.c b/tools/testing/selftests/bpf/progs/tcp_ca_kfunc.c --- a/tools/testing/selftests/bpf/progs/tcp_ca_kfunc.c 2026-10-11 15:27:50.000000000 +0200 +++ b/tools/testing/selftests/bpf/progs/tcp_ca_kfunc.c 2026-10-13 14:33:30.000000000 +0200 @@ -8,6 +8,7 @@ extern void bbr_main(struct sock *sk, u32 ack, int flag, const struct rate_sample *rs) __ksym; extern u32 bbr_sndbuf_expand(struct sock *sk) __ksym; extern u32 bbr_undo_cwnd(struct sock *sk) __ksym; +extern void bbr_cwnd_event(struct sock *sk, enum tcp_ca_event event) __ksym; extern void bbr_cwnd_event_tx_start(struct sock *sk) __ksym; extern u32 bbr_ssthresh(struct sock *sk) __ksym; extern u32 bbr_min_tso_segs(struct sock *sk) __ksym; @@ -70,6 +71,7 @@ SEC("struct_ops") void BPF_PROG(cwnd_event, struct sock *sk, enum tcp_ca_event event) { + bbr_cwnd_event(sk, event); dctcp_cwnd_event(sk, event); } diff -ruN --no-dereference -x .git a/tools/testing/selftests/mm/rmap.c b/tools/testing/selftests/mm/rmap.c --- a/tools/testing/selftests/mm/rmap.c 2026-10-11 15:27:50.000000000 +0200 +++ b/tools/testing/selftests/mm/rmap.c 2026-10-13 14:33:30.000000000 +0200 @@ -430,4 +430,85 @@ propagate_children(_metadata, data); } +static bool range_maps_the_same_pfn(int pagemap_fd, void *region, int nr_pages) +{ + int i; + int retries = 0; + unsigned long first_pfn; + +retry: + if (retries > 10) + return false; + + first_pfn = pagemap_get_pfn(pagemap_fd, region); + for (i = 0; i < nr_pages; i++) { + if (pagemap_get_pfn(pagemap_fd, region + i * getpagesize()) != first_pfn) { + /* + * Retry up to 10 times at most in case of the low chance of page + * compaction migrating the page while we check for pfn. + */ + retries++; + goto retry; + } + } + + return true; +} + +TEST_F(migrate, ksm_and_mremap) +{ + unsigned long old_pfn, new_pfn; + void *region, *mremap_region; + const int nr_pages = 16; + size_t mmap_size; + int pagemap_fd; + + /* Skip if KSM is not available */ + if (ksm_stop() < 0) + SKIP(return, "accessing \"/sys/kernel/mm/ksm/run\" failed"); + if (ksm_get_full_scans() < 0) + SKIP(return, "accessing \"/sys/kernel/mm/ksm/full_scan\" failed"); + + pagemap_fd = open("/proc/self/pagemap", O_RDONLY); + if (pagemap_fd < 0) + SKIP(return, "opening pagemap failed"); + + /* Allocate and populate twice the anon pages initially. */ + mmap_size = 2 * nr_pages * getpagesize(); + region = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, -1, 0); + ASSERT_NE(region, MAP_FAILED); + memset(region, 0x77, mmap_size); + + /* mremap the second half over the first half, to stress rmap handling */ + mmap_size /= 2; + mremap_region = mremap(region + mmap_size, mmap_size, mmap_size, + MREMAP_MAYMOVE | MREMAP_FIXED, region); + ASSERT_EQ(mremap_region, region); + + /* Merge all pages into a single KSM page. */ + madvise(region, mmap_size, MADV_MERGEABLE); + ASSERT_EQ(ksm_start(), 0); + + /* The whole range should map the same KSM page. */ + old_pfn = pagemap_get_pfn(pagemap_fd, region); + if (old_pfn == -1ul) + SKIP(return, "Obtaining PFN failed"); + ksm_start(); + ASSERT_TRUE(range_maps_the_same_pfn(pagemap_fd, region, nr_pages)); + + /* + * Migrate the KSM page; the whole range should map the new (migrated) + * KSM page. + */ + ASSERT_EQ(try_to_move_page(region), 0); + + new_pfn = pagemap_get_pfn(pagemap_fd, region); + if (new_pfn == -1ul) + SKIP(return, "Obtaining PFN failed"); + ASSERT_NE(new_pfn, old_pfn); + ASSERT_TRUE(range_maps_the_same_pfn(pagemap_fd, region, nr_pages)); +} + + TEST_HARNESS_MAIN