diff --git a/kernel/patch/common/kp_spinlock.c b/kernel/patch/common/kp_spinlock.c new file mode 100644 index 0000000..6602daa --- /dev/null +++ b/kernel/patch/common/kp_spinlock.c @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include + +#include + +static inline bool kp_native_spin_irq_pair_available(void) +{ + return kfunc(_raw_spin_lock_irqsave) && kfunc(_raw_spin_unlock_irqrestore); +} + +static inline unsigned long kp_local_irq_save(void) +{ + unsigned long flags; + + asm volatile("mrs %0, daif\n\t" + "msr daifset, #2" + : "=r"(flags) + : + : "memory"); + return flags; +} + +static inline void kp_local_irq_restore(unsigned long flags) +{ + asm volatile("msr daif, %0" : : "r"(flags) : "memory"); +} + +static inline void kp_local_raw_spin_lock(raw_spinlock_t *lock) +{ + while (cmpxchg(&lock->raw_lock.counter, 0, 1) != 0) { + while (__atomic_load_n(&lock->raw_lock.counter, __ATOMIC_RELAXED)) + asm volatile("yield" ::: "memory"); + } +} + +static inline void kp_local_raw_spin_unlock(raw_spinlock_t *lock) +{ + smp_store_release(&lock->raw_lock.counter, 0); +} + +unsigned long kp_private_spin_lock(spinlock_t *lock) +{ + raw_spinlock_t *raw_lock = &lock->rlock; + unsigned long flags; + + if (likely(kp_native_spin_irq_pair_available())) + return kfunc(_raw_spin_lock_irqsave)(raw_lock); + + /* Target preempt-count layouts vary, so the local fallback masks IRQs instead. */ + flags = kp_local_irq_save(); + kp_local_raw_spin_lock(raw_lock); + return flags; +} + +void kp_private_spin_unlock(spinlock_t *lock, unsigned long flags) +{ + raw_spinlock_t *raw_lock = &lock->rlock; + + if (likely(kp_native_spin_irq_pair_available())) { + kfunc(_raw_spin_unlock_irqrestore)(raw_lock, flags); + return; + } + + kp_local_raw_spin_unlock(raw_lock); + kp_local_irq_restore(flags); +} diff --git a/kernel/patch/common/kstorage.c b/kernel/patch/common/kstorage.c index a7509fd..93043d7 100644 --- a/kernel/patch/common/kstorage.c +++ b/kernel/patch/common/kstorage.c @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include @@ -42,13 +42,13 @@ static void reclaim_callback(struct rcu_head *rcu) int try_alloc_kstroage_group() { - spin_lock(&used_max_group_lock); + unsigned long flags = kp_private_spin_lock(&used_max_group_lock); if (used_max_group + 1 >= KSTRORAGE_MAX_GROUP_NUM) { - spin_unlock(&used_max_group_lock); + kp_private_spin_unlock(&used_max_group_lock, flags); return -1; } used_max_group++; - spin_unlock(&used_max_group_lock); + kp_private_spin_unlock(&used_max_group_lock, flags); return used_max_group; } @@ -100,14 +100,14 @@ int write_kstorage(int gid, long did, void *data, int offset, int len, bool data } } - spin_lock(lock); + unsigned long flags = kp_private_spin_lock(lock); if (old) { // update hlist_replace_rcu(&old->hnode, &new->hnode); } else { // add new one hlist_add_head_rcu(&new->hnode, bucket); group_sizes[gid]++; } - spin_unlock(lock); + kp_private_spin_unlock(lock, flags); rcu_read_unlock(); @@ -242,13 +242,13 @@ int remove_kstorage(int gid, long did) spinlock_t *lock = &kstorage_glocks[gid]; struct kstorage *pos = 0; - spin_lock(lock); + unsigned long flags = kp_private_spin_lock(lock); hlist_for_each_entry_rcu(pos, bucket, hnode) { if (pos->did == did) { hlist_del_rcu(&pos->hnode); - spin_unlock(lock); + kp_private_spin_unlock(lock, flags); group_sizes[gid]--; @@ -263,7 +263,7 @@ int remove_kstorage(int gid, long did) } } - spin_unlock(lock); + kp_private_spin_unlock(lock, flags); return 0; } diff --git a/kernel/patch/common/taskob.c b/kernel/patch/common/taskob.c index 8d87921..8ddac76 100644 --- a/kernel/patch/common/taskob.c +++ b/kernel/patch/common/taskob.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include @@ -86,7 +86,7 @@ struct task_ext *kf_get_task_ext(const struct task_struct *task) static struct task_ext *task_ext_create(struct task_struct *task) { struct task_ext *ret = NULL; - spin_lock(&task_ext_lock); + unsigned long flags = kp_private_spin_lock(&task_ext_lock); for (int i = 0; i < TASK_EXT_SLOT_NUM; i++) { if (task_ext_slots[i].task == task) { ret = &task_ext_slots[i].ext; @@ -103,14 +103,14 @@ static struct task_ext *task_ext_create(struct task_struct *task) } } } - spin_unlock(&task_ext_lock); + kp_private_spin_unlock(&task_ext_lock, flags); return ret; } static void task_ext_free(struct task_struct *task) { if (likely(!atomic_read(&task_ext_active_count))) return; - spin_lock(&task_ext_lock); + unsigned long flags = kp_private_spin_lock(&task_ext_lock); for (int i = 0; i < TASK_EXT_SLOT_NUM; i++) { if (task_ext_slots[i].task == task) { task_ext_slots[i].task = NULL; @@ -118,7 +118,7 @@ static void task_ext_free(struct task_struct *task) break; } } - spin_unlock(&task_ext_lock); + kp_private_spin_unlock(&task_ext_lock, flags); } /* diff --git a/kernel/patch/include/kp_spinlock.h b/kernel/patch/include/kp_spinlock.h new file mode 100644 index 0000000..a7b163c --- /dev/null +++ b/kernel/patch/include/kp_spinlock.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef __KP_SPINLOCK_H +#define __KP_SPINLOCK_H + +#include + +/* + * These helpers are only for zero-initialized, KP-owned locks. The local + * fallback uses a 0/1 test-and-set encoding and must not be used with locks + * owned by the target kernel. + */ +unsigned long kp_private_spin_lock(spinlock_t *lock); +void kp_private_spin_unlock(spinlock_t *lock, unsigned long flags); + +#endif