spinlock: handle missing raw-spin symbols for KP-owned locks

kernel/patch/common/kp_spinlock.c

@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#include <kp_spinlock.h>
+
+#include <asm/cmpxchg.h>
+
+static inline bool kp_native_spin_irq_pair_available(void)
+{
+    return kfunc(_raw_spin_lock_irqsave) && kfunc(_raw_spin_unlock_irqrestore);
+}
+
+static inline unsigned long kp_local_irq_save(void)
+{
+    unsigned long flags;
+
+    asm volatile("mrs %0, daif\n\t"
+                 "msr daifset, #2"
+                 : "=r"(flags)
+                 :
+                 : "memory");
+    return flags;
+}
+
+static inline void kp_local_irq_restore(unsigned long flags)
+{
+    asm volatile("msr daif, %0" : : "r"(flags) : "memory");
+}
+
+static inline void kp_local_raw_spin_lock(raw_spinlock_t *lock)
+{
+    while (cmpxchg(&lock->raw_lock.counter, 0, 1) != 0) {
+        while (__atomic_load_n(&lock->raw_lock.counter, __ATOMIC_RELAXED))
+            asm volatile("yield" ::: "memory");
+    }
+}
+
+static inline void kp_local_raw_spin_unlock(raw_spinlock_t *lock)
+{
+    smp_store_release(&lock->raw_lock.counter, 0);
+}
+
+unsigned long kp_private_spin_lock(spinlock_t *lock)
+{
+    raw_spinlock_t *raw_lock = &lock->rlock;
+    unsigned long flags;
+
+    if (likely(kp_native_spin_irq_pair_available()))
+        return kfunc(_raw_spin_lock_irqsave)(raw_lock);
+
+    /* Target preempt-count layouts vary, so the local fallback masks IRQs instead. */
+    flags = kp_local_irq_save();
+    kp_local_raw_spin_lock(raw_lock);
+    return flags;
+}
+
+void kp_private_spin_unlock(spinlock_t *lock, unsigned long flags)
+{
+    raw_spinlock_t *raw_lock = &lock->rlock;
+
+    if (likely(kp_native_spin_irq_pair_available())) {
+        kfunc(_raw_spin_unlock_irqrestore)(raw_lock, flags);
+        return;
+    }
+
+    kp_local_raw_spin_unlock(raw_lock);
+    kp_local_irq_restore(flags);
+}

kernel/patch/common/kstorage.c

@@ -3,7 +3,7 @@
 #include <linux/kernel.h>
 #include <linux/rculist.h>
 #include <linux/slab.h>
-#include <linux/spinlock.h>
+#include <kp_spinlock.h>
 #include <linux/list.h>
 #include <compiler.h>
 #include <stdbool.h>
@@ -42,13 +42,13 @@ static void reclaim_callback(struct rcu_head *rcu)
 
 int try_alloc_kstroage_group()
 {
-    spin_lock(&used_max_group_lock);
+    unsigned long flags = kp_private_spin_lock(&used_max_group_lock);
     if (used_max_group + 1 >= KSTRORAGE_MAX_GROUP_NUM) {
-        spin_unlock(&used_max_group_lock);
+        kp_private_spin_unlock(&used_max_group_lock, flags);
         return -1;
     }
     used_max_group++;
-    spin_unlock(&used_max_group_lock);
+    kp_private_spin_unlock(&used_max_group_lock, flags);
     return used_max_group;
 }
 
@@ -100,14 +100,14 @@ int write_kstorage(int gid, long did, void *data, int offset, int len, bool data
         }
     }
 
-    spin_lock(lock);
+    unsigned long flags = kp_private_spin_lock(lock);
     if (old) { // update
         hlist_replace_rcu(&old->hnode, &new->hnode);
     } else { // add new one
         hlist_add_head_rcu(&new->hnode, bucket);
         group_sizes[gid]++;
     }
-    spin_unlock(lock);
+    kp_private_spin_unlock(lock, flags);
 
     rcu_read_unlock();
 
@@ -242,13 +242,13 @@ int remove_kstorage(int gid, long did)
     spinlock_t *lock = &kstorage_glocks[gid];
     struct kstorage *pos = 0;
 
-    spin_lock(lock);
+    unsigned long flags = kp_private_spin_lock(lock);
 
     hlist_for_each_entry_rcu(pos, bucket, hnode)
     {
         if (pos->did == did) {
             hlist_del_rcu(&pos->hnode);
-            spin_unlock(lock);
+            kp_private_spin_unlock(lock, flags);
 
             group_sizes[gid]--;
 
@@ -263,7 +263,7 @@ int remove_kstorage(int gid, long did)
         }
     }
 
-    spin_unlock(lock);
+    kp_private_spin_unlock(lock, flags);
 
     return 0;
 }

kernel/patch/common/taskob.c

@@ -20,7 +20,7 @@
 #include <uapi/asm-generic/errno.h>
 #include <predata.h>
 #include <symbol.h>
-#include <linux/spinlock.h>
+#include <kp_spinlock.h>
 #include <stdarg.h>
 #include <asm/atomic.h>
 #include <baselib.h>
@@ -86,7 +86,7 @@ struct task_ext *kf_get_task_ext(const struct task_struct *task)
 static struct task_ext *task_ext_create(struct task_struct *task)
 {
     struct task_ext *ret = NULL;
-    spin_lock(&task_ext_lock);
+    unsigned long flags = kp_private_spin_lock(&task_ext_lock);
     for (int i = 0; i < TASK_EXT_SLOT_NUM; i++) {
         if (task_ext_slots[i].task == task) {
             ret = &task_ext_slots[i].ext;
@@ -103,22 +103,22 @@ static struct task_ext *task_ext_create(struct task_struct *task)
             }
         }
     }
-    spin_unlock(&task_ext_lock);
+    kp_private_spin_unlock(&task_ext_lock, flags);
     return ret;
 }
 
 static void task_ext_free(struct task_struct *task)
 {
     if (likely(!atomic_read(&task_ext_active_count))) return;
-    spin_lock(&task_ext_lock);
+    unsigned long flags = kp_private_spin_lock(&task_ext_lock);
     for (int i = 0; i < TASK_EXT_SLOT_NUM; i++) {
         if (task_ext_slots[i].task == task) {
             task_ext_slots[i].task = NULL;
             atomic_dec(&task_ext_active_count);
             break;
         }
     }
-    spin_unlock(&task_ext_lock);
+    kp_private_spin_unlock(&task_ext_lock, flags);
 }
 
 /*

kernel/patch/include/kp_spinlock.h

@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef __KP_SPINLOCK_H
+#define __KP_SPINLOCK_H
+
+#include <linux/spinlock.h>
+
+/*
+ * These helpers are only for zero-initialized, KP-owned locks. The local
+ * fallback uses a 0/1 test-and-set encoding and must not be used with locks
+ * owned by the target kernel.
+ */
+unsigned long kp_private_spin_lock(spinlock_t *lock);
+void kp_private_spin_unlock(spinlock_t *lock, unsigned long flags);
+
+#endif