diff --git a/Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml b/Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml index 3f6199fc9ae6a..d8e47db677ccc 100644 --- a/Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml +++ b/Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml @@ -18,7 +18,9 @@ description: | properties: compatible: - const: qcom,fastrpc + enum: + - qcom,kaanapali-fastrpc + - qcom,fastrpc label: enum: diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c index ee652ef01534a..845015cb357fb 100644 --- a/drivers/misc/fastrpc.c +++ b/drivers/misc/fastrpc.c @@ -22,6 +22,10 @@ #include #include #include +#include +#include +#include +#include #define ADSP_DOMAIN_ID (0) #define MDSP_DOMAIN_ID (1) @@ -33,11 +37,11 @@ #define FASTRPC_ALIGN 128 #define FASTRPC_MAX_FDLIST 16 #define FASTRPC_MAX_CRCLIST 64 -#define FASTRPC_PHYS(p) ((p) & 0xffffffff) #define FASTRPC_CTX_MAX (256) #define FASTRPC_INIT_HANDLE 1 #define FASTRPC_DSP_UTILITIES_HANDLE 2 -#define FASTRPC_CTXID_MASK (0xFF0) +#define FASTRPC_MAX_STATIC_HANDLE (20) +#define FASTRPC_CTXID_MASK GENMASK(15, 8) #define INIT_FILELEN_MAX (2 * 1024 * 1024) #define INIT_FILE_NAMELEN_MAX (128) #define FASTRPC_DEVICE_NAME "fastrpc" @@ -105,8 +109,14 @@ #define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev) +/* Poll response number from remote processor for call completion */ +#define FASTRPC_POLL_RESPONSE (0xdecaf) + +/* Polling mode timeout limit */ +#define FASTRPC_POLL_MAX_TIMEOUT_US (10000) + struct fastrpc_phy_page { - u64 addr; /* physical address */ + dma_addr_t addr; /* dma address */ u64 size; /* size of contiguous region */ }; @@ -171,7 +181,7 @@ struct fastrpc_msg { u64 ctx; /* invoke caller context */ u32 handle; /* handle to invoke */ u32 sc; /* scalars structure describing the data */ - u64 addr; /* physical address */ + dma_addr_t addr; /* dma address */ u64 size; /* size of contiguous region */ }; @@ -194,7 +204,7 @@ struct fastrpc_buf { struct dma_buf *dmabuf; struct device *dev; void *virt; - u64 phys; + dma_addr_t dma_addr; u64 size; /* Lock for dma buf attachments */ struct mutex lock; @@ -202,6 +212,8 @@ struct fastrpc_buf { /* mmap support */ struct list_head node; /* list of user requested mmaps */ uintptr_t raddr; + /* Lock for buf->node */ + spinlock_t *list_lock; }; struct fastrpc_dma_buf_attachment { @@ -217,7 +229,7 @@ struct fastrpc_map { struct dma_buf *buf; struct sg_table *table; struct dma_buf_attachment *attach; - u64 phys; + dma_addr_t dma_addr; u64 size; void *va; u64 len; @@ -233,9 +245,16 @@ struct fastrpc_invoke_ctx { int pid; int client_id; u32 sc; + u64 *fdlist; u32 *crc; + /* Poll memory that DSP updates */ + u32 *poll; u64 ctxid; u64 msg_sz; + /* work done status flag */ + bool is_work_done; + /* process updates poll memory instead of glink response */ + bool is_polled; struct kref refcount; struct list_head node; /* list of ctxs */ struct completion work; @@ -257,6 +276,12 @@ struct fastrpc_session_ctx { bool valid; }; +struct fastrpc_soc_data { + u32 sid_pos; + u32 dma_addr_bits_cdsp; + u32 dma_addr_bits_default; +}; + struct fastrpc_channel_ctx { int domain_id; int sesscount; @@ -270,6 +295,8 @@ struct fastrpc_channel_ctx { struct kref refcount; /* Flag if dsp attributes are cached */ bool valid_attributes; + /* Flag if audio PD init mem was allocated */ + bool audio_init_mem; u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES]; struct fastrpc_device *secure_fdevice; struct fastrpc_device *fdevice; @@ -278,6 +305,7 @@ struct fastrpc_channel_ctx { bool secure; bool unsigned_support; u64 dma_mask; + const struct fastrpc_soc_data *soc_data; }; struct fastrpc_device { @@ -299,12 +327,32 @@ struct fastrpc_user { int client_id; int pd; bool is_secure_dev; + /* Flags poll mode state */ + bool poll_mode; /* Lock for lists */ spinlock_t lock; /* lock for allocations */ struct mutex mutex; }; +/* Extract SMMU PA from consolidated IOVA */ +static inline dma_addr_t fastrpc_ipa_to_dma_addr(struct fastrpc_channel_ctx *cctx, dma_addr_t iova) +{ + if (!cctx->soc_data->sid_pos) + return 0; + return iova & GENMASK_ULL(cctx->soc_data->sid_pos - 1, 0); +} + +/* + * Prepare the consolidated iova to send to DSP by prepending the SID + * to smmu PA at the appropriate position + */ +static inline u64 fastrpc_sid_offset(struct fastrpc_channel_ctx *cctx, + struct fastrpc_session_ctx *sctx) +{ + return (u64)sctx->sid << cctx->soc_data->sid_pos; +} + static void fastrpc_free_map(struct kref *ref) { struct fastrpc_map *map; @@ -320,11 +368,12 @@ static void fastrpc_free_map(struct kref *ref) perm.vmid = QCOM_SCM_VMID_HLOS; perm.perm = QCOM_SCM_PERM_RWX; - err = qcom_scm_assign_mem(map->phys, map->len, - &src_perms, &perm, 1); + err = qcom_scm_assign_mem(map->dma_addr, map->len, + &src_perms, &perm, 1); if (err) { - dev_err(map->fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n", - map->phys, map->len, err); + dev_err(map->fl->sctx->dev, + "Failed to assign memory dma_addr %pad size 0x%llx err %d\n", + &map->dma_addr, map->len, err); return; } } @@ -388,8 +437,11 @@ static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd, static void fastrpc_buf_free(struct fastrpc_buf *buf) { + if (!buf) + return; + dma_free_coherent(buf->dev, buf->size, buf->virt, - FASTRPC_PHYS(buf->phys)); + fastrpc_ipa_to_dma_addr(buf->fl->cctx, buf->dma_addr)); kfree(buf); } @@ -408,12 +460,13 @@ static int __fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev, buf->fl = fl; buf->virt = NULL; - buf->phys = 0; + buf->dma_addr = 0; buf->size = size; buf->dev = dev; buf->raddr = 0; + buf->list_lock = &fl->lock; - buf->virt = dma_alloc_coherent(dev, buf->size, (dma_addr_t *)&buf->phys, + buf->virt = dma_alloc_coherent(dev, buf->size, &buf->dma_addr, GFP_KERNEL); if (!buf->virt) { mutex_destroy(&buf->lock); @@ -439,7 +492,7 @@ static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev, buf = *obuf; if (fl->sctx && fl->sctx->sid) - buf->phys += ((u64)fl->sctx->sid << 32); + buf->dma_addr += fastrpc_sid_offset(fl->cctx, fl->sctx); return 0; } @@ -484,11 +537,10 @@ static void fastrpc_context_free(struct kref *ref) for (i = 0; i < ctx->nbufs; i++) fastrpc_map_put(ctx->maps[i]); - if (ctx->buf) - fastrpc_buf_free(ctx->buf); + fastrpc_buf_free(ctx->buf); spin_lock_irqsave(&cctx->lock, flags); - idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4); + idr_remove(&cctx->ctx_idr, FIELD_GET(FASTRPC_CTXID_MASK, ctx->ctxid)); spin_unlock_irqrestore(&cctx->lock, flags); kfree(ctx->maps); @@ -624,7 +676,7 @@ static struct fastrpc_invoke_ctx *fastrpc_context_alloc( spin_unlock_irqrestore(&cctx->lock, flags); goto err_idr; } - ctx->ctxid = ret << 4; + ctx->ctxid = FIELD_PREP(FASTRPC_CTXID_MASK, ret); spin_unlock_irqrestore(&cctx->lock, flags); kref_init(&ctx->refcount); @@ -684,7 +736,8 @@ static int fastrpc_dma_buf_attach(struct dma_buf *dmabuf, return -ENOMEM; ret = dma_get_sgtable(buffer->dev, &a->sgt, buffer->virt, - FASTRPC_PHYS(buffer->phys), buffer->size); + fastrpc_ipa_to_dma_addr(buffer->fl->cctx, buffer->dma_addr), + buffer->size); if (ret < 0) { dev_err(buffer->dev, "failed to get scatterlist from DMA API\n"); kfree(a); @@ -733,7 +786,7 @@ static int fastrpc_mmap(struct dma_buf *dmabuf, dma_resv_assert_held(dmabuf->resv); return dma_mmap_coherent(buf->dev, vma, buf->virt, - FASTRPC_PHYS(buf->phys), size); + fastrpc_ipa_to_dma_addr(buf->fl->cctx, buf->dma_addr), size); } static const struct dma_buf_ops fastrpc_dma_buf_ops = { @@ -746,6 +799,11 @@ static const struct dma_buf_ops fastrpc_dma_buf_ops = { .release = fastrpc_release, }; +static dma_addr_t fastrpc_compute_dma_addr(struct fastrpc_user *fl, dma_addr_t sg_dma_addr) +{ + return sg_dma_addr + fastrpc_sid_offset(fl->cctx, fl->sctx); +} + static int fastrpc_map_attach(struct fastrpc_user *fl, int fd, u64 len, u32 attr, struct fastrpc_map **ppmap) { @@ -784,12 +842,10 @@ static int fastrpc_map_attach(struct fastrpc_user *fl, int fd, } map->table = table; - if (attr & FASTRPC_ATTR_SECUREMAP) { - map->phys = sg_phys(map->table->sgl); - } else { - map->phys = sg_dma_address(map->table->sgl); - map->phys += ((u64)fl->sctx->sid << 32); - } + if (attr & FASTRPC_ATTR_SECUREMAP) + map->dma_addr = sg_phys(map->table->sgl); + else + map->dma_addr = fastrpc_compute_dma_addr(fl, sg_dma_address(map->table->sgl)); for_each_sg(map->table->sgl, sgl, map->table->nents, sgl_index) map->size += sg_dma_len(sgl); @@ -815,10 +871,11 @@ static int fastrpc_map_attach(struct fastrpc_user *fl, int fd, dst_perms[1].vmid = fl->cctx->vmperms[0].vmid; dst_perms[1].perm = QCOM_SCM_PERM_RWX; map->attr = attr; - err = qcom_scm_assign_mem(map->phys, (u64)map->len, &src_perms, dst_perms, 2); + err = qcom_scm_assign_mem(map->dma_addr, (u64)map->len, &src_perms, dst_perms, 2); if (err) { - dev_err(sess->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d\n", - map->phys, map->len, err); + dev_err(sess->dev, + "Failed to assign memory with dma_addr %pad size 0x%llx err %d\n", + &map->dma_addr, map->len, err); goto map_err; } } @@ -892,7 +949,8 @@ static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx) sizeof(struct fastrpc_invoke_buf) + sizeof(struct fastrpc_phy_page)) * ctx->nscalars + sizeof(u64) * FASTRPC_MAX_FDLIST + - sizeof(u32) * FASTRPC_MAX_CRCLIST; + sizeof(u32) * FASTRPC_MAX_CRCLIST + + sizeof(u32); return size; } @@ -987,6 +1045,8 @@ static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx) rpra = ctx->buf->virt; list = fastrpc_invoke_buf_start(rpra, ctx->nscalars); pages = fastrpc_phy_page_start(list, ctx->nscalars); + ctx->fdlist = (u64 *)(pages + ctx->nscalars); + ctx->poll = (u32 *)(ctx->fdlist + FASTRPC_MAX_FDLIST + FASTRPC_MAX_CRCLIST); args = (uintptr_t)ctx->buf->virt + metalen; rlen = pkt_size - metalen; ctx->rpra = rpra; @@ -1009,7 +1069,7 @@ static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx) struct vm_area_struct *vma = NULL; rpra[i].buf.pv = (u64) ctx->args[i].ptr; - pages[i].addr = ctx->maps[i]->phys; + pages[i].addr = ctx->maps[i]->dma_addr; mmap_read_lock(current->mm); vma = find_vma(current->mm, ctx->args[i].ptr); @@ -1036,7 +1096,7 @@ static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx) goto bail; rpra[i].buf.pv = args - ctx->olaps[oix].offset; - pages[i].addr = ctx->buf->phys - + pages[i].addr = ctx->buf->dma_addr - ctx->olaps[oix].offset + (pkt_size - rlen); pages[i].addr = pages[i].addr & PAGE_MASK; @@ -1068,7 +1128,7 @@ static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx) list[i].num = ctx->args[i].length ? 1 : 0; list[i].pgidx = i; if (ctx->maps[i]) { - pages[i].addr = ctx->maps[i]->phys; + pages[i].addr = ctx->maps[i]->dma_addr; pages[i].size = ctx->maps[i]->size; } rpra[i].dma.fd = ctx->args[i].fd; @@ -1089,18 +1149,10 @@ static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx, union fastrpc_remote_arg *rpra = ctx->rpra; struct fastrpc_user *fl = ctx->fl; struct fastrpc_map *mmap = NULL; - struct fastrpc_invoke_buf *list; - struct fastrpc_phy_page *pages; - u64 *fdlist; - int i, inbufs, outbufs, handles; + int i, inbufs; int ret = 0; inbufs = REMOTE_SCALARS_INBUFS(ctx->sc); - outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc); - handles = REMOTE_SCALARS_INHANDLES(ctx->sc) + REMOTE_SCALARS_OUTHANDLES(ctx->sc); - list = fastrpc_invoke_buf_start(rpra, ctx->nscalars); - pages = fastrpc_phy_page_start(list, ctx->nscalars); - fdlist = (uint64_t *)(pages + inbufs + outbufs + handles); for (i = inbufs; i < ctx->nbufs; ++i) { if (!ctx->maps[i]) { @@ -1122,9 +1174,9 @@ static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx, cleanup_fdlist: /* Clean up fdlist which is updated by DSP */ for (i = 0; i < FASTRPC_MAX_FDLIST; i++) { - if (!fdlist[i]) + if (!ctx->fdlist[i]) break; - if (!fastrpc_map_lookup(fl, (int)fdlist[i], &mmap)) + if (!fastrpc_map_lookup(fl, (int)ctx->fdlist[i], &mmap)) fastrpc_map_put(mmap); } @@ -1150,7 +1202,7 @@ static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx, msg->ctx = ctx->ctxid | fl->pd; msg->handle = handle; msg->sc = ctx->sc; - msg->addr = ctx->buf ? ctx->buf->phys : 0; + msg->addr = ctx->buf ? ctx->buf->dma_addr : 0; msg->size = roundup(ctx->msg_sz, PAGE_SIZE); fastrpc_context_get(ctx); @@ -1163,6 +1215,75 @@ static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx, } +static inline u32 fastrpc_poll_op(void *p) +{ + struct fastrpc_invoke_ctx *ctx = p; + + dma_rmb(); + return READ_ONCE(*ctx->poll); +} + +static int poll_for_remote_response(struct fastrpc_invoke_ctx *ctx) +{ + u32 val; + int ret; + + /* + * Poll until DSP writes FASTRPC_POLL_RESPONSE into *ctx->poll + * or until another path marks the work done. + */ + ret = read_poll_timeout_atomic(fastrpc_poll_op, val, + (val == FASTRPC_POLL_RESPONSE) || + ctx->is_work_done, 1, + FASTRPC_POLL_MAX_TIMEOUT_US, false, ctx); + + if (!ret && val == FASTRPC_POLL_RESPONSE) { + ctx->is_work_done = true; + ctx->retval = 0; + } + + if (ret == -ETIMEDOUT) + ret = -EIO; + + return ret; +} + +static inline int fastrpc_wait_for_response(struct fastrpc_invoke_ctx *ctx, + u32 kernel) +{ + int err = 0; + + if (kernel) { + if (!wait_for_completion_timeout(&ctx->work, 10 * HZ)) + err = -ETIMEDOUT; + } else { + err = wait_for_completion_interruptible(&ctx->work); + } + + return err; +} + +static int fastrpc_wait_for_completion(struct fastrpc_invoke_ctx *ctx, + u32 kernel) +{ + int err; + + do { + if (ctx->is_polled) { + err = poll_for_remote_response(ctx); + /* If polling timed out, move to normal response mode */ + if (err) + ctx->is_polled = false; + } else { + err = fastrpc_wait_for_response(ctx, kernel); + if (err) + return err; + } + } while (!ctx->is_work_done); + + return err; +} + static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel, u32 handle, u32 sc, struct fastrpc_invoke_args *args) @@ -1198,16 +1319,25 @@ static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel, if (err) goto bail; - if (kernel) { - if (!wait_for_completion_timeout(&ctx->work, 10 * HZ)) - err = -ETIMEDOUT; - } else { - err = wait_for_completion_interruptible(&ctx->work); - } + /* + * Set message context as polled if the call is for a user PD + * dynamic module and user has enabled poll mode. + */ + if (handle > FASTRPC_MAX_STATIC_HANDLE && fl->pd == USER_PD && + fl->poll_mode) + ctx->is_polled = true; + + err = fastrpc_wait_for_completion(ctx, kernel); if (err) goto bail; + if (!ctx->is_work_done) { + err = -ETIMEDOUT; + dev_dbg(fl->sctx->dev, "Invalid workdone state for handle 0x%x, sc 0x%x\n", + handle, sc); + goto bail; + } /* make sure that all memory writes by DSP are seen by CPU */ dma_rmb(); /* populate all the output buffers with results */ @@ -1268,7 +1398,6 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl, struct fastrpc_phy_page pages[1]; char *name; int err; - bool scm_done = false; struct { int client_id; u32 namelen; @@ -1295,33 +1424,10 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl, err = PTR_ERR(name); goto err; } - - if (!fl->cctx->remote_heap) { - err = fastrpc_remote_heap_alloc(fl, fl->sctx->dev, init.memlen, - &fl->cctx->remote_heap); - if (err) - goto err_name; - - /* Map if we have any heap VMIDs associated with this ADSP Static Process. */ - if (fl->cctx->vmcount) { - u64 src_perms = BIT(QCOM_SCM_VMID_HLOS); - - err = qcom_scm_assign_mem(fl->cctx->remote_heap->phys, - (u64)fl->cctx->remote_heap->size, - &src_perms, - fl->cctx->vmperms, fl->cctx->vmcount); - if (err) { - dev_err(fl->sctx->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d\n", - fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err); - goto err_map; - } - scm_done = true; - } - } - inbuf.client_id = fl->client_id; inbuf.namelen = init.namelen; inbuf.pageslen = 0; + fl->pd = USER_PD; args[0].ptr = (u64)(uintptr_t)&inbuf; @@ -1332,8 +1438,15 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl, args[1].length = inbuf.namelen; args[1].fd = -1; - pages[0].addr = fl->cctx->remote_heap->phys; - pages[0].size = fl->cctx->remote_heap->size; + if (!fl->cctx->audio_init_mem) { + pages[0].addr = fl->cctx->remote_heap->dma_addr; + pages[0].size = fl->cctx->remote_heap->size; + fl->cctx->audio_init_mem = true; + inbuf.pageslen = 1; + } else { + pages[0].addr = 0; + pages[0].size = 0; + } args[2].ptr = (u64)(uintptr_t) pages; args[2].length = sizeof(*pages); @@ -1351,26 +1464,7 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl, return 0; err_invoke: - if (fl->cctx->vmcount && scm_done) { - u64 src_perms = 0; - struct qcom_scm_vmperm dst_perms; - u32 i; - - for (i = 0; i < fl->cctx->vmcount; i++) - src_perms |= BIT(fl->cctx->vmperms[i].vmid); - - dst_perms.vmid = QCOM_SCM_VMID_HLOS; - dst_perms.perm = QCOM_SCM_PERM_RWX; - err = qcom_scm_assign_mem(fl->cctx->remote_heap->phys, - (u64)fl->cctx->remote_heap->size, - &src_perms, &dst_perms, 1); - if (err) - dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n", - fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err); - } -err_map: - fastrpc_buf_free(fl->cctx->remote_heap); -err_name: + fl->cctx->audio_init_mem = false; kfree(name); err: kfree(args); @@ -1455,7 +1549,7 @@ static int fastrpc_init_create_process(struct fastrpc_user *fl, args[2].length = inbuf.filelen; args[2].fd = init.filefd; - pages[0].addr = imem->phys; + pages[0].addr = imem->dma_addr; pages[0].size = imem->size; args[3].ptr = (u64)(uintptr_t) pages; @@ -1558,8 +1652,7 @@ static int fastrpc_device_release(struct inode *inode, struct file *file) list_del(&fl->user); spin_unlock_irqrestore(&cctx->lock, flags); - if (fl->init_mem) - fastrpc_buf_free(fl->init_mem); + fastrpc_buf_free(fl->init_mem); list_for_each_entry_safe(ctx, n, &fl->pending, node) { list_del(&ctx->node); @@ -1785,6 +1878,30 @@ static int fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability *cap, return 0; } +static int fastrpc_set_option(struct fastrpc_user *fl, char __user *argp) +{ + struct fastrpc_ioctl_set_option opt = {0}; + int i; + + if (copy_from_user(&opt, argp, sizeof(opt))) + return -EFAULT; + + for (i = 0; i < ARRAY_SIZE(opt.reserved); i++) { + if (opt.reserved[i] != 0) + return -EINVAL; + } + + if (opt.req != FASTRPC_POLL_MODE) + return -EINVAL; + + if (opt.value) + fl->poll_mode = true; + else + fl->poll_mode = false; + + return 0; +} + static int fastrpc_get_dsp_info(struct fastrpc_user *fl, char __user *argp) { struct fastrpc_ioctl_capability cap = {0}; @@ -1831,9 +1948,6 @@ static int fastrpc_req_munmap_impl(struct fastrpc_user *fl, struct fastrpc_buf * &args[0]); if (!err) { dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr); - spin_lock(&fl->lock); - list_del(&buf->node); - spin_unlock(&fl->lock); fastrpc_buf_free(buf); } else { dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr); @@ -1847,6 +1961,7 @@ static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp) struct fastrpc_buf *buf = NULL, *iter, *b; struct fastrpc_req_munmap req; struct device *dev = fl->sctx->dev; + int err; if (copy_from_user(&req, argp, sizeof(req))) return -EFAULT; @@ -1854,6 +1969,7 @@ static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp) spin_lock(&fl->lock); list_for_each_entry_safe(iter, b, &fl->mmaps, node) { if ((iter->raddr == req.vaddrout) && (iter->size == req.size)) { + list_del(&iter->node); buf = iter; break; } @@ -1866,7 +1982,14 @@ static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp) return -EINVAL; } - return fastrpc_req_munmap_impl(fl, buf); + err = fastrpc_req_munmap_impl(fl, buf); + if (err) { + spin_lock(buf->list_lock); + list_add_tail(&buf->node, &fl->mmaps); + spin_unlock(buf->list_lock); + } + + return err; } static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp) @@ -1913,7 +2036,7 @@ static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp) args[0].ptr = (u64) (uintptr_t) &req_msg; args[0].length = sizeof(req_msg); - pages.addr = buf->phys; + pages.addr = buf->dma_addr; pages.size = buf->size; args[1].ptr = (u64) (uintptr_t) &pages; @@ -1941,29 +2064,34 @@ static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp) if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR && fl->cctx->vmcount) { u64 src_perms = BIT(QCOM_SCM_VMID_HLOS); - err = qcom_scm_assign_mem(buf->phys, (u64)buf->size, - &src_perms, fl->cctx->vmperms, fl->cctx->vmcount); + err = qcom_scm_assign_mem(buf->dma_addr, (u64)buf->size, + &src_perms, fl->cctx->vmperms, + fl->cctx->vmcount); if (err) { - dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d", - buf->phys, buf->size, err); + dev_err(fl->sctx->dev, + "Failed to assign memory dma_addr %pad size 0x%llx err %d", + &buf->dma_addr, buf->size, err); goto err_assign; } } - spin_lock(&fl->lock); + spin_lock(buf->list_lock); list_add_tail(&buf->node, &fl->mmaps); - spin_unlock(&fl->lock); + spin_unlock(buf->list_lock); if (copy_to_user((void __user *)argp, &req, sizeof(req))) { err = -EFAULT; - goto err_assign; + goto err_copy; } dev_dbg(dev, "mmap\t\tpt 0x%09lx OK [len 0x%08llx]\n", buf->raddr, buf->size); return 0; - +err_copy: + spin_lock(buf->list_lock); + list_del(&buf->node); + spin_unlock(buf->list_lock); err_assign: fastrpc_req_munmap_impl(fl, buf); @@ -2059,7 +2187,7 @@ static int fastrpc_req_mem_map(struct fastrpc_user *fl, char __user *argp) args[0].ptr = (u64) (uintptr_t) &req_msg; args[0].length = sizeof(req_msg); - pages.addr = map->phys; + pages.addr = map->dma_addr; pages.size = map->len; args[1].ptr = (u64) (uintptr_t) &pages; @@ -2139,6 +2267,9 @@ static long fastrpc_device_ioctl(struct file *file, unsigned int cmd, case FASTRPC_IOCTL_MEM_UNMAP: err = fastrpc_req_mem_unmap(fl, argp); break; + case FASTRPC_IOCTL_SET_OPTION: + err = fastrpc_set_option(fl, argp); + break; case FASTRPC_IOCTL_GET_DSP_INFO: err = fastrpc_get_dsp_info(fl, argp); break; @@ -2165,6 +2296,7 @@ static int fastrpc_cb_probe(struct platform_device *pdev) int i, sessions = 0; unsigned long flags; int rc; + u32 dma_bits; cctx = dev_get_drvdata(dev->parent); if (!cctx) @@ -2178,12 +2310,16 @@ static int fastrpc_cb_probe(struct platform_device *pdev) spin_unlock_irqrestore(&cctx->lock, flags); return -ENOSPC; } + dma_bits = cctx->soc_data->dma_addr_bits_default; sess = &cctx->session[cctx->sesscount++]; sess->used = false; sess->valid = true; sess->dev = dev; dev_set_drvdata(dev, sess); + if (cctx->domain_id == CDSP_DOMAIN_ID) + dma_bits = cctx->soc_data->dma_addr_bits_cdsp; + if (of_property_read_u32(dev->of_node, "reg", &sess->sid)) dev_info(dev, "FastRPC Session ID not specified in DT\n"); @@ -2198,9 +2334,9 @@ static int fastrpc_cb_probe(struct platform_device *pdev) } } spin_unlock_irqrestore(&cctx->lock, flags); - rc = dma_set_mask(dev, DMA_BIT_MASK(32)); + rc = dma_set_mask(dev, DMA_BIT_MASK(dma_bits)); if (rc) { - dev_err(dev, "32-bit DMA enable failed\n"); + dev_err(dev, "%u-bit DMA enable failed\n", dma_bits); return rc; } @@ -2285,6 +2421,18 @@ static int fastrpc_get_domain_id(const char *domain) return -EINVAL; } +static const struct fastrpc_soc_data kaanapali_soc_data = { + .sid_pos = 56, + .dma_addr_bits_cdsp = 34, + .dma_addr_bits_default = 32, +}; + +static const struct fastrpc_soc_data default_soc_data = { + .sid_pos = 32, + .dma_addr_bits_cdsp = 32, + .dma_addr_bits_default = 32, +}; + static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) { struct device *rdev = &rpdev->dev; @@ -2293,6 +2441,9 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) const char *domain; bool secure_dsp; unsigned int vmids[FASTRPC_MAX_VMIDS]; + const struct fastrpc_soc_data *soc_data; + + soc_data = device_get_match_data(rdev); err = of_property_read_string(rdev->of_node, "label", &domain); if (err) { @@ -2329,7 +2480,7 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) } } - if (domain_id == SDSP_DOMAIN_ID) { + if (domain_id == SDSP_DOMAIN_ID || domain_id == ADSP_DOMAIN_ID) { struct resource res; u64 src_perms; @@ -2338,13 +2489,23 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) src_perms = BIT(QCOM_SCM_VMID_HLOS); qcom_scm_assign_mem(res.start, resource_size(&res), &src_perms, - data->vmperms, data->vmcount); + data->vmperms, data->vmcount); + } + if (domain_id == ADSP_DOMAIN_ID) { + data->remote_heap = + kzalloc(sizeof(*data->remote_heap), GFP_KERNEL); + if (!data->remote_heap) + return -ENOMEM; + + data->remote_heap->dma_addr = res.start; + data->remote_heap->size = resource_size(&res); } } secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain")); data->secure = secure_dsp; + data->soc_data = soc_data; switch (domain_id) { case ADSP_DOMAIN_ID: @@ -2420,10 +2581,13 @@ static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev) struct fastrpc_buf *buf, *b; struct fastrpc_user *user; unsigned long flags; + bool skip_free = false; + int err; /* No invocations past this point */ spin_lock_irqsave(&cctx->lock, flags); cctx->rpdev = NULL; + cctx->audio_init_mem = false; list_for_each_entry(user, &cctx->users, user) fastrpc_notify_users(user); spin_unlock_irqrestore(&cctx->lock, flags); @@ -2437,8 +2601,26 @@ static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev) list_for_each_entry_safe(buf, b, &cctx->invoke_interrupted_mmaps, node) list_del(&buf->node); - if (cctx->remote_heap) - fastrpc_buf_free(cctx->remote_heap); + if (cctx->remote_heap) { + if (cctx->vmcount) { + u64 src_perms = 0; + struct qcom_scm_vmperm dst_perms; + + for (u32 i = 0; i < cctx->vmcount; i++) + src_perms |= BIT(cctx->vmperms[i].vmid); + + dst_perms.vmid = QCOM_SCM_VMID_HLOS; + dst_perms.perm = QCOM_SCM_PERM_RWX; + + err = qcom_scm_assign_mem(cctx->remote_heap->dma_addr, + cctx->remote_heap->size, + &src_perms, &dst_perms, 1); + if (err) + skip_free = true; + } + if (!skip_free) + fastrpc_buf_free(cctx->remote_heap); + } of_platform_depopulate(&rpdev->dev); @@ -2457,7 +2639,7 @@ static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data, if (len < sizeof(*rsp)) return -EINVAL; - ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4); + ctxid = FIELD_GET(FASTRPC_CTXID_MASK, rsp->ctx); spin_lock_irqsave(&cctx->lock, flags); ctx = idr_find(&cctx->ctx_idr, ctxid); @@ -2470,6 +2652,7 @@ static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data, ctx->retval = rsp->retval; complete(&ctx->work); + ctx->is_work_done = true; /* * The DMA buffer associated with the context cannot be freed in @@ -2483,6 +2666,8 @@ static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data, static const struct of_device_id fastrpc_rpmsg_of_match[] = { { .compatible = "qcom,fastrpc" }, + { .compatible = "qcom,kaanapali-fastrpc", .data = &kaanapali_soc_data }, + { .compatible = "qcom,fastrpc", .data = &default_soc_data }, { }, }; MODULE_DEVICE_TABLE(of, fastrpc_rpmsg_of_match); diff --git a/include/uapi/misc/fastrpc.h b/include/uapi/misc/fastrpc.h index c6e2925f47e69..3207c42fb3180 100644 --- a/include/uapi/misc/fastrpc.h +++ b/include/uapi/misc/fastrpc.h @@ -16,6 +16,7 @@ #define FASTRPC_IOCTL_INIT_CREATE_STATIC _IOWR('R', 9, struct fastrpc_init_create_static) #define FASTRPC_IOCTL_MEM_MAP _IOWR('R', 10, struct fastrpc_mem_map) #define FASTRPC_IOCTL_MEM_UNMAP _IOWR('R', 11, struct fastrpc_mem_unmap) +#define FASTRPC_IOCTL_SET_OPTION _IOWR('R', 12, struct fastrpc_ioctl_set_option) #define FASTRPC_IOCTL_GET_DSP_INFO _IOWR('R', 13, struct fastrpc_ioctl_capability) /** @@ -66,6 +67,8 @@ enum fastrpc_proc_attr { /* Fastrpc attribute for memory protection of buffers */ #define FASTRPC_ATTR_SECUREMAP (1) +/* Set option request ID to enable poll mode */ +#define FASTRPC_POLL_MODE (1) struct fastrpc_invoke_args { __u64 ptr; @@ -133,6 +136,12 @@ struct fastrpc_mem_unmap { __s32 reserved[5]; }; +struct fastrpc_ioctl_set_option { + __u32 req; /* request id */ + __u32 value; /* value */ + __s32 reserved[6]; +}; + struct fastrpc_ioctl_capability { __u32 unused; /* deprecated, ignored by the kernel */ __u32 attribute_id;