Lines Matching +full:data +full:- +full:enable +full:- +full:active
1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
25 #include <soc/qcom/cmd-db.h>
27 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
29 #include "rpmh-internal.h"
32 #include "trace-rpmh.h"
92 * - The main rpmh-rsc address is the base of a register space that can
94 * (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register
96 * specified in the device tree by "qcom,tcs-offset" and used to
98 * - TCS blocks come one after another. Type, count, and order are
99 * specified by the device tree as "qcom,tcs-config".
100 * - Each TCS block has some registers, then space for up to 16 commands.
106 * +---------------------------------------------------+
111 * | +-----------------------------------------------+ |
117 * | | +------------------------------------------+ | |
121 * | | +------------------------------------------+ | |
122 * | | +------------------------------------------+ | |
126 * | | +------------------------------------------+ | |
127 * | | +------------------------------------------+ | |
131 * | | +------------------------------------------+ | |
133 * | +-----------------------------------------------+ |
134 * | +-----------------------------------------------+ |
137 * | +-----------------------------------------------+ |
139 * +---------------------------------------------------+
145 return drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg; in tcs_reg_addr()
166 int cmd_id, u32 data) in write_tcs_cmd() argument
168 writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id)); in write_tcs_cmd()
172 u32 data) in write_tcs_reg() argument
174 writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id)); in write_tcs_reg()
178 u32 data) in write_tcs_reg_sync() argument
182 writel(data, tcs_reg_addr(drv, reg, tcs_id)); in write_tcs_reg_sync()
189 if (readl(tcs_reg_addr(drv, reg, tcs_id)) == data) in write_tcs_reg_sync()
193 pr_err("%s: error writing %#x to %d:%#x\n", drv->name, in write_tcs_reg_sync()
194 data, tcs_id, reg); in write_tcs_reg_sync()
198 * tcs_is_free() - Return if a TCS is totally free.
204 * Context: Must be called with the drv->lock held.
210 return !test_bit(tcs_id, drv->tcs_in_use); in tcs_is_free()
214 * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake).
228 struct tcs_group *tcs = &drv->tcs[type]; in tcs_invalidate()
231 if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS)) in tcs_invalidate()
234 for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) { in tcs_invalidate()
238 bitmap_zero(tcs->slots, MAX_TCS_SLOTS); in tcs_invalidate()
242 * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes.
256 * get_tcs_for_msg() - Get the tcs_group used to send the given message.
271 switch (msg->state) { in get_tcs_for_msg()
282 return ERR_PTR(-EINVAL); in get_tcs_for_msg()
286 * If we are making an active request on a RSC that does not have a in get_tcs_for_msg()
287 * dedicated TCS for active state use, then re-purpose a wake TCS to in get_tcs_for_msg()
288 * send active votes. This is safe because we ensure any active-only in get_tcs_for_msg()
292 tcs = &drv->tcs[type]; in get_tcs_for_msg()
293 if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs) in get_tcs_for_msg()
294 tcs = &drv->tcs[WAKE_TCS]; in get_tcs_for_msg()
300 * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS.
310 * ones we track sending (the only ones we enable interrupts for and the only
322 tcs = &drv->tcs[i]; in get_req_from_tcs()
323 if (tcs->mask & BIT(tcs_id)) in get_req_from_tcs()
324 return tcs->req[tcs_id - tcs->offset]; in get_req_from_tcs()
331 * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS
337 * transfer. That will un-trigger/disable the TCS from the last transfer
338 * then trigger/enable for this transfer.
340 * If we borrowed a wake TCS for an active-only transfer we'll also call
341 * this function with "trigger=false" to just do the un-trigger/disable
344 * Note that the AP is only in charge of triggering active-only transfers.
349 u32 enable; in __tcs_set_trigger() local
352 * HW req: Clear the DRV_CONTROL and enable TCS again in __tcs_set_trigger()
354 * and then the mode enable is cleared. in __tcs_set_trigger()
356 enable = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id); in __tcs_set_trigger()
357 enable &= ~TCS_AMC_MODE_TRIGGER; in __tcs_set_trigger()
358 write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); in __tcs_set_trigger()
359 enable &= ~TCS_AMC_MODE_ENABLE; in __tcs_set_trigger()
360 write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); in __tcs_set_trigger()
363 /* Enable the AMC mode on the TCS and then trigger the TCS */ in __tcs_set_trigger()
364 enable = TCS_AMC_MODE_ENABLE; in __tcs_set_trigger()
365 write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); in __tcs_set_trigger()
366 enable |= TCS_AMC_MODE_TRIGGER; in __tcs_set_trigger()
367 write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); in __tcs_set_trigger()
372 * enable_tcs_irq() - Enable or disable interrupts on the given TCS.
375 * @enable: If true then enable; if false then disable
377 * We only ever call this when we borrow a wake TCS for an active-only
378 * transfer. For active-only TCSes interrupts are always left enabled.
380 static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable) in enable_tcs_irq() argument
382 u32 data; in enable_tcs_irq() local
384 data = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_ENABLE); in enable_tcs_irq()
385 if (enable) in enable_tcs_irq()
386 data |= BIT(tcs_id); in enable_tcs_irq()
388 data &= ~BIT(tcs_id); in enable_tcs_irq()
389 writel_relaxed(data, drv->tcs_base + RSC_DRV_IRQ_ENABLE); in enable_tcs_irq()
393 * tcs_tx_done() - TX Done interrupt handler.
397 * Called for ACTIVE_ONLY transfers (those are the only ones we enable the
410 irq_status = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_STATUS); in tcs_tx_done()
420 for (j = 0; j < req->num_cmds; j++) { in tcs_tx_done()
423 cmd = &req->cmds[j]; in tcs_tx_done()
426 ((req->wait_for_compl || cmd->wait) && in tcs_tx_done()
428 pr_err("Incomplete request: %s: addr=%#x data=%#x", in tcs_tx_done()
429 drv->name, cmd->addr, cmd->data); in tcs_tx_done()
430 err = -EIO; in tcs_tx_done()
437 * If wake tcs was re-purposed for sending active in tcs_tx_done()
438 * votes, clear AMC trigger & enable modes and in tcs_tx_done()
441 if (!drv->tcs[ACTIVE_TCS].num_tcs) in tcs_tx_done()
447 writel_relaxed(BIT(i), drv->tcs_base + RSC_DRV_IRQ_CLEAR); in tcs_tx_done()
448 spin_lock(&drv->lock); in tcs_tx_done()
449 clear_bit(i, drv->tcs_in_use); in tcs_tx_done()
455 if (!drv->tcs[ACTIVE_TCS].num_tcs) in tcs_tx_done()
457 spin_unlock(&drv->lock); in tcs_tx_done()
458 wake_up(&drv->tcs_wait); in tcs_tx_done()
467 * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger.
471 * @msg: The message we want to send, which will contain several addr/data
474 * This is used for all types of transfers (active, sleep, and wake).
486 cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0; in __tcs_buffer_write()
491 for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) { in __tcs_buffer_write()
492 cmd = &msg->cmds[i]; in __tcs_buffer_write()
494 cmd_complete |= cmd->wait << j; in __tcs_buffer_write()
496 msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0; in __tcs_buffer_write()
499 write_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j, cmd->addr); in __tcs_buffer_write()
500 write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, j, cmd->data); in __tcs_buffer_write()
510 * check_for_req_inflight() - Look to see if conflicting cmds are in flight.
513 * @msg: The message we want to send, which will contain several addr/data
518 * one it'll return -EBUSY.
520 * Only for use for active-only transfers.
522 * Must be called with the drv->lock held since that protects tcs_in_use.
524 * Return: 0 if nothing in flight or -EBUSY if we should try again later.
525 * The caller must re-enable interrupts between tries since that's
535 int tcs_id = tcs->offset; in check_for_req_inflight()
537 for (i = 0; i < tcs->num_tcs; i++, tcs_id++) { in check_for_req_inflight()
545 for (k = 0; k < msg->num_cmds; k++) { in check_for_req_inflight()
546 if (addr == msg->cmds[k].addr) in check_for_req_inflight()
547 return -EBUSY; in check_for_req_inflight()
556 * find_free_tcs() - Find free tcs in the given tcs_group; only for active.
557 * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if
558 * we borrowed it because there are zero active-only ones).
560 * Must be called with the drv->lock held since that protects tcs_in_use.
568 for (i = 0; i < tcs->num_tcs; i++) { in find_free_tcs()
569 if (tcs_is_free(tcs->drv, tcs->offset + i)) in find_free_tcs()
570 return tcs->offset + i; in find_free_tcs()
573 return -EBUSY; in find_free_tcs()
577 * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active.
580 * @msg: The data to be sent.
585 * Context: Must be called with the drv->lock held since that protects
588 * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight
598 * when one is already in-flight or being processed. in claim_tcs_for_req()
608 * rpmh_rsc_send_data() - Write / trigger active-only message.
610 * @msg: The data to be sent.
613 * - This is only used for "ACTIVE_ONLY" since the limitations of this
615 * - To do the transfer, we will grab a whole TCS for ourselves--we don't
618 * - This function will not wait for the commands to be finished, only for
619 * data to be programmed into the RPMh. See rpmh_tx_done() which will
621 * - This function must be called with interrupts enabled. If the hardware
625 * active CPU this can never happen if interrupts are disabled.
627 * Return: 0 on success, -EINVAL on error.
639 spin_lock_irqsave(&drv->lock, flags); in rpmh_rsc_send_data()
642 wait_event_lock_irq(drv->tcs_wait, in rpmh_rsc_send_data()
644 drv->lock); in rpmh_rsc_send_data()
646 tcs->req[tcs_id - tcs->offset] = msg; in rpmh_rsc_send_data()
647 set_bit(tcs_id, drv->tcs_in_use); in rpmh_rsc_send_data()
648 if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) { in rpmh_rsc_send_data()
651 * repurposed TCS to avoid triggering them. tcs->slots will be in rpmh_rsc_send_data()
658 spin_unlock_irqrestore(&drv->lock, flags); in rpmh_rsc_send_data()
662 * - We marked "tcs_in_use" under lock. in rpmh_rsc_send_data()
663 * - Once "tcs_in_use" has been marked nobody else could be writing in rpmh_rsc_send_data()
665 * - The interrupt can't go off until we trigger w/ the last line in rpmh_rsc_send_data()
675 * find_slots() - Find a place to write the given message.
685 * tcs->slots for.
687 * Return: -ENOMEM if there was no room, else 0.
697 slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS, in find_slots()
698 i, msg->num_cmds, 0); in find_slots()
699 if (slot >= tcs->num_tcs * tcs->ncpt) in find_slots()
700 return -ENOMEM; in find_slots()
701 i += tcs->ncpt; in find_slots()
702 } while (slot + msg->num_cmds - 1 >= i); in find_slots()
704 bitmap_set(tcs->slots, slot, msg->num_cmds); in find_slots()
706 offset = slot / tcs->ncpt; in find_slots()
707 *tcs_id = offset + tcs->offset; in find_slots()
708 *cmd_id = slot % tcs->ncpt; in find_slots()
714 * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger.
716 * @msg: The data to be written to the controller.
718 * This should only be called for for sleep/wake state, never active-only
724 * Return: 0 if no error; else -error.
745 * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy.
748 * Checks if any of the AMCs are busy in handling ACTIVE sets.
753 * Context: Must be called with the drv->lock held.
756 * * False - AMCs are idle
757 * * True - AMCs are busy
762 struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS]; in rpmh_rsc_ctrlr_is_busy()
765 * If we made an active request on a RSC that does not have a in rpmh_rsc_ctrlr_is_busy()
766 * dedicated TCS for active state use, then re-purposed wake TCSes in rpmh_rsc_ctrlr_is_busy()
768 * active requests in this case. in rpmh_rsc_ctrlr_is_busy()
770 if (!tcs->num_tcs) in rpmh_rsc_ctrlr_is_busy()
771 tcs = &drv->tcs[WAKE_TCS]; in rpmh_rsc_ctrlr_is_busy()
773 for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) { in rpmh_rsc_ctrlr_is_busy()
782 * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy.
788 * about when CPUs go down. When all CPUs go down we know no more active
807 cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm); in rpmh_rsc_cpu_pm_callback()
814 * AND that CPU was not idle AND that CPU was the last non-idle in rpmh_rsc_cpu_pm_callback()
823 atomic_dec(&drv->cpus_in_pm); in rpmh_rsc_cpu_pm_callback()
830 * It's likely we're on the last CPU. Grab the drv->lock and write in rpmh_rsc_cpu_pm_callback()
834 * and has grabbed the lock or started an active transfer then we'll in rpmh_rsc_cpu_pm_callback()
836 * flushing it will be blocked from starting an active transfer until in rpmh_rsc_cpu_pm_callback()
837 * we're done flushing. If another CPU starts an active transfer after in rpmh_rsc_cpu_pm_callback()
841 if (spin_trylock(&drv->lock)) { in rpmh_rsc_cpu_pm_callback()
842 if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client)) in rpmh_rsc_cpu_pm_callback()
844 spin_unlock(&drv->lock); in rpmh_rsc_cpu_pm_callback()
851 /* Double-check if we're here because someone else is up */ in rpmh_rsc_cpu_pm_callback()
856 atomic_dec(&drv->cpus_in_pm); in rpmh_rsc_cpu_pm_callback()
869 struct device_node *dn = pdev->dev.of_node; in rpmh_probe_tcs_config()
874 ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset); in rpmh_probe_tcs_config()
877 drv->tcs_base = base + offset; in rpmh_probe_tcs_config()
882 max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id); in rpmh_probe_tcs_config()
883 max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id); in rpmh_probe_tcs_config()
888 n = of_property_count_u32_elems(dn, "qcom,tcs-config"); in rpmh_probe_tcs_config()
890 return -EINVAL; in rpmh_probe_tcs_config()
893 ret = of_property_read_u32_index(dn, "qcom,tcs-config", in rpmh_probe_tcs_config()
898 return -EINVAL; in rpmh_probe_tcs_config()
900 ret = of_property_read_u32_index(dn, "qcom,tcs-config", in rpmh_probe_tcs_config()
905 return -EINVAL; in rpmh_probe_tcs_config()
909 tcs = &drv->tcs[tcs_cfg[i].type]; in rpmh_probe_tcs_config()
910 if (tcs->drv) in rpmh_probe_tcs_config()
911 return -EINVAL; in rpmh_probe_tcs_config()
912 tcs->drv = drv; in rpmh_probe_tcs_config()
913 tcs->type = tcs_cfg[i].type; in rpmh_probe_tcs_config()
914 tcs->num_tcs = tcs_cfg[i].n; in rpmh_probe_tcs_config()
915 tcs->ncpt = ncpt; in rpmh_probe_tcs_config()
917 if (!tcs->num_tcs || tcs->type == CONTROL_TCS) in rpmh_probe_tcs_config()
920 if (st + tcs->num_tcs > max_tcs || in rpmh_probe_tcs_config()
921 st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask)) in rpmh_probe_tcs_config()
922 return -EINVAL; in rpmh_probe_tcs_config()
924 tcs->mask = ((1 << tcs->num_tcs) - 1) << st; in rpmh_probe_tcs_config()
925 tcs->offset = st; in rpmh_probe_tcs_config()
926 st += tcs->num_tcs; in rpmh_probe_tcs_config()
929 drv->num_tcs = st; in rpmh_probe_tcs_config()
936 struct device_node *dn = pdev->dev.of_node; in rpmh_rsc_probe()
945 * Even though RPMh doesn't directly use cmd-db, all of its children in rpmh_rsc_probe()
950 if (ret != -EPROBE_DEFER) in rpmh_rsc_probe()
951 dev_err(&pdev->dev, "Command DB not available (%d)\n", in rpmh_rsc_probe()
956 drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL); in rpmh_rsc_probe()
958 return -ENOMEM; in rpmh_rsc_probe()
960 ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id); in rpmh_rsc_probe()
964 drv->name = of_get_property(dn, "label", NULL); in rpmh_rsc_probe()
965 if (!drv->name) in rpmh_rsc_probe()
966 drv->name = dev_name(&pdev->dev); in rpmh_rsc_probe()
968 snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id); in rpmh_rsc_probe()
970 base = devm_ioremap_resource(&pdev->dev, res); in rpmh_rsc_probe()
978 spin_lock_init(&drv->lock); in rpmh_rsc_probe()
979 init_waitqueue_head(&drv->tcs_wait); in rpmh_rsc_probe()
980 bitmap_zero(drv->tcs_in_use, MAX_TCS_NR); in rpmh_rsc_probe()
982 irq = platform_get_irq(pdev, drv->id); in rpmh_rsc_probe()
986 ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done, in rpmh_rsc_probe()
988 drv->name, drv); in rpmh_rsc_probe()
1001 drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback; in rpmh_rsc_probe()
1002 cpu_pm_register_notifier(&drv->rsc_pm); in rpmh_rsc_probe()
1005 /* Enable the active TCS to send requests immediately */ in rpmh_rsc_probe()
1006 writel_relaxed(drv->tcs[ACTIVE_TCS].mask, in rpmh_rsc_probe()
1007 drv->tcs_base + RSC_DRV_IRQ_ENABLE); in rpmh_rsc_probe()
1009 spin_lock_init(&drv->client.cache_lock); in rpmh_rsc_probe()
1010 INIT_LIST_HEAD(&drv->client.cache); in rpmh_rsc_probe()
1011 INIT_LIST_HEAD(&drv->client.batch_cache); in rpmh_rsc_probe()
1013 dev_set_drvdata(&pdev->dev, drv); in rpmh_rsc_probe()
1015 return devm_of_platform_populate(&pdev->dev); in rpmh_rsc_probe()
1019 { .compatible = "qcom,rpmh-rsc", },