1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Universal Flash Storage Host controller driver Core
4 * Copyright (C) 2011-2013 Samsung India Software Operations
5 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
6 *
7 * Authors:
8 * Santosh Yaraganavi <santosh.sy@samsung.com>
9 * Vinayak Holikatti <h.vinayak@samsung.com>
10 */
11
12 #include <linux/async.h>
13 #include <linux/devfreq.h>
14 #include <linux/nls.h>
15 #include <linux/of.h>
16 #include <linux/bitfield.h>
17 #include <linux/blk-pm.h>
18 #include <linux/blkdev.h>
19 #include "ufshcd.h"
20 #include "ufshcd-add-info.h"
21 #include "ufs_quirks.h"
22 #include "unipro.h"
23 #include "ufs-sysfs.h"
24 #include "ufs-debugfs.h"
25 #include "ufs_bsg.h"
26 #include "ufshcd-crypto.h"
27 #include "ufshpb.h"
28 #include <asm/unaligned.h>
29 #include <linux/blkdev.h>
30
31 #define CREATE_TRACE_POINTS
32 #include <trace/events/ufs.h>
33
34 #undef CREATE_TRACE_POINTS
35 #include <trace/hooks/ufshcd.h>
36
37 #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
38 UTP_TASK_REQ_COMPL |\
39 UFSHCD_ERROR_MASK)
40 /* UIC command timeout, unit: ms */
41 #define UIC_CMD_TIMEOUT 5000
42
43 /* NOP OUT retries waiting for NOP IN response */
44 #define NOP_OUT_RETRIES 10
45 /* Timeout after 50 msecs if NOP OUT hangs without response */
46 #define NOP_OUT_TIMEOUT 50 /* msecs */
47
48 /* Query request retries */
49 #define QUERY_REQ_RETRIES 3
50 /* Query request timeout */
51 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
52
53 /* Task management command timeout */
54 #define TM_CMD_TIMEOUT 100 /* msecs */
55
56 /* maximum number of retries for a general UIC command */
57 #define UFS_UIC_COMMAND_RETRIES 3
58
59 /* maximum number of link-startup retries */
60 #define DME_LINKSTARTUP_RETRIES 3
61
62 /* Maximum retries for Hibern8 enter */
63 #define UIC_HIBERN8_ENTER_RETRIES 3
64
65 /* maximum number of reset retries before giving up */
66 #define MAX_HOST_RESET_RETRIES 5
67
68 /* Expose the flag value from utp_upiu_query.value */
69 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
70
71 /* Interrupt aggregation default timeout, unit: 40us */
72 #define INT_AGGR_DEF_TO 0x02
73
74 /* default delay of autosuspend: 2000 ms */
75 #define RPM_AUTOSUSPEND_DELAY_MS 2000
76
77 /* Default delay of RPM device flush delayed work */
78 #define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000
79
80 /* Default value of wait time before gating device ref clock */
81 #define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */
82
83 /* Polling time to wait for fDeviceInit */
84 #define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */
85
86 #define ufshcd_toggle_vreg(_dev, _vreg, _on) \
87 ({ \
88 int _ret; \
89 if (_on) \
90 _ret = ufshcd_enable_vreg(_dev, _vreg); \
91 else \
92 _ret = ufshcd_disable_vreg(_dev, _vreg); \
93 _ret; \
94 })
95
96 #define ufshcd_hex_dump(prefix_str, buf, len) do { \
97 size_t __len = (len); \
98 print_hex_dump(KERN_ERR, prefix_str, \
99 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
100 16, 4, buf, __len, false); \
101 } while (0)
102
ufshcd_dump_regs(struct ufs_hba * hba,size_t offset,size_t len,const char * prefix)103 int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
104 const char *prefix)
105 {
106 u32 *regs;
107 size_t pos;
108
109 if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
110 return -EINVAL;
111
112 regs = kzalloc(len, GFP_ATOMIC);
113 if (!regs)
114 return -ENOMEM;
115
116 for (pos = 0; pos < len; pos += 4) {
117 if (offset == 0 &&
118 pos >= REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER &&
119 pos <= REG_UIC_ERROR_CODE_DME)
120 continue;
121 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
122 }
123
124 ufshcd_hex_dump(prefix, regs, len);
125 kfree(regs);
126
127 return 0;
128 }
129 EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
130
131 enum {
132 UFSHCD_MAX_CHANNEL = 0,
133 UFSHCD_MAX_ID = 1,
134 UFSHCD_NUM_RESERVED = 1,
135 UFSHCD_CMD_PER_LUN = 32 - UFSHCD_NUM_RESERVED,
136 UFSHCD_CAN_QUEUE = 32 - UFSHCD_NUM_RESERVED,
137 };
138
139 /* UFSHCD states */
140 enum {
141 UFSHCD_STATE_RESET,
142 UFSHCD_STATE_ERROR,
143 UFSHCD_STATE_OPERATIONAL,
144 UFSHCD_STATE_EH_SCHEDULED_FATAL,
145 UFSHCD_STATE_EH_SCHEDULED_NON_FATAL,
146 };
147
148 /* UFSHCD error handling flags */
149 enum {
150 UFSHCD_EH_IN_PROGRESS = (1 << 0),
151 };
152
153 /* UFSHCD UIC layer error flags */
154 enum {
155 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
156 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
157 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
158 UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
159 UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
160 UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
161 UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */
162 };
163
164 #define ufshcd_set_eh_in_progress(h) \
165 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
166 #define ufshcd_eh_in_progress(h) \
167 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
168 #define ufshcd_clear_eh_in_progress(h) \
169 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
170
171 struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
172 {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
173 {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
174 {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
175 {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
176 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
177 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
178 };
179
180 static inline enum ufs_dev_pwr_mode
ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)181 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
182 {
183 return ufs_pm_lvl_states[lvl].dev_state;
184 }
185
186 static inline enum uic_link_state
ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)187 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
188 {
189 return ufs_pm_lvl_states[lvl].link_state;
190 }
191
192 static inline enum ufs_pm_level
ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,enum uic_link_state link_state)193 ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
194 enum uic_link_state link_state)
195 {
196 enum ufs_pm_level lvl;
197
198 for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
199 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
200 (ufs_pm_lvl_states[lvl].link_state == link_state))
201 return lvl;
202 }
203
204 /* if no match found, return the level 0 */
205 return UFS_PM_LVL_0;
206 }
207
208 static struct ufs_dev_fix ufs_fixups[] = {
209 /* UFS cards deviations table */
210 UFS_FIX(UFS_VENDOR_MICRON, UFS_ANY_MODEL,
211 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
212 UFS_DEVICE_QUIRK_SWAP_L2P_ENTRY_FOR_HPB_READ),
213 UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
214 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
215 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE |
216 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
217 UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
218 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
219 UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/,
220 UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
221 UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
222 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
223 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
224 UFS_DEVICE_QUIRK_PA_TACTIVATE),
225 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
226 UFS_DEVICE_QUIRK_PA_TACTIVATE),
227 END_FIX
228 };
229
230 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
231 static void ufshcd_async_scan(void *data, async_cookie_t cookie);
232 static int ufshcd_reset_and_restore(struct ufs_hba *hba);
233 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
234 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
235 static void ufshcd_hba_exit(struct ufs_hba *hba);
236 static int ufshcd_probe_hba(struct ufs_hba *hba, bool async);
237 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
238 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
239 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
240 static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
241 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
242 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
243 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
244 static irqreturn_t ufshcd_intr(int irq, void *__hba);
245 static int ufshcd_change_power_mode(struct ufs_hba *hba,
246 struct ufs_pa_layer_attr *pwr_mode);
247 static void ufshcd_schedule_eh_work(struct ufs_hba *hba);
248 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on);
249 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on);
250 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
251 struct ufs_vreg *vreg);
252 static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
253 static int ufshcd_wb_buf_flush_enable(struct ufs_hba *hba);
254 static int ufshcd_wb_buf_flush_disable(struct ufs_hba *hba);
255 static int ufshcd_wb_ctrl(struct ufs_hba *hba, bool enable);
256 static int ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set);
257 static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable);
258 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba);
259 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba);
260
ufshcd_valid_tag(struct ufs_hba * hba,int tag)261 static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
262 {
263 return tag >= 0 && tag < hba->nutrs;
264 }
265
ufshcd_enable_irq(struct ufs_hba * hba)266 static inline void ufshcd_enable_irq(struct ufs_hba *hba)
267 {
268 if (!hba->is_irq_enabled) {
269 enable_irq(hba->irq);
270 hba->is_irq_enabled = true;
271 }
272 }
273
ufshcd_disable_irq(struct ufs_hba * hba)274 static inline void ufshcd_disable_irq(struct ufs_hba *hba)
275 {
276 if (hba->is_irq_enabled) {
277 disable_irq(hba->irq);
278 hba->is_irq_enabled = false;
279 }
280 }
281
ufshcd_wb_config(struct ufs_hba * hba)282 static inline void ufshcd_wb_config(struct ufs_hba *hba)
283 {
284 int ret;
285
286 if (!ufshcd_is_wb_allowed(hba))
287 return;
288
289 ret = ufshcd_wb_ctrl(hba, true);
290 if (ret)
291 dev_err(hba->dev, "%s: Enable WB failed: %d\n", __func__, ret);
292 else
293 dev_info(hba->dev, "%s: Write Booster Configured\n", __func__);
294 ret = ufshcd_wb_toggle_flush_during_h8(hba, true);
295 if (ret)
296 dev_err(hba->dev, "%s: En WB flush during H8: failed: %d\n",
297 __func__, ret);
298 if (!(hba->quirks & UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL))
299 ufshcd_wb_toggle_flush(hba, true);
300 }
301
ufshcd_scsi_unblock_requests(struct ufs_hba * hba)302 static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
303 {
304 if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
305 scsi_unblock_requests(hba->host);
306 }
307
ufshcd_scsi_block_requests(struct ufs_hba * hba)308 static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
309 {
310 if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
311 scsi_block_requests(hba->host);
312 }
313
ufshcd_add_cmd_upiu_trace(struct ufs_hba * hba,unsigned int tag,const char * str)314 static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
315 const char *str)
316 {
317 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
318
319 trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->sc.cdb);
320 }
321
ufshcd_add_query_upiu_trace(struct ufs_hba * hba,unsigned int tag,const char * str)322 static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, unsigned int tag,
323 const char *str)
324 {
325 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
326
327 trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->qr);
328 }
329
ufshcd_add_tm_upiu_trace(struct ufs_hba * hba,unsigned int tag,const char * str)330 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
331 const char *str)
332 {
333 struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[tag];
334
335 trace_android_vh_ufs_send_tm_command(hba, tag, str);
336 trace_ufshcd_upiu(dev_name(hba->dev), str, &descp->req_header,
337 &descp->input_param1);
338 }
339
ufshcd_add_uic_command_trace(struct ufs_hba * hba,struct uic_command * ucmd,const char * str)340 static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
341 struct uic_command *ucmd,
342 const char *str)
343 {
344 u32 cmd;
345
346 trace_android_vh_ufs_send_uic_command(hba, ucmd, str);
347
348 if (!trace_ufshcd_uic_command_enabled())
349 return;
350
351 if (!strcmp(str, "send"))
352 cmd = ucmd->command;
353 else
354 cmd = ufshcd_readl(hba, REG_UIC_COMMAND);
355
356 trace_ufshcd_uic_command(dev_name(hba->dev), str, cmd,
357 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1),
358 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2),
359 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3));
360 }
361
ufshcd_add_command_trace(struct ufs_hba * hba,unsigned int tag,const char * str)362 static void ufshcd_add_command_trace(struct ufs_hba *hba,
363 unsigned int tag, const char *str)
364 {
365 sector_t lba = -1;
366 u8 opcode = 0, group_id = 0;
367 u32 intr, doorbell;
368 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
369 struct scsi_cmnd *cmd = lrbp->cmd;
370 int transfer_len = -1;
371
372 if (!trace_ufshcd_command_enabled()) {
373 /* trace UPIU W/O tracing command */
374 if (cmd)
375 ufshcd_add_cmd_upiu_trace(hba, tag, str);
376 return;
377 }
378
379 if (cmd) { /* data phase exists */
380 /* trace UPIU also */
381 ufshcd_add_cmd_upiu_trace(hba, tag, str);
382 opcode = cmd->cmnd[0];
383 if ((opcode == READ_10) || (opcode == WRITE_10)) {
384 /*
385 * Currently we only fully trace read(10) and write(10)
386 * commands
387 */
388 if (cmd->request && cmd->request->bio)
389 lba = cmd->request->bio->bi_iter.bi_sector;
390 transfer_len = be32_to_cpu(
391 lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
392 if (opcode == WRITE_10)
393 group_id = lrbp->cmd->cmnd[6];
394 } else if (opcode == UNMAP) {
395 if (cmd->request) {
396 lba = scsi_get_lba(cmd);
397 transfer_len = blk_rq_bytes(cmd->request);
398 }
399 }
400 }
401
402 intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
403 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
404 trace_ufshcd_command(dev_name(hba->dev), str, tag,
405 doorbell, transfer_len, intr, lba, opcode, group_id);
406 }
407
ufshcd_print_clk_freqs(struct ufs_hba * hba)408 static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
409 {
410 struct ufs_clk_info *clki;
411 struct list_head *head = &hba->clk_list_head;
412
413 if (list_empty(head))
414 return;
415
416 list_for_each_entry(clki, head, list) {
417 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
418 clki->max_freq)
419 dev_err(hba->dev, "clk: %s, rate: %u\n",
420 clki->name, clki->curr_freq);
421 }
422 }
423
ufshcd_print_evt(struct ufs_hba * hba,u32 id,char * err_name)424 static void ufshcd_print_evt(struct ufs_hba *hba, u32 id,
425 char *err_name)
426 {
427 int i;
428 bool found = false;
429 struct ufs_event_hist *e;
430
431 if (id >= UFS_EVT_CNT)
432 return;
433
434 e = &hba->ufs_stats.event[id];
435
436 for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) {
437 int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH;
438
439 if (e->tstamp[p] == 0)
440 continue;
441 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
442 e->val[p], ktime_to_us(e->tstamp[p]));
443 found = true;
444 }
445
446 if (!found)
447 dev_err(hba->dev, "No record of %s\n", err_name);
448 }
449
ufshcd_print_evt_hist(struct ufs_hba * hba)450 static void ufshcd_print_evt_hist(struct ufs_hba *hba)
451 {
452 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
453
454 ufshcd_print_evt(hba, UFS_EVT_PA_ERR, "pa_err");
455 ufshcd_print_evt(hba, UFS_EVT_DL_ERR, "dl_err");
456 ufshcd_print_evt(hba, UFS_EVT_NL_ERR, "nl_err");
457 ufshcd_print_evt(hba, UFS_EVT_TL_ERR, "tl_err");
458 ufshcd_print_evt(hba, UFS_EVT_DME_ERR, "dme_err");
459 ufshcd_print_evt(hba, UFS_EVT_AUTO_HIBERN8_ERR,
460 "auto_hibern8_err");
461 ufshcd_print_evt(hba, UFS_EVT_FATAL_ERR, "fatal_err");
462 ufshcd_print_evt(hba, UFS_EVT_LINK_STARTUP_FAIL,
463 "link_startup_fail");
464 ufshcd_print_evt(hba, UFS_EVT_RESUME_ERR, "resume_fail");
465 ufshcd_print_evt(hba, UFS_EVT_SUSPEND_ERR,
466 "suspend_fail");
467 ufshcd_print_evt(hba, UFS_EVT_DEV_RESET, "dev_reset");
468 ufshcd_print_evt(hba, UFS_EVT_HOST_RESET, "host_reset");
469 ufshcd_print_evt(hba, UFS_EVT_ABORT, "task_abort");
470
471 ufshcd_vops_dbg_register_dump(hba);
472 }
473
474 static
ufshcd_print_trs(struct ufs_hba * hba,unsigned long bitmap,bool pr_prdt)475 void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
476 {
477 struct ufshcd_lrb *lrbp;
478 int prdt_length;
479 int tag;
480
481 for_each_set_bit(tag, &bitmap, hba->nutrs) {
482 lrbp = &hba->lrb[tag];
483
484 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
485 tag, ktime_to_us(lrbp->issue_time_stamp));
486 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
487 tag, ktime_to_us(lrbp->compl_time_stamp));
488 dev_err(hba->dev,
489 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
490 tag, (u64)lrbp->utrd_dma_addr);
491
492 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
493 sizeof(struct utp_transfer_req_desc));
494 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
495 (u64)lrbp->ucd_req_dma_addr);
496 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
497 sizeof(struct utp_upiu_req));
498 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
499 (u64)lrbp->ucd_rsp_dma_addr);
500 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
501 sizeof(struct utp_upiu_rsp));
502
503 prdt_length = le16_to_cpu(
504 lrbp->utr_descriptor_ptr->prd_table_length);
505 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
506 prdt_length /= hba->sg_entry_size;
507
508 dev_err(hba->dev,
509 "UPIU[%d] - PRDT - %d entries phys@0x%llx\n",
510 tag, prdt_length,
511 (u64)lrbp->ucd_prdt_dma_addr);
512
513 if (pr_prdt)
514 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
515 hba->sg_entry_size * prdt_length);
516 }
517 }
518
ufshcd_print_tmrs(struct ufs_hba * hba,unsigned long bitmap)519 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
520 {
521 int tag;
522
523 for_each_set_bit(tag, &bitmap, hba->nutmrs) {
524 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
525
526 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
527 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
528 }
529 }
530
ufshcd_print_host_state(struct ufs_hba * hba)531 static void ufshcd_print_host_state(struct ufs_hba *hba)
532 {
533 struct scsi_device *sdev_ufs = hba->sdev_ufs_device;
534
535 dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
536 dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
537 hba->outstanding_reqs, hba->outstanding_tasks);
538 dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
539 hba->saved_err, hba->saved_uic_err);
540 dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
541 hba->curr_dev_pwr_mode, hba->uic_link_state);
542 dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
543 hba->pm_op_in_progress, hba->is_sys_suspended);
544 dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
545 hba->auto_bkops_enabled, hba->host->host_self_blocked);
546 dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
547 dev_err(hba->dev,
548 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n",
549 ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
550 hba->ufs_stats.hibern8_exit_cnt);
551 dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n",
552 ktime_to_us(hba->ufs_stats.last_intr_ts),
553 hba->ufs_stats.last_intr_status);
554 dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
555 hba->eh_flags, hba->req_abort_count);
556 dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n",
557 hba->ufs_version, hba->capabilities, hba->caps);
558 dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
559 hba->dev_quirks);
560 if (sdev_ufs)
561 dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n",
562 sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev);
563
564 ufshcd_print_clk_freqs(hba);
565 }
566
567 /**
568 * ufshcd_print_pwr_info - print power params as saved in hba
569 * power info
570 * @hba: per-adapter instance
571 */
ufshcd_print_pwr_info(struct ufs_hba * hba)572 static void ufshcd_print_pwr_info(struct ufs_hba *hba)
573 {
574 static const char * const names[] = {
575 "INVALID MODE",
576 "FAST MODE",
577 "SLOW_MODE",
578 "INVALID MODE",
579 "FASTAUTO_MODE",
580 "SLOWAUTO_MODE",
581 "INVALID MODE",
582 };
583
584 dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
585 __func__,
586 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
587 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
588 names[hba->pwr_info.pwr_rx],
589 names[hba->pwr_info.pwr_tx],
590 hba->pwr_info.hs_rate);
591 }
592
ufshcd_delay_us(unsigned long us,unsigned long tolerance)593 void ufshcd_delay_us(unsigned long us, unsigned long tolerance)
594 {
595 if (!us)
596 return;
597
598 if (us < 10)
599 udelay(us);
600 else
601 usleep_range(us, us + tolerance);
602 }
603 EXPORT_SYMBOL_GPL(ufshcd_delay_us);
604
605 /**
606 * ufshcd_wait_for_register - wait for register value to change
607 * @hba: per-adapter interface
608 * @reg: mmio register offset
609 * @mask: mask to apply to the read register value
610 * @val: value to wait for
611 * @interval_us: polling interval in microseconds
612 * @timeout_ms: timeout in milliseconds
613 *
614 * Return:
615 * -ETIMEDOUT on error, zero on success.
616 */
ufshcd_wait_for_register(struct ufs_hba * hba,u32 reg,u32 mask,u32 val,unsigned long interval_us,unsigned long timeout_ms)617 int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
618 u32 val, unsigned long interval_us,
619 unsigned long timeout_ms)
620 {
621 int err = 0;
622 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
623
624 /* ignore bits that we don't intend to wait on */
625 val = val & mask;
626
627 while ((ufshcd_readl(hba, reg) & mask) != val) {
628 usleep_range(interval_us, interval_us + 50);
629 if (time_after(jiffies, timeout)) {
630 if ((ufshcd_readl(hba, reg) & mask) != val)
631 err = -ETIMEDOUT;
632 break;
633 }
634 }
635
636 return err;
637 }
638
639 /**
640 * ufshcd_get_intr_mask - Get the interrupt bit mask
641 * @hba: Pointer to adapter instance
642 *
643 * Returns interrupt bit mask per version
644 */
ufshcd_get_intr_mask(struct ufs_hba * hba)645 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
646 {
647 if (hba->ufs_version == ufshci_version(1, 0))
648 return INTERRUPT_MASK_ALL_VER_10;
649 if (hba->ufs_version <= ufshci_version(2, 0))
650 return INTERRUPT_MASK_ALL_VER_11;
651
652 return INTERRUPT_MASK_ALL_VER_21;
653 }
654
655 /**
656 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
657 * @hba: Pointer to adapter instance
658 *
659 * Returns UFSHCI version supported by the controller
660 */
ufshcd_get_ufs_version(struct ufs_hba * hba)661 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
662 {
663 u32 ufshci_ver;
664
665 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
666 ufshci_ver = ufshcd_vops_get_ufs_hci_version(hba);
667 else
668 ufshci_ver = ufshcd_readl(hba, REG_UFS_VERSION);
669
670 /*
671 * UFSHCI v1.x uses a different version scheme, in order
672 * to allow the use of comparisons with the ufshci_version
673 * function, we convert it to the same scheme as ufs 2.0+.
674 */
675 if (ufshci_ver & 0x00010000)
676 return ufshci_version(1, ufshci_ver & 0x00000100);
677
678 return ufshci_ver;
679 }
680
681 /**
682 * ufshcd_is_device_present - Check if any device connected to
683 * the host controller
684 * @hba: pointer to adapter instance
685 *
686 * Returns true if device present, false if no device detected
687 */
ufshcd_is_device_present(struct ufs_hba * hba)688 static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
689 {
690 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
691 DEVICE_PRESENT) ? true : false;
692 }
693
694 /**
695 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
696 * @lrbp: pointer to local command reference block
697 *
698 * This function is used to get the OCS field from UTRD
699 * Returns the OCS field in the UTRD
700 */
ufshcd_get_tr_ocs(struct ufshcd_lrb * lrbp)701 static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
702 {
703 return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
704 }
705
706 /**
707 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
708 * @hba: per adapter instance
709 * @pos: position of the bit to be cleared
710 */
ufshcd_utrl_clear(struct ufs_hba * hba,u32 pos)711 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
712 {
713 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
714 ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
715 else
716 ufshcd_writel(hba, ~(1 << pos),
717 REG_UTP_TRANSFER_REQ_LIST_CLEAR);
718 }
719
720 /**
721 * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
722 * @hba: per adapter instance
723 * @pos: position of the bit to be cleared
724 */
ufshcd_utmrl_clear(struct ufs_hba * hba,u32 pos)725 static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
726 {
727 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
728 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
729 else
730 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
731 }
732
733 /**
734 * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
735 * @hba: per adapter instance
736 * @tag: position of the bit to be cleared
737 */
ufshcd_outstanding_req_clear(struct ufs_hba * hba,int tag)738 static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
739 {
740 clear_bit(tag, &hba->outstanding_reqs);
741 }
742
743 /**
744 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
745 * @reg: Register value of host controller status
746 *
747 * Returns integer, 0 on Success and positive value if failed
748 */
ufshcd_get_lists_status(u32 reg)749 static inline int ufshcd_get_lists_status(u32 reg)
750 {
751 return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
752 }
753
754 /**
755 * ufshcd_get_uic_cmd_result - Get the UIC command result
756 * @hba: Pointer to adapter instance
757 *
758 * This function gets the result of UIC command completion
759 * Returns 0 on success, non zero value on error
760 */
ufshcd_get_uic_cmd_result(struct ufs_hba * hba)761 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
762 {
763 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
764 MASK_UIC_COMMAND_RESULT;
765 }
766
767 /**
768 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
769 * @hba: Pointer to adapter instance
770 *
771 * This function gets UIC command argument3
772 * Returns 0 on success, non zero value on error
773 */
ufshcd_get_dme_attr_val(struct ufs_hba * hba)774 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
775 {
776 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
777 }
778
779 /**
780 * ufshcd_get_req_rsp - returns the TR response transaction type
781 * @ucd_rsp_ptr: pointer to response UPIU
782 */
783 static inline int
ufshcd_get_req_rsp(struct utp_upiu_rsp * ucd_rsp_ptr)784 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
785 {
786 return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
787 }
788
789 /**
790 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
791 * @ucd_rsp_ptr: pointer to response UPIU
792 *
793 * This function gets the response status and scsi_status from response UPIU
794 * Returns the response result code.
795 */
796 static inline int
ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp * ucd_rsp_ptr)797 ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
798 {
799 return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
800 }
801
802 /*
803 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
804 * from response UPIU
805 * @ucd_rsp_ptr: pointer to response UPIU
806 *
807 * Return the data segment length.
808 */
809 static inline unsigned int
ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp * ucd_rsp_ptr)810 ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
811 {
812 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
813 MASK_RSP_UPIU_DATA_SEG_LEN;
814 }
815
816 /**
817 * ufshcd_is_exception_event - Check if the device raised an exception event
818 * @ucd_rsp_ptr: pointer to response UPIU
819 *
820 * The function checks if the device raised an exception event indicated in
821 * the Device Information field of response UPIU.
822 *
823 * Returns true if exception is raised, false otherwise.
824 */
ufshcd_is_exception_event(struct utp_upiu_rsp * ucd_rsp_ptr)825 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
826 {
827 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
828 MASK_RSP_EXCEPTION_EVENT ? true : false;
829 }
830
831 /**
832 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
833 * @hba: per adapter instance
834 */
835 static inline void
ufshcd_reset_intr_aggr(struct ufs_hba * hba)836 ufshcd_reset_intr_aggr(struct ufs_hba *hba)
837 {
838 ufshcd_writel(hba, INT_AGGR_ENABLE |
839 INT_AGGR_COUNTER_AND_TIMER_RESET,
840 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
841 }
842
843 /**
844 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
845 * @hba: per adapter instance
846 * @cnt: Interrupt aggregation counter threshold
847 * @tmout: Interrupt aggregation timeout value
848 */
849 static inline void
ufshcd_config_intr_aggr(struct ufs_hba * hba,u8 cnt,u8 tmout)850 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
851 {
852 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
853 INT_AGGR_COUNTER_THLD_VAL(cnt) |
854 INT_AGGR_TIMEOUT_VAL(tmout),
855 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
856 }
857
858 /**
859 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
860 * @hba: per adapter instance
861 */
ufshcd_disable_intr_aggr(struct ufs_hba * hba)862 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
863 {
864 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
865 }
866
867 /**
868 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
869 * When run-stop registers are set to 1, it indicates the
870 * host controller that it can process the requests
871 * @hba: per adapter instance
872 */
ufshcd_enable_run_stop_reg(struct ufs_hba * hba)873 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
874 {
875 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
876 REG_UTP_TASK_REQ_LIST_RUN_STOP);
877 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
878 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
879 }
880
881 /**
882 * ufshcd_hba_start - Start controller initialization sequence
883 * @hba: per adapter instance
884 */
ufshcd_hba_start(struct ufs_hba * hba)885 static inline void ufshcd_hba_start(struct ufs_hba *hba)
886 {
887 u32 val = CONTROLLER_ENABLE;
888
889 if (ufshcd_crypto_enable(hba))
890 val |= CRYPTO_GENERAL_ENABLE;
891
892 ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE);
893 }
894
895 /**
896 * ufshcd_is_hba_active - Get controller state
897 * @hba: per adapter instance
898 *
899 * Returns false if controller is active, true otherwise
900 */
ufshcd_is_hba_active(struct ufs_hba * hba)901 static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
902 {
903 return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
904 ? false : true;
905 }
906
ufshcd_get_local_unipro_ver(struct ufs_hba * hba)907 u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
908 {
909 /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
910 if (hba->ufs_version <= ufshci_version(1, 1))
911 return UFS_UNIPRO_VER_1_41;
912 else
913 return UFS_UNIPRO_VER_1_6;
914 }
915 EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
916
ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba * hba)917 static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
918 {
919 /*
920 * If both host and device support UniPro ver1.6 or later, PA layer
921 * parameters tuning happens during link startup itself.
922 *
923 * We can manually tune PA layer parameters if either host or device
924 * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
925 * logic simple, we will only do manual tuning if local unipro version
926 * doesn't support ver1.6 or later.
927 */
928 if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
929 return true;
930 else
931 return false;
932 }
933
934 /**
935 * ufshcd_set_clk_freq - set UFS controller clock frequencies
936 * @hba: per adapter instance
937 * @scale_up: If True, set max possible frequency othewise set low frequency
938 *
939 * Returns 0 if successful
940 * Returns < 0 for any other errors
941 */
ufshcd_set_clk_freq(struct ufs_hba * hba,bool scale_up)942 static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up)
943 {
944 int ret = 0;
945 struct ufs_clk_info *clki;
946 struct list_head *head = &hba->clk_list_head;
947
948 if (list_empty(head))
949 goto out;
950
951 list_for_each_entry(clki, head, list) {
952 if (!IS_ERR_OR_NULL(clki->clk)) {
953 if (scale_up && clki->max_freq) {
954 if (clki->curr_freq == clki->max_freq)
955 continue;
956
957 ret = clk_set_rate(clki->clk, clki->max_freq);
958 if (ret) {
959 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
960 __func__, clki->name,
961 clki->max_freq, ret);
962 break;
963 }
964 trace_ufshcd_clk_scaling(dev_name(hba->dev),
965 "scaled up", clki->name,
966 clki->curr_freq,
967 clki->max_freq);
968
969 clki->curr_freq = clki->max_freq;
970
971 } else if (!scale_up && clki->min_freq) {
972 if (clki->curr_freq == clki->min_freq)
973 continue;
974
975 ret = clk_set_rate(clki->clk, clki->min_freq);
976 if (ret) {
977 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
978 __func__, clki->name,
979 clki->min_freq, ret);
980 break;
981 }
982 trace_ufshcd_clk_scaling(dev_name(hba->dev),
983 "scaled down", clki->name,
984 clki->curr_freq,
985 clki->min_freq);
986 clki->curr_freq = clki->min_freq;
987 }
988 }
989 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
990 clki->name, clk_get_rate(clki->clk));
991 }
992
993 out:
994 return ret;
995 }
996
997 /**
998 * ufshcd_scale_clks - scale up or scale down UFS controller clocks
999 * @hba: per adapter instance
1000 * @scale_up: True if scaling up and false if scaling down
1001 *
1002 * Returns 0 if successful
1003 * Returns < 0 for any other errors
1004 */
ufshcd_scale_clks(struct ufs_hba * hba,bool scale_up)1005 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
1006 {
1007 int ret = 0;
1008 ktime_t start = ktime_get();
1009
1010 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
1011 if (ret)
1012 goto out;
1013
1014 ret = ufshcd_set_clk_freq(hba, scale_up);
1015 if (ret)
1016 goto out;
1017
1018 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1019 if (ret)
1020 ufshcd_set_clk_freq(hba, !scale_up);
1021
1022 out:
1023 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1024 (scale_up ? "up" : "down"),
1025 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1026 return ret;
1027 }
1028
1029 /**
1030 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
1031 * @hba: per adapter instance
1032 * @scale_up: True if scaling up and false if scaling down
1033 *
1034 * Returns true if scaling is required, false otherwise.
1035 */
ufshcd_is_devfreq_scaling_required(struct ufs_hba * hba,bool scale_up)1036 static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
1037 bool scale_up)
1038 {
1039 struct ufs_clk_info *clki;
1040 struct list_head *head = &hba->clk_list_head;
1041
1042 if (list_empty(head))
1043 return false;
1044
1045 list_for_each_entry(clki, head, list) {
1046 if (!IS_ERR_OR_NULL(clki->clk)) {
1047 if (scale_up && clki->max_freq) {
1048 if (clki->curr_freq == clki->max_freq)
1049 continue;
1050 return true;
1051 } else if (!scale_up && clki->min_freq) {
1052 if (clki->curr_freq == clki->min_freq)
1053 continue;
1054 return true;
1055 }
1056 }
1057 }
1058
1059 return false;
1060 }
1061
ufshcd_wait_for_doorbell_clr(struct ufs_hba * hba,u64 wait_timeout_us)1062 static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
1063 u64 wait_timeout_us)
1064 {
1065 unsigned long flags;
1066 int ret = 0;
1067 u32 tm_doorbell;
1068 u32 tr_doorbell;
1069 bool timeout = false, do_last_check = false;
1070 ktime_t start;
1071
1072 ufshcd_hold(hba, false);
1073 spin_lock_irqsave(hba->host->host_lock, flags);
1074 /*
1075 * Wait for all the outstanding tasks/transfer requests.
1076 * Verify by checking the doorbell registers are clear.
1077 */
1078 start = ktime_get();
1079 do {
1080 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1081 ret = -EBUSY;
1082 goto out;
1083 }
1084
1085 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1086 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1087 if (!tm_doorbell && !tr_doorbell) {
1088 timeout = false;
1089 break;
1090 } else if (do_last_check) {
1091 break;
1092 }
1093
1094 spin_unlock_irqrestore(hba->host->host_lock, flags);
1095 schedule();
1096 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1097 wait_timeout_us) {
1098 timeout = true;
1099 /*
1100 * We might have scheduled out for long time so make
1101 * sure to check if doorbells are cleared by this time
1102 * or not.
1103 */
1104 do_last_check = true;
1105 }
1106 spin_lock_irqsave(hba->host->host_lock, flags);
1107 } while (tm_doorbell || tr_doorbell);
1108
1109 if (timeout) {
1110 dev_err(hba->dev,
1111 "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1112 __func__, tm_doorbell, tr_doorbell);
1113 ret = -EBUSY;
1114 }
1115 out:
1116 spin_unlock_irqrestore(hba->host->host_lock, flags);
1117 ufshcd_release(hba);
1118 return ret;
1119 }
1120
1121 /**
1122 * ufshcd_scale_gear - scale up/down UFS gear
1123 * @hba: per adapter instance
1124 * @scale_up: True for scaling up gear and false for scaling down
1125 *
1126 * Returns 0 for success,
1127 * Returns -EBUSY if scaling can't happen at this time
1128 * Returns non-zero for any other errors
1129 */
ufshcd_scale_gear(struct ufs_hba * hba,bool scale_up)1130 static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1131 {
1132 int ret = 0;
1133 struct ufs_pa_layer_attr new_pwr_info;
1134
1135 if (scale_up) {
1136 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1137 sizeof(struct ufs_pa_layer_attr));
1138 } else {
1139 memcpy(&new_pwr_info, &hba->pwr_info,
1140 sizeof(struct ufs_pa_layer_attr));
1141
1142 if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear ||
1143 hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) {
1144 /* save the current power mode */
1145 memcpy(&hba->clk_scaling.saved_pwr_info.info,
1146 &hba->pwr_info,
1147 sizeof(struct ufs_pa_layer_attr));
1148
1149 /* scale down gear */
1150 new_pwr_info.gear_tx = hba->clk_scaling.min_gear;
1151 new_pwr_info.gear_rx = hba->clk_scaling.min_gear;
1152 }
1153 }
1154
1155 /* check if the power mode needs to be changed or not? */
1156 ret = ufshcd_config_pwr_mode(hba, &new_pwr_info);
1157 if (ret)
1158 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1159 __func__, ret,
1160 hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1161 new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1162
1163 return ret;
1164 }
1165
ufshcd_clock_scaling_prepare(struct ufs_hba * hba)1166 static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1167 {
1168 #define DOORBELL_CLR_TOUT_US (1000 * 1000) /* 1 sec */
1169 int ret = 0;
1170 /*
1171 * make sure that there are no outstanding requests when
1172 * clock scaling is in progress
1173 */
1174 ufshcd_scsi_block_requests(hba);
1175 down_write(&hba->clk_scaling_lock);
1176
1177 if (!hba->clk_scaling.is_allowed ||
1178 ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
1179 ret = -EBUSY;
1180 up_write(&hba->clk_scaling_lock);
1181 ufshcd_scsi_unblock_requests(hba);
1182 goto out;
1183 }
1184
1185 /* let's not get into low power until clock scaling is completed */
1186 ufshcd_hold(hba, false);
1187
1188 out:
1189 return ret;
1190 }
1191
ufshcd_clock_scaling_unprepare(struct ufs_hba * hba,bool writelock)1192 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, bool writelock)
1193 {
1194 if (writelock)
1195 up_write(&hba->clk_scaling_lock);
1196 else
1197 up_read(&hba->clk_scaling_lock);
1198 ufshcd_scsi_unblock_requests(hba);
1199 ufshcd_release(hba);
1200 }
1201
1202 /**
1203 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1204 * @hba: per adapter instance
1205 * @scale_up: True for scaling up and false for scalin down
1206 *
1207 * Returns 0 for success,
1208 * Returns -EBUSY if scaling can't happen at this time
1209 * Returns non-zero for any other errors
1210 */
ufshcd_devfreq_scale(struct ufs_hba * hba,bool scale_up)1211 static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1212 {
1213 int ret = 0;
1214 bool is_writelock = true;
1215
1216 ret = ufshcd_clock_scaling_prepare(hba);
1217 if (ret)
1218 return ret;
1219
1220 /* scale down the gear before scaling down clocks */
1221 if (!scale_up) {
1222 ret = ufshcd_scale_gear(hba, false);
1223 if (ret)
1224 goto out_unprepare;
1225 }
1226
1227 ret = ufshcd_scale_clks(hba, scale_up);
1228 if (ret) {
1229 if (!scale_up)
1230 ufshcd_scale_gear(hba, true);
1231 goto out_unprepare;
1232 }
1233
1234 /* scale up the gear after scaling up clocks */
1235 if (scale_up) {
1236 ret = ufshcd_scale_gear(hba, true);
1237 if (ret) {
1238 ufshcd_scale_clks(hba, false);
1239 goto out_unprepare;
1240 }
1241 }
1242
1243 /* Enable Write Booster if we have scaled up else disable it */
1244 downgrade_write(&hba->clk_scaling_lock);
1245 is_writelock = false;
1246 ufshcd_wb_ctrl(hba, scale_up);
1247
1248 out_unprepare:
1249 ufshcd_clock_scaling_unprepare(hba, is_writelock);
1250 return ret;
1251 }
1252
ufshcd_clk_scaling_suspend_work(struct work_struct * work)1253 static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1254 {
1255 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1256 clk_scaling.suspend_work);
1257 unsigned long irq_flags;
1258
1259 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1260 if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1261 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1262 return;
1263 }
1264 hba->clk_scaling.is_suspended = true;
1265 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1266
1267 __ufshcd_suspend_clkscaling(hba);
1268 }
1269
ufshcd_clk_scaling_resume_work(struct work_struct * work)1270 static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1271 {
1272 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1273 clk_scaling.resume_work);
1274 unsigned long irq_flags;
1275
1276 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1277 if (!hba->clk_scaling.is_suspended) {
1278 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1279 return;
1280 }
1281 hba->clk_scaling.is_suspended = false;
1282 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1283
1284 devfreq_resume_device(hba->devfreq);
1285 }
1286
ufshcd_devfreq_target(struct device * dev,unsigned long * freq,u32 flags)1287 static int ufshcd_devfreq_target(struct device *dev,
1288 unsigned long *freq, u32 flags)
1289 {
1290 int ret = 0;
1291 struct ufs_hba *hba = dev_get_drvdata(dev);
1292 ktime_t start;
1293 bool scale_up, sched_clk_scaling_suspend_work = false;
1294 struct list_head *clk_list = &hba->clk_list_head;
1295 struct ufs_clk_info *clki;
1296 unsigned long irq_flags;
1297 bool force_out = false;
1298 bool force_scaling = false;
1299
1300 if (!ufshcd_is_clkscaling_supported(hba))
1301 return -EINVAL;
1302
1303 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1304 /* Override with the closest supported frequency */
1305 *freq = (unsigned long) clk_round_rate(clki->clk, *freq);
1306 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1307 if (ufshcd_eh_in_progress(hba)) {
1308 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1309 return 0;
1310 }
1311
1312 if (!hba->clk_scaling.active_reqs)
1313 sched_clk_scaling_suspend_work = true;
1314
1315 if (list_empty(clk_list)) {
1316 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1317 goto out;
1318 }
1319
1320 /* Decide based on the rounded-off frequency and update */
1321 scale_up = (*freq == clki->max_freq) ? true : false;
1322 if (!scale_up)
1323 *freq = clki->min_freq;
1324
1325 trace_android_vh_ufs_clock_scaling(hba, &force_out, &force_scaling, &scale_up);
1326
1327 /* Update the frequency */
1328 if (force_out || (!force_scaling && !ufshcd_is_devfreq_scaling_required(hba, scale_up))) {
1329 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1330 ret = 0;
1331 goto out; /* no state change required */
1332 }
1333 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1334
1335 start = ktime_get();
1336 ret = ufshcd_devfreq_scale(hba, scale_up);
1337
1338 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1339 (scale_up ? "up" : "down"),
1340 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1341
1342 out:
1343 if (sched_clk_scaling_suspend_work)
1344 queue_work(hba->clk_scaling.workq,
1345 &hba->clk_scaling.suspend_work);
1346
1347 return ret;
1348 }
1349
ufshcd_is_busy(struct request * req,void * priv,bool reserved)1350 static bool ufshcd_is_busy(struct request *req, void *priv, bool reserved)
1351 {
1352 int *busy = priv;
1353
1354 WARN_ON_ONCE(reserved);
1355 (*busy)++;
1356 return false;
1357 }
1358
1359 /* Whether or not any tag is in use by a request that is in progress. */
ufshcd_any_tag_in_use(struct ufs_hba * hba)1360 static bool ufshcd_any_tag_in_use(struct ufs_hba *hba)
1361 {
1362 struct request_queue *q = hba->cmd_queue;
1363 int busy = 0;
1364
1365 blk_mq_tagset_busy_iter(q->tag_set, ufshcd_is_busy, &busy);
1366 return busy;
1367 }
1368
ufshcd_devfreq_get_dev_status(struct device * dev,struct devfreq_dev_status * stat)1369 static int ufshcd_devfreq_get_dev_status(struct device *dev,
1370 struct devfreq_dev_status *stat)
1371 {
1372 struct ufs_hba *hba = dev_get_drvdata(dev);
1373 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1374 unsigned long flags;
1375 struct list_head *clk_list = &hba->clk_list_head;
1376 struct ufs_clk_info *clki;
1377 ktime_t curr_t;
1378
1379 if (!ufshcd_is_clkscaling_supported(hba))
1380 return -EINVAL;
1381
1382 memset(stat, 0, sizeof(*stat));
1383
1384 spin_lock_irqsave(hba->host->host_lock, flags);
1385 curr_t = ktime_get();
1386 if (!scaling->window_start_t)
1387 goto start_window;
1388
1389 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1390 /*
1391 * If current frequency is 0, then the ondemand governor considers
1392 * there's no initial frequency set. And it always requests to set
1393 * to max. frequency.
1394 */
1395 stat->current_frequency = clki->curr_freq;
1396 if (scaling->is_busy_started)
1397 scaling->tot_busy_t += ktime_us_delta(curr_t,
1398 scaling->busy_start_t);
1399
1400 stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t);
1401 stat->busy_time = scaling->tot_busy_t;
1402 start_window:
1403 scaling->window_start_t = curr_t;
1404 scaling->tot_busy_t = 0;
1405
1406 if (hba->outstanding_reqs) {
1407 scaling->busy_start_t = curr_t;
1408 scaling->is_busy_started = true;
1409 } else {
1410 scaling->busy_start_t = 0;
1411 scaling->is_busy_started = false;
1412 }
1413 spin_unlock_irqrestore(hba->host->host_lock, flags);
1414 return 0;
1415 }
1416
ufshcd_devfreq_init(struct ufs_hba * hba)1417 static int ufshcd_devfreq_init(struct ufs_hba *hba)
1418 {
1419 struct list_head *clk_list = &hba->clk_list_head;
1420 struct ufs_clk_info *clki;
1421 struct devfreq *devfreq;
1422 int ret;
1423
1424 /* Skip devfreq if we don't have any clocks in the list */
1425 if (list_empty(clk_list))
1426 return 0;
1427
1428 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1429 dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1430 dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1431
1432 ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile,
1433 &hba->vps->ondemand_data);
1434 devfreq = devfreq_add_device(hba->dev,
1435 &hba->vps->devfreq_profile,
1436 DEVFREQ_GOV_SIMPLE_ONDEMAND,
1437 &hba->vps->ondemand_data);
1438 if (IS_ERR(devfreq)) {
1439 ret = PTR_ERR(devfreq);
1440 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
1441
1442 dev_pm_opp_remove(hba->dev, clki->min_freq);
1443 dev_pm_opp_remove(hba->dev, clki->max_freq);
1444 return ret;
1445 }
1446
1447 hba->devfreq = devfreq;
1448
1449 return 0;
1450 }
1451
ufshcd_devfreq_remove(struct ufs_hba * hba)1452 static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1453 {
1454 struct list_head *clk_list = &hba->clk_list_head;
1455 struct ufs_clk_info *clki;
1456
1457 if (!hba->devfreq)
1458 return;
1459
1460 devfreq_remove_device(hba->devfreq);
1461 hba->devfreq = NULL;
1462
1463 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1464 dev_pm_opp_remove(hba->dev, clki->min_freq);
1465 dev_pm_opp_remove(hba->dev, clki->max_freq);
1466 }
1467
__ufshcd_suspend_clkscaling(struct ufs_hba * hba)1468 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1469 {
1470 unsigned long flags;
1471
1472 devfreq_suspend_device(hba->devfreq);
1473 spin_lock_irqsave(hba->host->host_lock, flags);
1474 hba->clk_scaling.window_start_t = 0;
1475 spin_unlock_irqrestore(hba->host->host_lock, flags);
1476 }
1477
ufshcd_suspend_clkscaling(struct ufs_hba * hba)1478 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1479 {
1480 unsigned long flags;
1481 bool suspend = false;
1482
1483 cancel_work_sync(&hba->clk_scaling.suspend_work);
1484 cancel_work_sync(&hba->clk_scaling.resume_work);
1485
1486 spin_lock_irqsave(hba->host->host_lock, flags);
1487 if (!hba->clk_scaling.is_suspended) {
1488 suspend = true;
1489 hba->clk_scaling.is_suspended = true;
1490 }
1491 spin_unlock_irqrestore(hba->host->host_lock, flags);
1492
1493 if (suspend)
1494 __ufshcd_suspend_clkscaling(hba);
1495 }
1496
ufshcd_resume_clkscaling(struct ufs_hba * hba)1497 static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1498 {
1499 unsigned long flags;
1500 bool resume = false;
1501
1502 spin_lock_irqsave(hba->host->host_lock, flags);
1503 if (hba->clk_scaling.is_suspended) {
1504 resume = true;
1505 hba->clk_scaling.is_suspended = false;
1506 }
1507 spin_unlock_irqrestore(hba->host->host_lock, flags);
1508
1509 if (resume)
1510 devfreq_resume_device(hba->devfreq);
1511 }
1512
ufshcd_clkscale_enable_show(struct device * dev,struct device_attribute * attr,char * buf)1513 static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1514 struct device_attribute *attr, char *buf)
1515 {
1516 struct ufs_hba *hba = dev_get_drvdata(dev);
1517
1518 return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_scaling.is_enabled);
1519 }
1520
ufshcd_clkscale_enable_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1521 static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1522 struct device_attribute *attr, const char *buf, size_t count)
1523 {
1524 struct ufs_hba *hba = dev_get_drvdata(dev);
1525 u32 value;
1526 int err = 0;
1527
1528 if (kstrtou32(buf, 0, &value))
1529 return -EINVAL;
1530
1531 down(&hba->host_sem);
1532 if (!ufshcd_is_user_access_allowed(hba)) {
1533 err = -EBUSY;
1534 goto out;
1535 }
1536
1537 value = !!value;
1538 if (value == hba->clk_scaling.is_enabled)
1539 goto out;
1540
1541 pm_runtime_get_sync(hba->dev);
1542 ufshcd_hold(hba, false);
1543
1544 hba->clk_scaling.is_enabled = value;
1545
1546 if (value) {
1547 ufshcd_resume_clkscaling(hba);
1548 } else {
1549 ufshcd_suspend_clkscaling(hba);
1550 err = ufshcd_devfreq_scale(hba, true);
1551 if (err)
1552 dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1553 __func__, err);
1554 }
1555
1556 ufshcd_release(hba);
1557 pm_runtime_put_sync(hba->dev);
1558 out:
1559 up(&hba->host_sem);
1560 return err ? err : count;
1561 }
1562
ufshcd_init_clk_scaling_sysfs(struct ufs_hba * hba)1563 static void ufshcd_init_clk_scaling_sysfs(struct ufs_hba *hba)
1564 {
1565 hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1566 hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1567 sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1568 hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1569 hba->clk_scaling.enable_attr.attr.mode = 0644;
1570 if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1571 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1572 }
1573
ufshcd_remove_clk_scaling_sysfs(struct ufs_hba * hba)1574 static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba)
1575 {
1576 if (hba->clk_scaling.enable_attr.attr.name)
1577 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
1578 }
1579
ufshcd_init_clk_scaling(struct ufs_hba * hba)1580 static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1581 {
1582 char wq_name[sizeof("ufs_clkscaling_00")];
1583
1584 if (!ufshcd_is_clkscaling_supported(hba))
1585 return;
1586
1587 if (!hba->clk_scaling.min_gear)
1588 hba->clk_scaling.min_gear = UFS_HS_G1;
1589
1590 INIT_WORK(&hba->clk_scaling.suspend_work,
1591 ufshcd_clk_scaling_suspend_work);
1592 INIT_WORK(&hba->clk_scaling.resume_work,
1593 ufshcd_clk_scaling_resume_work);
1594
1595 snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1596 hba->host->host_no);
1597 hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1598
1599 hba->clk_scaling.is_initialized = true;
1600 }
1601
ufshcd_exit_clk_scaling(struct ufs_hba * hba)1602 static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1603 {
1604 if (!hba->clk_scaling.is_initialized)
1605 return;
1606
1607 ufshcd_remove_clk_scaling_sysfs(hba);
1608 destroy_workqueue(hba->clk_scaling.workq);
1609 ufshcd_devfreq_remove(hba);
1610 hba->clk_scaling.is_initialized = false;
1611 }
1612
ufshcd_ungate_work(struct work_struct * work)1613 static void ufshcd_ungate_work(struct work_struct *work)
1614 {
1615 int ret;
1616 unsigned long flags;
1617 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1618 clk_gating.ungate_work);
1619
1620 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1621
1622 spin_lock_irqsave(hba->host->host_lock, flags);
1623 if (hba->clk_gating.state == CLKS_ON) {
1624 spin_unlock_irqrestore(hba->host->host_lock, flags);
1625 goto unblock_reqs;
1626 }
1627
1628 spin_unlock_irqrestore(hba->host->host_lock, flags);
1629 ufshcd_hba_vreg_set_hpm(hba);
1630 ufshcd_setup_clocks(hba, true);
1631
1632 ufshcd_enable_irq(hba);
1633
1634 /* Exit from hibern8 */
1635 if (ufshcd_can_hibern8_during_gating(hba)) {
1636 /* Prevent gating in this path */
1637 hba->clk_gating.is_suspended = true;
1638 if (ufshcd_is_link_hibern8(hba)) {
1639 ret = ufshcd_uic_hibern8_exit(hba);
1640 if (ret)
1641 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1642 __func__, ret);
1643 else
1644 ufshcd_set_link_active(hba);
1645 }
1646 hba->clk_gating.is_suspended = false;
1647 }
1648 unblock_reqs:
1649 ufshcd_scsi_unblock_requests(hba);
1650 }
1651
1652 /**
1653 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1654 * Also, exit from hibern8 mode and set the link as active.
1655 * @hba: per adapter instance
1656 * @async: This indicates whether caller should ungate clocks asynchronously.
1657 */
ufshcd_hold(struct ufs_hba * hba,bool async)1658 int ufshcd_hold(struct ufs_hba *hba, bool async)
1659 {
1660 int rc = 0;
1661 bool flush_result;
1662 unsigned long flags;
1663
1664 if (!ufshcd_is_clkgating_allowed(hba))
1665 goto out;
1666 spin_lock_irqsave(hba->host->host_lock, flags);
1667 hba->clk_gating.active_reqs++;
1668
1669 start:
1670 switch (hba->clk_gating.state) {
1671 case CLKS_ON:
1672 /*
1673 * Wait for the ungate work to complete if in progress.
1674 * Though the clocks may be in ON state, the link could
1675 * still be in hibner8 state if hibern8 is allowed
1676 * during clock gating.
1677 * Make sure we exit hibern8 state also in addition to
1678 * clocks being ON.
1679 */
1680 if (ufshcd_can_hibern8_during_gating(hba) &&
1681 ufshcd_is_link_hibern8(hba)) {
1682 if (async) {
1683 rc = -EAGAIN;
1684 hba->clk_gating.active_reqs--;
1685 break;
1686 }
1687 spin_unlock_irqrestore(hba->host->host_lock, flags);
1688 flush_result = flush_work(&hba->clk_gating.ungate_work);
1689 if (hba->clk_gating.is_suspended && !flush_result)
1690 goto out;
1691 spin_lock_irqsave(hba->host->host_lock, flags);
1692 goto start;
1693 }
1694 break;
1695 case REQ_CLKS_OFF:
1696 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1697 hba->clk_gating.state = CLKS_ON;
1698 trace_ufshcd_clk_gating(dev_name(hba->dev),
1699 hba->clk_gating.state);
1700 break;
1701 }
1702 /*
1703 * If we are here, it means gating work is either done or
1704 * currently running. Hence, fall through to cancel gating
1705 * work and to enable clocks.
1706 */
1707 fallthrough;
1708 case CLKS_OFF:
1709 hba->clk_gating.state = REQ_CLKS_ON;
1710 trace_ufshcd_clk_gating(dev_name(hba->dev),
1711 hba->clk_gating.state);
1712 if (queue_work(hba->clk_gating.clk_gating_workq,
1713 &hba->clk_gating.ungate_work))
1714 ufshcd_scsi_block_requests(hba);
1715 /*
1716 * fall through to check if we should wait for this
1717 * work to be done or not.
1718 */
1719 fallthrough;
1720 case REQ_CLKS_ON:
1721 if (async) {
1722 rc = -EAGAIN;
1723 hba->clk_gating.active_reqs--;
1724 break;
1725 }
1726
1727 spin_unlock_irqrestore(hba->host->host_lock, flags);
1728 flush_work(&hba->clk_gating.ungate_work);
1729 /* Make sure state is CLKS_ON before returning */
1730 spin_lock_irqsave(hba->host->host_lock, flags);
1731 goto start;
1732 default:
1733 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1734 __func__, hba->clk_gating.state);
1735 break;
1736 }
1737 spin_unlock_irqrestore(hba->host->host_lock, flags);
1738 out:
1739 return rc;
1740 }
1741 EXPORT_SYMBOL_GPL(ufshcd_hold);
1742
ufshcd_gate_work(struct work_struct * work)1743 static void ufshcd_gate_work(struct work_struct *work)
1744 {
1745 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1746 clk_gating.gate_work.work);
1747 unsigned long flags;
1748 int ret;
1749
1750 spin_lock_irqsave(hba->host->host_lock, flags);
1751 /*
1752 * In case you are here to cancel this work the gating state
1753 * would be marked as REQ_CLKS_ON. In this case save time by
1754 * skipping the gating work and exit after changing the clock
1755 * state to CLKS_ON.
1756 */
1757 if (hba->clk_gating.is_suspended ||
1758 (hba->clk_gating.state != REQ_CLKS_OFF)) {
1759 hba->clk_gating.state = CLKS_ON;
1760 trace_ufshcd_clk_gating(dev_name(hba->dev),
1761 hba->clk_gating.state);
1762 goto rel_lock;
1763 }
1764
1765 if (hba->clk_gating.active_reqs
1766 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1767 || ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks
1768 || hba->active_uic_cmd || hba->uic_async_done)
1769 goto rel_lock;
1770
1771 spin_unlock_irqrestore(hba->host->host_lock, flags);
1772
1773 /* put the link into hibern8 mode before turning off clocks */
1774 if (ufshcd_can_hibern8_during_gating(hba)) {
1775 ret = ufshcd_uic_hibern8_enter(hba);
1776 if (ret) {
1777 hba->clk_gating.state = CLKS_ON;
1778 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
1779 __func__, ret);
1780 trace_ufshcd_clk_gating(dev_name(hba->dev),
1781 hba->clk_gating.state);
1782 goto out;
1783 }
1784 ufshcd_set_link_hibern8(hba);
1785 }
1786
1787 ufshcd_disable_irq(hba);
1788
1789 ufshcd_setup_clocks(hba, false);
1790
1791 /* Put the host controller in low power mode if possible */
1792 ufshcd_hba_vreg_set_lpm(hba);
1793 /*
1794 * In case you are here to cancel this work the gating state
1795 * would be marked as REQ_CLKS_ON. In this case keep the state
1796 * as REQ_CLKS_ON which would anyway imply that clocks are off
1797 * and a request to turn them on is pending. By doing this way,
1798 * we keep the state machine in tact and this would ultimately
1799 * prevent from doing cancel work multiple times when there are
1800 * new requests arriving before the current cancel work is done.
1801 */
1802 spin_lock_irqsave(hba->host->host_lock, flags);
1803 if (hba->clk_gating.state == REQ_CLKS_OFF) {
1804 hba->clk_gating.state = CLKS_OFF;
1805 trace_ufshcd_clk_gating(dev_name(hba->dev),
1806 hba->clk_gating.state);
1807 }
1808 rel_lock:
1809 spin_unlock_irqrestore(hba->host->host_lock, flags);
1810 out:
1811 return;
1812 }
1813
1814 /* host lock must be held before calling this variant */
__ufshcd_release(struct ufs_hba * hba)1815 static void __ufshcd_release(struct ufs_hba *hba)
1816 {
1817 if (!ufshcd_is_clkgating_allowed(hba))
1818 return;
1819
1820 hba->clk_gating.active_reqs--;
1821
1822 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended ||
1823 hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL ||
1824 hba->outstanding_tasks ||
1825 hba->active_uic_cmd || hba->uic_async_done ||
1826 hba->clk_gating.state == CLKS_OFF)
1827 return;
1828
1829 hba->clk_gating.state = REQ_CLKS_OFF;
1830 trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
1831 queue_delayed_work(hba->clk_gating.clk_gating_workq,
1832 &hba->clk_gating.gate_work,
1833 msecs_to_jiffies(hba->clk_gating.delay_ms));
1834 }
1835
ufshcd_release(struct ufs_hba * hba)1836 void ufshcd_release(struct ufs_hba *hba)
1837 {
1838 unsigned long flags;
1839
1840 spin_lock_irqsave(hba->host->host_lock, flags);
1841 __ufshcd_release(hba);
1842 spin_unlock_irqrestore(hba->host->host_lock, flags);
1843 }
1844 EXPORT_SYMBOL_GPL(ufshcd_release);
1845
ufshcd_clkgate_delay_show(struct device * dev,struct device_attribute * attr,char * buf)1846 static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1847 struct device_attribute *attr, char *buf)
1848 {
1849 struct ufs_hba *hba = dev_get_drvdata(dev);
1850
1851 return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms);
1852 }
1853
ufshcd_clkgate_delay_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1854 static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1855 struct device_attribute *attr, const char *buf, size_t count)
1856 {
1857 struct ufs_hba *hba = dev_get_drvdata(dev);
1858 unsigned long flags, value;
1859
1860 if (kstrtoul(buf, 0, &value))
1861 return -EINVAL;
1862
1863 spin_lock_irqsave(hba->host->host_lock, flags);
1864 hba->clk_gating.delay_ms = value;
1865 spin_unlock_irqrestore(hba->host->host_lock, flags);
1866 return count;
1867 }
1868
ufshcd_clkgate_enable_show(struct device * dev,struct device_attribute * attr,char * buf)1869 static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1870 struct device_attribute *attr, char *buf)
1871 {
1872 struct ufs_hba *hba = dev_get_drvdata(dev);
1873
1874 return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_gating.is_enabled);
1875 }
1876
ufshcd_clkgate_enable_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1877 static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1878 struct device_attribute *attr, const char *buf, size_t count)
1879 {
1880 struct ufs_hba *hba = dev_get_drvdata(dev);
1881 unsigned long flags;
1882 u32 value;
1883
1884 if (kstrtou32(buf, 0, &value))
1885 return -EINVAL;
1886
1887 value = !!value;
1888
1889 spin_lock_irqsave(hba->host->host_lock, flags);
1890 if (value == hba->clk_gating.is_enabled)
1891 goto out;
1892
1893 if (value)
1894 __ufshcd_release(hba);
1895 else
1896 hba->clk_gating.active_reqs++;
1897
1898 hba->clk_gating.is_enabled = value;
1899 out:
1900 spin_unlock_irqrestore(hba->host->host_lock, flags);
1901 return count;
1902 }
1903
ufshcd_init_clk_gating_sysfs(struct ufs_hba * hba)1904 static void ufshcd_init_clk_gating_sysfs(struct ufs_hba *hba)
1905 {
1906 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1907 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1908 sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1909 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
1910 hba->clk_gating.delay_attr.attr.mode = 0644;
1911 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1912 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
1913
1914 hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1915 hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1916 sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1917 hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1918 hba->clk_gating.enable_attr.attr.mode = 0644;
1919 if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1920 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1921 }
1922
ufshcd_remove_clk_gating_sysfs(struct ufs_hba * hba)1923 static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba)
1924 {
1925 if (hba->clk_gating.delay_attr.attr.name)
1926 device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
1927 if (hba->clk_gating.enable_attr.attr.name)
1928 device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
1929 }
1930
ufshcd_init_clk_gating(struct ufs_hba * hba)1931 static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1932 {
1933 char wq_name[sizeof("ufs_clk_gating_00")];
1934
1935 if (!ufshcd_is_clkgating_allowed(hba))
1936 return;
1937
1938 hba->clk_gating.state = CLKS_ON;
1939
1940 hba->clk_gating.delay_ms = 150;
1941 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1942 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1943
1944 snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1945 hba->host->host_no);
1946 hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
1947 WQ_MEM_RECLAIM | WQ_HIGHPRI);
1948
1949 ufshcd_init_clk_gating_sysfs(hba);
1950
1951 hba->clk_gating.is_enabled = true;
1952 hba->clk_gating.is_initialized = true;
1953 }
1954
ufshcd_exit_clk_gating(struct ufs_hba * hba)1955 static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
1956 {
1957 if (!hba->clk_gating.is_initialized)
1958 return;
1959 ufshcd_remove_clk_gating_sysfs(hba);
1960 cancel_work_sync(&hba->clk_gating.ungate_work);
1961 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1962 destroy_workqueue(hba->clk_gating.clk_gating_workq);
1963 hba->clk_gating.is_initialized = false;
1964 }
1965
1966 /* Must be called with host lock acquired */
ufshcd_clk_scaling_start_busy(struct ufs_hba * hba)1967 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
1968 {
1969 bool queue_resume_work = false;
1970 ktime_t curr_t = ktime_get();
1971 unsigned long flags;
1972
1973 if (!ufshcd_is_clkscaling_supported(hba))
1974 return;
1975
1976 spin_lock_irqsave(hba->host->host_lock, flags);
1977 if (!hba->clk_scaling.active_reqs++)
1978 queue_resume_work = true;
1979
1980 if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) {
1981 spin_unlock_irqrestore(hba->host->host_lock, flags);
1982 return;
1983 }
1984
1985 if (queue_resume_work)
1986 queue_work(hba->clk_scaling.workq,
1987 &hba->clk_scaling.resume_work);
1988
1989 if (!hba->clk_scaling.window_start_t) {
1990 hba->clk_scaling.window_start_t = curr_t;
1991 hba->clk_scaling.tot_busy_t = 0;
1992 hba->clk_scaling.is_busy_started = false;
1993 }
1994
1995 if (!hba->clk_scaling.is_busy_started) {
1996 hba->clk_scaling.busy_start_t = curr_t;
1997 hba->clk_scaling.is_busy_started = true;
1998 }
1999 spin_unlock_irqrestore(hba->host->host_lock, flags);
2000 }
2001
ufshcd_clk_scaling_update_busy(struct ufs_hba * hba)2002 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
2003 {
2004 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
2005 unsigned long flags;
2006
2007 if (!ufshcd_is_clkscaling_supported(hba))
2008 return;
2009
2010 spin_lock_irqsave(hba->host->host_lock, flags);
2011 hba->clk_scaling.active_reqs--;
2012 if (!hba->outstanding_reqs && scaling->is_busy_started) {
2013 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
2014 scaling->busy_start_t));
2015 scaling->busy_start_t = 0;
2016 scaling->is_busy_started = false;
2017 }
2018 spin_unlock_irqrestore(hba->host->host_lock, flags);
2019 }
2020
ufshcd_monitor_opcode2dir(u8 opcode)2021 static inline int ufshcd_monitor_opcode2dir(u8 opcode)
2022 {
2023 if (opcode == READ_6 || opcode == READ_10 || opcode == READ_16)
2024 return READ;
2025 else if (opcode == WRITE_6 || opcode == WRITE_10 || opcode == WRITE_16)
2026 return WRITE;
2027 else
2028 return -EINVAL;
2029 }
2030
ufshcd_should_inform_monitor(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2031 static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba,
2032 struct ufshcd_lrb *lrbp)
2033 {
2034 struct ufs_hba_monitor *m = &hba->monitor;
2035
2036 return (m->enabled && lrbp && lrbp->cmd &&
2037 (!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) &&
2038 ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp));
2039 }
2040
ufshcd_start_monitor(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2041 static void ufshcd_start_monitor(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2042 {
2043 int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2044 unsigned long flags;
2045
2046 spin_lock_irqsave(hba->host->host_lock, flags);
2047 if (dir >= 0 && hba->monitor.nr_queued[dir]++ == 0)
2048 hba->monitor.busy_start_ts[dir] = ktime_get();
2049 spin_unlock_irqrestore(hba->host->host_lock, flags);
2050 }
2051
ufshcd_update_monitor(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2052 static void ufshcd_update_monitor(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2053 {
2054 int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2055 unsigned long flags;
2056
2057 spin_lock_irqsave(hba->host->host_lock, flags);
2058 if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) {
2059 struct request *req = lrbp->cmd->request;
2060 struct ufs_hba_monitor *m = &hba->monitor;
2061 ktime_t now, inc, lat;
2062
2063 now = lrbp->compl_time_stamp;
2064 inc = ktime_sub(now, m->busy_start_ts[dir]);
2065 m->total_busy[dir] = ktime_add(m->total_busy[dir], inc);
2066 m->nr_sec_rw[dir] += blk_rq_sectors(req);
2067
2068 /* Update latencies */
2069 m->nr_req[dir]++;
2070 lat = ktime_sub(now, lrbp->issue_time_stamp);
2071 m->lat_sum[dir] += lat;
2072 if (m->lat_max[dir] < lat || !m->lat_max[dir])
2073 m->lat_max[dir] = lat;
2074 if (m->lat_min[dir] > lat || !m->lat_min[dir])
2075 m->lat_min[dir] = lat;
2076
2077 m->nr_queued[dir]--;
2078 /* Push forward the busy start of monitor */
2079 m->busy_start_ts[dir] = now;
2080 }
2081 spin_unlock_irqrestore(hba->host->host_lock, flags);
2082 }
2083
2084 /**
2085 * ufshcd_send_command - Send SCSI or device management commands
2086 * @hba: per adapter instance
2087 * @task_tag: Task tag of the command
2088 */
2089 static inline
ufshcd_send_command(struct ufs_hba * hba,unsigned int task_tag)2090 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
2091 {
2092 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
2093
2094 lrbp->issue_time_stamp = ktime_get();
2095 lrbp->compl_time_stamp = ktime_set(0, 0);
2096 trace_android_vh_ufs_send_command(hba, lrbp);
2097 ufshcd_add_command_trace(hba, task_tag, "send");
2098 ufshcd_clk_scaling_start_busy(hba);
2099 if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
2100 ufshcd_start_monitor(hba, lrbp);
2101 if (hba->vops && hba->vops->setup_xfer_req)
2102 hba->vops->setup_xfer_req(hba, task_tag, !!lrbp->cmd);
2103 if (ufshcd_has_utrlcnr(hba)) {
2104 set_bit(task_tag, &hba->outstanding_reqs);
2105 ufshcd_writel(hba, 1 << task_tag,
2106 REG_UTP_TRANSFER_REQ_DOOR_BELL);
2107 } else {
2108 unsigned long flags;
2109
2110 spin_lock_irqsave(hba->host->host_lock, flags);
2111 set_bit(task_tag, &hba->outstanding_reqs);
2112 ufshcd_writel(hba, 1 << task_tag,
2113 REG_UTP_TRANSFER_REQ_DOOR_BELL);
2114 spin_unlock_irqrestore(hba->host->host_lock, flags);
2115 }
2116 /* Make sure that doorbell is committed immediately */
2117 wmb();
2118 }
2119
2120 /**
2121 * ufshcd_copy_sense_data - Copy sense data in case of check condition
2122 * @lrbp: pointer to local reference block
2123 */
ufshcd_copy_sense_data(struct ufshcd_lrb * lrbp)2124 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
2125 {
2126 int len;
2127 if (lrbp->sense_buffer &&
2128 ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
2129 int len_to_copy;
2130
2131 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
2132 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
2133
2134 memcpy(lrbp->sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
2135 len_to_copy);
2136 }
2137 }
2138
2139 /**
2140 * ufshcd_copy_query_response() - Copy the Query Response and the data
2141 * descriptor
2142 * @hba: per adapter instance
2143 * @lrbp: pointer to local reference block
2144 */
2145 static
ufshcd_copy_query_response(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2146 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2147 {
2148 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2149
2150 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
2151
2152 /* Get the descriptor */
2153 if (hba->dev_cmd.query.descriptor &&
2154 lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
2155 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
2156 GENERAL_UPIU_REQUEST_SIZE;
2157 u16 resp_len;
2158 u16 buf_len;
2159
2160 /* data segment length */
2161 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
2162 MASK_QUERY_DATA_SEG_LEN;
2163 buf_len = be16_to_cpu(
2164 hba->dev_cmd.query.request.upiu_req.length);
2165 if (likely(buf_len >= resp_len)) {
2166 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
2167 } else {
2168 dev_warn(hba->dev,
2169 "%s: rsp size %d is bigger than buffer size %d",
2170 __func__, resp_len, buf_len);
2171 return -EINVAL;
2172 }
2173 }
2174
2175 return 0;
2176 }
2177
2178 /**
2179 * ufshcd_hba_capabilities - Read controller capabilities
2180 * @hba: per adapter instance
2181 *
2182 * Return: 0 on success, negative on error.
2183 */
ufshcd_hba_capabilities(struct ufs_hba * hba)2184 static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
2185 {
2186 int err;
2187
2188 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
2189
2190 /* nutrs and nutmrs are 0 based values */
2191 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
2192 hba->nutmrs =
2193 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
2194 ufs_hba_add_info(hba)->reserved_slot = hba->nutrs - 1;
2195
2196 /* Read crypto capabilities */
2197 err = ufshcd_hba_init_crypto_capabilities(hba);
2198 if (err)
2199 dev_err(hba->dev, "crypto setup failed\n");
2200
2201 return err;
2202 }
2203
2204 /**
2205 * ufshcd_ready_for_uic_cmd - Check if controller is ready
2206 * to accept UIC commands
2207 * @hba: per adapter instance
2208 * Return true on success, else false
2209 */
ufshcd_ready_for_uic_cmd(struct ufs_hba * hba)2210 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
2211 {
2212 if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
2213 return true;
2214 else
2215 return false;
2216 }
2217
2218 /**
2219 * ufshcd_get_upmcrs - Get the power mode change request status
2220 * @hba: Pointer to adapter instance
2221 *
2222 * This function gets the UPMCRS field of HCS register
2223 * Returns value of UPMCRS field
2224 */
ufshcd_get_upmcrs(struct ufs_hba * hba)2225 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
2226 {
2227 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
2228 }
2229
2230 /**
2231 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
2232 * @hba: per adapter instance
2233 * @uic_cmd: UIC command
2234 *
2235 * Mutex must be held.
2236 */
2237 static inline void
ufshcd_dispatch_uic_cmd(struct ufs_hba * hba,struct uic_command * uic_cmd)2238 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2239 {
2240 WARN_ON(hba->active_uic_cmd);
2241
2242 hba->active_uic_cmd = uic_cmd;
2243
2244 /* Write Args */
2245 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2246 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2247 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
2248
2249 ufshcd_add_uic_command_trace(hba, uic_cmd, "send");
2250
2251 /* Write UIC Cmd */
2252 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
2253 REG_UIC_COMMAND);
2254 }
2255
2256 /**
2257 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
2258 * @hba: per adapter instance
2259 * @uic_cmd: UIC command
2260 *
2261 * Must be called with mutex held.
2262 * Returns 0 only if success.
2263 */
2264 static int
ufshcd_wait_for_uic_cmd(struct ufs_hba * hba,struct uic_command * uic_cmd)2265 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2266 {
2267 int ret;
2268 unsigned long flags;
2269
2270 if (wait_for_completion_timeout(&uic_cmd->done,
2271 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
2272 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2273 } else {
2274 ret = -ETIMEDOUT;
2275 dev_err(hba->dev,
2276 "uic cmd 0x%x with arg3 0x%x completion timeout\n",
2277 uic_cmd->command, uic_cmd->argument3);
2278
2279 if (!uic_cmd->cmd_active) {
2280 dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n",
2281 __func__);
2282 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2283 }
2284 }
2285
2286 spin_lock_irqsave(hba->host->host_lock, flags);
2287 hba->active_uic_cmd = NULL;
2288 spin_unlock_irqrestore(hba->host->host_lock, flags);
2289
2290 return ret;
2291 }
2292
2293 /**
2294 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2295 * @hba: per adapter instance
2296 * @uic_cmd: UIC command
2297 * @completion: initialize the completion only if this is set to true
2298 *
2299 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
2300 * with mutex held and host_lock locked.
2301 * Returns 0 only if success.
2302 */
2303 static int
__ufshcd_send_uic_cmd(struct ufs_hba * hba,struct uic_command * uic_cmd,bool completion)2304 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2305 bool completion)
2306 {
2307 if (!ufshcd_ready_for_uic_cmd(hba)) {
2308 dev_err(hba->dev,
2309 "Controller not ready to accept UIC commands\n");
2310 return -EIO;
2311 }
2312
2313 if (completion)
2314 init_completion(&uic_cmd->done);
2315
2316 uic_cmd->cmd_active = 1;
2317 ufshcd_dispatch_uic_cmd(hba, uic_cmd);
2318
2319 return 0;
2320 }
2321
2322 /**
2323 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2324 * @hba: per adapter instance
2325 * @uic_cmd: UIC command
2326 *
2327 * Returns 0 only if success.
2328 */
ufshcd_send_uic_cmd(struct ufs_hba * hba,struct uic_command * uic_cmd)2329 int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2330 {
2331 int ret;
2332 unsigned long flags;
2333
2334 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD)
2335 return 0;
2336
2337 ufshcd_hold(hba, false);
2338 mutex_lock(&hba->uic_cmd_mutex);
2339 ufshcd_add_delay_before_dme_cmd(hba);
2340
2341 spin_lock_irqsave(hba->host->host_lock, flags);
2342 ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
2343 spin_unlock_irqrestore(hba->host->host_lock, flags);
2344 if (!ret)
2345 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2346
2347 mutex_unlock(&hba->uic_cmd_mutex);
2348
2349 ufshcd_release(hba);
2350 return ret;
2351 }
2352
2353 /**
2354 * ufshcd_map_sg - Map scatter-gather list to prdt
2355 * @hba: per adapter instance
2356 * @lrbp: pointer to local reference block
2357 *
2358 * Returns 0 in case of success, non-zero value in case of failure
2359 */
ufshcd_map_sg(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2360 static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2361 {
2362 struct ufshcd_sg_entry *prd;
2363 struct scatterlist *sg;
2364 struct scsi_cmnd *cmd;
2365 int sg_segments;
2366 int i;
2367 int err;
2368
2369 cmd = lrbp->cmd;
2370 sg_segments = scsi_dma_map(cmd);
2371 if (sg_segments < 0)
2372 return sg_segments;
2373
2374 if (sg_segments) {
2375
2376 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2377 lrbp->utr_descriptor_ptr->prd_table_length =
2378 cpu_to_le16(sg_segments * hba->sg_entry_size);
2379 else
2380 lrbp->utr_descriptor_ptr->prd_table_length =
2381 cpu_to_le16((u16) (sg_segments));
2382
2383 prd = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
2384
2385 scsi_for_each_sg(cmd, sg, sg_segments, i) {
2386 prd->size =
2387 cpu_to_le32(((u32) sg_dma_len(sg))-1);
2388 prd->base_addr =
2389 cpu_to_le32(lower_32_bits(sg->dma_address));
2390 prd->upper_addr =
2391 cpu_to_le32(upper_32_bits(sg->dma_address));
2392 prd->reserved = 0;
2393 prd = (void *)prd + hba->sg_entry_size;
2394 }
2395 } else {
2396 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2397 }
2398
2399 err = 0;
2400 trace_android_vh_ufs_fill_prdt(hba, lrbp, sg_segments, &err);
2401 return err;
2402 }
2403
2404 /**
2405 * ufshcd_enable_intr - enable interrupts
2406 * @hba: per adapter instance
2407 * @intrs: interrupt bits
2408 */
ufshcd_enable_intr(struct ufs_hba * hba,u32 intrs)2409 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
2410 {
2411 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2412
2413 if (hba->ufs_version == ufshci_version(1, 0)) {
2414 u32 rw;
2415 rw = set & INTERRUPT_MASK_RW_VER_10;
2416 set = rw | ((set ^ intrs) & intrs);
2417 } else {
2418 set |= intrs;
2419 }
2420
2421 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2422 }
2423
2424 /**
2425 * ufshcd_disable_intr - disable interrupts
2426 * @hba: per adapter instance
2427 * @intrs: interrupt bits
2428 */
ufshcd_disable_intr(struct ufs_hba * hba,u32 intrs)2429 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2430 {
2431 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2432
2433 if (hba->ufs_version == ufshci_version(1, 0)) {
2434 u32 rw;
2435 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2436 ~(intrs & INTERRUPT_MASK_RW_VER_10);
2437 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2438
2439 } else {
2440 set &= ~intrs;
2441 }
2442
2443 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2444 }
2445
2446 /**
2447 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2448 * descriptor according to request
2449 * @lrbp: pointer to local reference block
2450 * @upiu_flags: flags required in the header
2451 * @cmd_dir: requests data direction
2452 */
ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb * lrbp,u8 * upiu_flags,enum dma_data_direction cmd_dir)2453 static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
2454 u8 *upiu_flags, enum dma_data_direction cmd_dir)
2455 {
2456 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2457 u32 data_direction;
2458 u32 dword_0;
2459 u32 dword_1 = 0;
2460 u32 dword_3 = 0;
2461
2462 if (cmd_dir == DMA_FROM_DEVICE) {
2463 data_direction = UTP_DEVICE_TO_HOST;
2464 *upiu_flags = UPIU_CMD_FLAGS_READ;
2465 } else if (cmd_dir == DMA_TO_DEVICE) {
2466 data_direction = UTP_HOST_TO_DEVICE;
2467 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2468 } else {
2469 data_direction = UTP_NO_DATA_TRANSFER;
2470 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2471 }
2472
2473 dword_0 = data_direction | (lrbp->command_type
2474 << UPIU_COMMAND_TYPE_OFFSET);
2475 if (lrbp->intr_cmd)
2476 dword_0 |= UTP_REQ_DESC_INT_CMD;
2477
2478 /* Prepare crypto related dwords */
2479 ufshcd_prepare_req_desc_hdr_crypto(lrbp, &dword_0, &dword_1, &dword_3);
2480
2481 /* Transfer request descriptor header fields */
2482 req_desc->header.dword_0 = cpu_to_le32(dword_0);
2483 req_desc->header.dword_1 = cpu_to_le32(dword_1);
2484 /*
2485 * assigning invalid value for command status. Controller
2486 * updates OCS on command completion, with the command
2487 * status
2488 */
2489 req_desc->header.dword_2 =
2490 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
2491 req_desc->header.dword_3 = cpu_to_le32(dword_3);
2492
2493 req_desc->prd_table_length = 0;
2494 }
2495
2496 /**
2497 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2498 * for scsi commands
2499 * @lrbp: local reference block pointer
2500 * @upiu_flags: flags
2501 */
2502 static
ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb * lrbp,u8 upiu_flags)2503 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
2504 {
2505 struct scsi_cmnd *cmd = lrbp->cmd;
2506 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2507 unsigned short cdb_len;
2508
2509 /* command descriptor fields */
2510 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2511 UPIU_TRANSACTION_COMMAND, upiu_flags,
2512 lrbp->lun, lrbp->task_tag);
2513 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2514 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2515
2516 /* Total EHS length and Data segment length will be zero */
2517 ucd_req_ptr->header.dword_2 = 0;
2518
2519 ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
2520
2521 cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE);
2522 memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
2523 memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len);
2524
2525 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2526 }
2527
2528 /**
2529 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2530 * for query requsts
2531 * @hba: UFS hba
2532 * @lrbp: local reference block pointer
2533 * @upiu_flags: flags
2534 */
ufshcd_prepare_utp_query_req_upiu(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,u8 upiu_flags)2535 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2536 struct ufshcd_lrb *lrbp, u8 upiu_flags)
2537 {
2538 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2539 struct ufs_query *query = &hba->dev_cmd.query;
2540 u16 len = be16_to_cpu(query->request.upiu_req.length);
2541
2542 /* Query request header */
2543 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2544 UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2545 lrbp->lun, lrbp->task_tag);
2546 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2547 0, query->request.query_func, 0, 0);
2548
2549 /* Data segment length only need for WRITE_DESC */
2550 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2551 ucd_req_ptr->header.dword_2 =
2552 UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2553 else
2554 ucd_req_ptr->header.dword_2 = 0;
2555
2556 /* Copy the Query Request buffer as is */
2557 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2558 QUERY_OSF_SIZE);
2559
2560 /* Copy the Descriptor */
2561 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2562 memcpy(ucd_req_ptr + 1, query->descriptor, len);
2563
2564 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2565 }
2566
ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb * lrbp)2567 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2568 {
2569 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2570
2571 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2572
2573 /* command descriptor fields */
2574 ucd_req_ptr->header.dword_0 =
2575 UPIU_HEADER_DWORD(
2576 UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
2577 /* clear rest of the fields of basic header */
2578 ucd_req_ptr->header.dword_1 = 0;
2579 ucd_req_ptr->header.dword_2 = 0;
2580
2581 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2582 }
2583
2584 /**
2585 * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU)
2586 * for Device Management Purposes
2587 * @hba: per adapter instance
2588 * @lrbp: pointer to local reference block
2589 */
ufshcd_compose_devman_upiu(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2590 static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
2591 struct ufshcd_lrb *lrbp)
2592 {
2593 u8 upiu_flags;
2594 int ret = 0;
2595
2596 if (hba->ufs_version <= ufshci_version(1, 1))
2597 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
2598 else
2599 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2600
2601 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2602 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2603 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2604 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2605 ufshcd_prepare_utp_nop_upiu(lrbp);
2606 else
2607 ret = -EINVAL;
2608
2609 return ret;
2610 }
2611
2612 /**
2613 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2614 * for SCSI Purposes
2615 * @hba: per adapter instance
2616 * @lrbp: pointer to local reference block
2617 */
ufshcd_comp_scsi_upiu(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2618 static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2619 {
2620 u8 upiu_flags;
2621 int ret = 0;
2622
2623 if (hba->ufs_version <= ufshci_version(1, 1))
2624 lrbp->command_type = UTP_CMD_TYPE_SCSI;
2625 else
2626 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2627
2628 if (likely(lrbp->cmd)) {
2629 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2630 lrbp->cmd->sc_data_direction);
2631 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2632 } else {
2633 ret = -EINVAL;
2634 }
2635
2636 return ret;
2637 }
2638
2639 /**
2640 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2641 * @upiu_wlun_id: UPIU W-LUN id
2642 *
2643 * Returns SCSI W-LUN id
2644 */
ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)2645 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2646 {
2647 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2648 }
2649
ufshcd_init_lrb(struct ufs_hba * hba,struct ufshcd_lrb * lrb,int i)2650 static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
2651 {
2652 struct utp_transfer_cmd_desc *cmd_descp = (void *)hba->ucdl_base_addr +
2653 i * sizeof_utp_transfer_cmd_desc(hba);
2654 struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
2655 dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
2656 i * sizeof_utp_transfer_cmd_desc(hba);
2657 u16 response_offset = offsetof(struct utp_transfer_cmd_desc,
2658 response_upiu);
2659 u16 prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table);
2660
2661 lrb->utr_descriptor_ptr = utrdlp + i;
2662 lrb->utrd_dma_addr = hba->utrdl_dma_addr +
2663 i * sizeof(struct utp_transfer_req_desc);
2664 lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp;
2665 lrb->ucd_req_dma_addr = cmd_desc_element_addr;
2666 lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu;
2667 lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
2668 lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table;
2669 lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
2670 }
2671
2672 /**
2673 * ufshcd_queuecommand - main entry point for SCSI requests
2674 * @host: SCSI host pointer
2675 * @cmd: command from SCSI Midlayer
2676 *
2677 * Returns 0 for success, non-zero in case of failure
2678 */
ufshcd_queuecommand(struct Scsi_Host * host,struct scsi_cmnd * cmd)2679 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2680 {
2681 struct ufshcd_lrb *lrbp;
2682 struct ufs_hba *hba;
2683 int tag;
2684 int err = 0;
2685
2686 hba = shost_priv(host);
2687
2688 tag = cmd->request->tag;
2689 if (!ufshcd_valid_tag(hba, tag)) {
2690 dev_err(hba->dev,
2691 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
2692 __func__, tag, cmd, cmd->request);
2693 BUG();
2694 }
2695
2696 if (!down_read_trylock(&hba->clk_scaling_lock))
2697 return SCSI_MLQUEUE_HOST_BUSY;
2698
2699 switch (hba->ufshcd_state) {
2700 case UFSHCD_STATE_OPERATIONAL:
2701 case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL:
2702 break;
2703 case UFSHCD_STATE_EH_SCHEDULED_FATAL:
2704 /*
2705 * pm_runtime_get_sync() is used at error handling preparation
2706 * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's
2707 * PM ops, it can never be finished if we let SCSI layer keep
2708 * retrying it, which gets err handler stuck forever. Neither
2709 * can we let the scsi cmd pass through, because UFS is in bad
2710 * state, the scsi cmd may eventually time out, which will get
2711 * err handler blocked for too long. So, just fail the scsi cmd
2712 * sent from PM ops, err handler can recover PM error anyways.
2713 */
2714 if (hba->pm_op_in_progress) {
2715 hba->force_reset = true;
2716 set_host_byte(cmd, DID_BAD_TARGET);
2717 cmd->scsi_done(cmd);
2718 goto out;
2719 }
2720 fallthrough;
2721 case UFSHCD_STATE_RESET:
2722 err = SCSI_MLQUEUE_HOST_BUSY;
2723 goto out;
2724 case UFSHCD_STATE_ERROR:
2725 set_host_byte(cmd, DID_ERROR);
2726 cmd->scsi_done(cmd);
2727 goto out;
2728 default:
2729 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
2730 __func__, hba->ufshcd_state);
2731 set_host_byte(cmd, DID_BAD_TARGET);
2732 cmd->scsi_done(cmd);
2733 goto out;
2734 }
2735
2736 hba->req_abort_count = 0;
2737
2738 err = ufshcd_hold(hba, true);
2739 if (err) {
2740 err = SCSI_MLQUEUE_HOST_BUSY;
2741 goto out;
2742 }
2743 WARN_ON(ufshcd_is_clkgating_allowed(hba) &&
2744 (hba->clk_gating.state != CLKS_ON));
2745
2746 lrbp = &hba->lrb[tag];
2747 WARN_ON(lrbp->cmd);
2748 lrbp->cmd = cmd;
2749 lrbp->sense_bufflen = UFS_SENSE_SIZE;
2750 lrbp->sense_buffer = cmd->sense_buffer;
2751 lrbp->task_tag = tag;
2752 lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
2753 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
2754
2755 ufshcd_prepare_lrbp_crypto(cmd->request, lrbp);
2756
2757 trace_android_vh_ufs_prepare_command(hba, cmd->request, lrbp, &err);
2758 if (err) {
2759 lrbp->cmd = NULL;
2760 ufshcd_release(hba);
2761 goto out;
2762 }
2763
2764 lrbp->req_abort_skip = false;
2765
2766 err = ufshpb_prep(hba, lrbp);
2767 if (err == -EAGAIN) {
2768 lrbp->cmd = NULL;
2769 ufshcd_release(hba);
2770 goto out;
2771 }
2772
2773 ufshcd_comp_scsi_upiu(hba, lrbp);
2774
2775 err = ufshcd_map_sg(hba, lrbp);
2776 if (err) {
2777 lrbp->cmd = NULL;
2778 ufshcd_release(hba);
2779 goto out;
2780 }
2781 /* Make sure descriptors are ready before ringing the doorbell */
2782 wmb();
2783
2784 ufshcd_send_command(hba, tag);
2785 out:
2786 up_read(&hba->clk_scaling_lock);
2787 return err;
2788 }
2789
ufshcd_compose_dev_cmd(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,enum dev_cmd_type cmd_type,int tag)2790 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2791 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2792 {
2793 lrbp->cmd = NULL;
2794 lrbp->sense_bufflen = 0;
2795 lrbp->sense_buffer = NULL;
2796 lrbp->task_tag = tag;
2797 lrbp->lun = 0; /* device management cmd is not specific to any LUN */
2798 lrbp->intr_cmd = true; /* No interrupt aggregation */
2799 ufshcd_prepare_lrbp_crypto(NULL, lrbp);
2800 hba->dev_cmd.type = cmd_type;
2801
2802 return ufshcd_compose_devman_upiu(hba, lrbp);
2803 }
2804
2805 static int
ufshcd_clear_cmd(struct ufs_hba * hba,int tag)2806 ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
2807 {
2808 int err = 0;
2809 unsigned long flags;
2810 u32 mask = 1 << tag;
2811
2812 /* clear outstanding transaction before retry */
2813 spin_lock_irqsave(hba->host->host_lock, flags);
2814 ufshcd_utrl_clear(hba, tag);
2815 spin_unlock_irqrestore(hba->host->host_lock, flags);
2816
2817 /*
2818 * wait for for h/w to clear corresponding bit in door-bell.
2819 * max. wait is 1 sec.
2820 */
2821 err = ufshcd_wait_for_register(hba,
2822 REG_UTP_TRANSFER_REQ_DOOR_BELL,
2823 mask, ~mask, 1000, 1000);
2824
2825 return err;
2826 }
2827
2828 static int
ufshcd_check_query_response(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2829 ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2830 {
2831 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2832
2833 /* Get the UPIU response */
2834 query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2835 UPIU_RSP_CODE_OFFSET;
2836 return query_res->response;
2837 }
2838
2839 /**
2840 * ufshcd_dev_cmd_completion() - handles device management command responses
2841 * @hba: per adapter instance
2842 * @lrbp: pointer to local reference block
2843 */
2844 static int
ufshcd_dev_cmd_completion(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2845 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2846 {
2847 int resp;
2848 int err = 0;
2849
2850 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
2851 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2852
2853 switch (resp) {
2854 case UPIU_TRANSACTION_NOP_IN:
2855 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2856 err = -EINVAL;
2857 dev_err(hba->dev, "%s: unexpected response %x\n",
2858 __func__, resp);
2859 }
2860 break;
2861 case UPIU_TRANSACTION_QUERY_RSP:
2862 err = ufshcd_check_query_response(hba, lrbp);
2863 if (!err)
2864 err = ufshcd_copy_query_response(hba, lrbp);
2865 break;
2866 case UPIU_TRANSACTION_REJECT_UPIU:
2867 /* TODO: handle Reject UPIU Response */
2868 err = -EPERM;
2869 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2870 __func__);
2871 break;
2872 default:
2873 err = -EINVAL;
2874 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2875 __func__, resp);
2876 break;
2877 }
2878
2879 return err;
2880 }
2881
ufshcd_wait_for_dev_cmd(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,int max_timeout)2882 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2883 struct ufshcd_lrb *lrbp, int max_timeout)
2884 {
2885 int err = 0;
2886 unsigned long time_left;
2887 unsigned long flags;
2888
2889 time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2890 msecs_to_jiffies(max_timeout));
2891
2892 /* Make sure descriptors are ready before ringing the doorbell */
2893 wmb();
2894 spin_lock_irqsave(hba->host->host_lock, flags);
2895 hba->dev_cmd.complete = NULL;
2896 if (likely(time_left)) {
2897 err = ufshcd_get_tr_ocs(lrbp);
2898 if (!err)
2899 err = ufshcd_dev_cmd_completion(hba, lrbp);
2900 }
2901 spin_unlock_irqrestore(hba->host->host_lock, flags);
2902
2903 if (!time_left) {
2904 err = -ETIMEDOUT;
2905 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2906 __func__, lrbp->task_tag);
2907 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
2908 /* successfully cleared the command, retry if needed */
2909 err = -EAGAIN;
2910 /*
2911 * in case of an error, after clearing the doorbell,
2912 * we also need to clear the outstanding_request
2913 * field in hba
2914 */
2915 ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
2916 }
2917
2918 return err;
2919 }
2920
2921 /**
2922 * ufshcd_exec_dev_cmd - API for sending device management requests
2923 * @hba: UFS hba
2924 * @cmd_type: specifies the type (NOP, Query...)
2925 * @timeout: timeout in milliseconds
2926 *
2927 * NOTE: Since there is only one available tag for device management commands,
2928 * it is expected you hold the hba->dev_cmd.lock mutex.
2929 */
ufshcd_exec_dev_cmd(struct ufs_hba * hba,enum dev_cmd_type cmd_type,int timeout)2930 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
2931 enum dev_cmd_type cmd_type, int timeout)
2932 {
2933 DECLARE_COMPLETION_ONSTACK(wait);
2934 const u32 tag = ufs_hba_add_info(hba)->reserved_slot;
2935 struct ufshcd_lrb *lrbp;
2936 int err;
2937
2938 /* Protects use of ufs_hba_add_info(hba)->reserved_slot. */
2939 lockdep_assert_held(&hba->dev_cmd.lock);
2940
2941 down_read(&hba->clk_scaling_lock);
2942
2943 lrbp = &hba->lrb[tag];
2944 WARN_ON(lrbp->cmd);
2945 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
2946 if (unlikely(err))
2947 goto out;
2948
2949 hba->dev_cmd.complete = &wait;
2950
2951 ufshcd_add_query_upiu_trace(hba, tag, "query_send");
2952 /* Make sure descriptors are ready before ringing the doorbell */
2953 wmb();
2954
2955 ufshcd_send_command(hba, tag);
2956 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
2957 ufshcd_add_query_upiu_trace(hba, tag,
2958 err ? "query_complete_err" : "query_complete");
2959
2960 out:
2961 up_read(&hba->clk_scaling_lock);
2962 return err;
2963 }
2964
2965 /**
2966 * ufshcd_init_query() - init the query response and request parameters
2967 * @hba: per-adapter instance
2968 * @request: address of the request pointer to be initialized
2969 * @response: address of the response pointer to be initialized
2970 * @opcode: operation to perform
2971 * @idn: flag idn to access
2972 * @index: LU number to access
2973 * @selector: query/flag/descriptor further identification
2974 */
ufshcd_init_query(struct ufs_hba * hba,struct ufs_query_req ** request,struct ufs_query_res ** response,enum query_opcode opcode,u8 idn,u8 index,u8 selector)2975 static inline void ufshcd_init_query(struct ufs_hba *hba,
2976 struct ufs_query_req **request, struct ufs_query_res **response,
2977 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
2978 {
2979 *request = &hba->dev_cmd.query.request;
2980 *response = &hba->dev_cmd.query.response;
2981 memset(*request, 0, sizeof(struct ufs_query_req));
2982 memset(*response, 0, sizeof(struct ufs_query_res));
2983 (*request)->upiu_req.opcode = opcode;
2984 (*request)->upiu_req.idn = idn;
2985 (*request)->upiu_req.index = index;
2986 (*request)->upiu_req.selector = selector;
2987 }
2988
ufshcd_query_flag_retry(struct ufs_hba * hba,enum query_opcode opcode,enum flag_idn idn,u8 index,bool * flag_res)2989 int ufshcd_query_flag_retry(struct ufs_hba *hba,
2990 enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res)
2991 {
2992 int ret;
2993 int retries;
2994
2995 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
2996 ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res);
2997 if (ret)
2998 dev_dbg(hba->dev,
2999 "%s: failed with error %d, retries %d\n",
3000 __func__, ret, retries);
3001 else
3002 break;
3003 }
3004
3005 if (ret)
3006 dev_err(hba->dev,
3007 "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
3008 __func__, opcode, idn, ret, retries);
3009 return ret;
3010 }
3011 EXPORT_SYMBOL_GPL(ufshcd_query_flag_retry);
3012
3013 /**
3014 * ufshcd_query_flag() - API function for sending flag query requests
3015 * @hba: per-adapter instance
3016 * @opcode: flag query to perform
3017 * @idn: flag idn to access
3018 * @index: flag index to access
3019 * @flag_res: the flag value after the query request completes
3020 *
3021 * Returns 0 for success, non-zero in case of failure
3022 */
ufshcd_query_flag(struct ufs_hba * hba,enum query_opcode opcode,enum flag_idn idn,u8 index,bool * flag_res)3023 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
3024 enum flag_idn idn, u8 index, bool *flag_res)
3025 {
3026 struct ufs_query_req *request = NULL;
3027 struct ufs_query_res *response = NULL;
3028 int err, selector = 0;
3029 int timeout = QUERY_REQ_TIMEOUT;
3030
3031 BUG_ON(!hba);
3032
3033 ufshcd_hold(hba, false);
3034 mutex_lock(&hba->dev_cmd.lock);
3035 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3036 selector);
3037
3038 switch (opcode) {
3039 case UPIU_QUERY_OPCODE_SET_FLAG:
3040 case UPIU_QUERY_OPCODE_CLEAR_FLAG:
3041 case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
3042 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3043 break;
3044 case UPIU_QUERY_OPCODE_READ_FLAG:
3045 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3046 if (!flag_res) {
3047 /* No dummy reads */
3048 dev_err(hba->dev, "%s: Invalid argument for read request\n",
3049 __func__);
3050 err = -EINVAL;
3051 goto out_unlock;
3052 }
3053 break;
3054 default:
3055 dev_err(hba->dev,
3056 "%s: Expected query flag opcode but got = %d\n",
3057 __func__, opcode);
3058 err = -EINVAL;
3059 goto out_unlock;
3060 }
3061
3062 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
3063
3064 if (err) {
3065 dev_err(hba->dev,
3066 "%s: Sending flag query for idn %d failed, err = %d\n",
3067 __func__, idn, err);
3068 goto out_unlock;
3069 }
3070
3071 if (flag_res)
3072 *flag_res = (be32_to_cpu(response->upiu_res.value) &
3073 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
3074
3075 out_unlock:
3076 mutex_unlock(&hba->dev_cmd.lock);
3077 ufshcd_release(hba);
3078 return err;
3079 }
3080 EXPORT_SYMBOL_GPL(ufshcd_query_flag);
3081
3082 /**
3083 * ufshcd_query_attr - API function for sending attribute requests
3084 * @hba: per-adapter instance
3085 * @opcode: attribute opcode
3086 * @idn: attribute idn to access
3087 * @index: index field
3088 * @selector: selector field
3089 * @attr_val: the attribute value after the query request completes
3090 *
3091 * Returns 0 for success, non-zero in case of failure
3092 */
ufshcd_query_attr(struct ufs_hba * hba,enum query_opcode opcode,enum attr_idn idn,u8 index,u8 selector,u32 * attr_val)3093 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
3094 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
3095 {
3096 struct ufs_query_req *request = NULL;
3097 struct ufs_query_res *response = NULL;
3098 int err;
3099
3100 BUG_ON(!hba);
3101
3102 if (!attr_val) {
3103 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
3104 __func__, opcode);
3105 return -EINVAL;
3106 }
3107
3108 ufshcd_hold(hba, false);
3109
3110 mutex_lock(&hba->dev_cmd.lock);
3111 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3112 selector);
3113
3114 switch (opcode) {
3115 case UPIU_QUERY_OPCODE_WRITE_ATTR:
3116 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3117 request->upiu_req.value = cpu_to_be32(*attr_val);
3118 break;
3119 case UPIU_QUERY_OPCODE_READ_ATTR:
3120 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3121 break;
3122 default:
3123 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
3124 __func__, opcode);
3125 err = -EINVAL;
3126 goto out_unlock;
3127 }
3128
3129 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3130
3131 if (err) {
3132 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3133 __func__, opcode, idn, index, err);
3134 goto out_unlock;
3135 }
3136
3137 *attr_val = be32_to_cpu(response->upiu_res.value);
3138
3139 out_unlock:
3140 mutex_unlock(&hba->dev_cmd.lock);
3141 ufshcd_release(hba);
3142 return err;
3143 }
3144 EXPORT_SYMBOL_GPL(ufshcd_query_attr);
3145
3146 /**
3147 * ufshcd_query_attr_retry() - API function for sending query
3148 * attribute with retries
3149 * @hba: per-adapter instance
3150 * @opcode: attribute opcode
3151 * @idn: attribute idn to access
3152 * @index: index field
3153 * @selector: selector field
3154 * @attr_val: the attribute value after the query request
3155 * completes
3156 *
3157 * Returns 0 for success, non-zero in case of failure
3158 */
ufshcd_query_attr_retry(struct ufs_hba * hba,enum query_opcode opcode,enum attr_idn idn,u8 index,u8 selector,u32 * attr_val)3159 int ufshcd_query_attr_retry(struct ufs_hba *hba,
3160 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
3161 u32 *attr_val)
3162 {
3163 int ret = 0;
3164 u32 retries;
3165
3166 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3167 ret = ufshcd_query_attr(hba, opcode, idn, index,
3168 selector, attr_val);
3169 if (ret)
3170 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
3171 __func__, ret, retries);
3172 else
3173 break;
3174 }
3175
3176 if (ret)
3177 dev_err(hba->dev,
3178 "%s: query attribute, idn %d, failed with error %d after %d retires\n",
3179 __func__, idn, ret, QUERY_REQ_RETRIES);
3180 return ret;
3181 }
3182 EXPORT_SYMBOL_GPL(ufshcd_query_attr_retry);
3183
__ufshcd_query_descriptor(struct ufs_hba * hba,enum query_opcode opcode,enum desc_idn idn,u8 index,u8 selector,u8 * desc_buf,int * buf_len)3184 static int __ufshcd_query_descriptor(struct ufs_hba *hba,
3185 enum query_opcode opcode, enum desc_idn idn, u8 index,
3186 u8 selector, u8 *desc_buf, int *buf_len)
3187 {
3188 struct ufs_query_req *request = NULL;
3189 struct ufs_query_res *response = NULL;
3190 int err;
3191
3192 BUG_ON(!hba);
3193
3194 if (!desc_buf) {
3195 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
3196 __func__, opcode);
3197 return -EINVAL;
3198 }
3199
3200 if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
3201 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
3202 __func__, *buf_len);
3203 return -EINVAL;
3204 }
3205
3206 ufshcd_hold(hba, false);
3207
3208 mutex_lock(&hba->dev_cmd.lock);
3209 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3210 selector);
3211 hba->dev_cmd.query.descriptor = desc_buf;
3212 request->upiu_req.length = cpu_to_be16(*buf_len);
3213
3214 switch (opcode) {
3215 case UPIU_QUERY_OPCODE_WRITE_DESC:
3216 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3217 break;
3218 case UPIU_QUERY_OPCODE_READ_DESC:
3219 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3220 break;
3221 default:
3222 dev_err(hba->dev,
3223 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
3224 __func__, opcode);
3225 err = -EINVAL;
3226 goto out_unlock;
3227 }
3228
3229 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3230
3231 if (err) {
3232 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3233 __func__, opcode, idn, index, err);
3234 goto out_unlock;
3235 }
3236
3237 *buf_len = be16_to_cpu(response->upiu_res.length);
3238
3239 out_unlock:
3240 hba->dev_cmd.query.descriptor = NULL;
3241 mutex_unlock(&hba->dev_cmd.lock);
3242 ufshcd_release(hba);
3243 return err;
3244 }
3245
3246 /**
3247 * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3248 * @hba: per-adapter instance
3249 * @opcode: attribute opcode
3250 * @idn: attribute idn to access
3251 * @index: index field
3252 * @selector: selector field
3253 * @desc_buf: the buffer that contains the descriptor
3254 * @buf_len: length parameter passed to the device
3255 *
3256 * Returns 0 for success, non-zero in case of failure.
3257 * The buf_len parameter will contain, on return, the length parameter
3258 * received on the response.
3259 */
ufshcd_query_descriptor_retry(struct ufs_hba * hba,enum query_opcode opcode,enum desc_idn idn,u8 index,u8 selector,u8 * desc_buf,int * buf_len)3260 int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3261 enum query_opcode opcode,
3262 enum desc_idn idn, u8 index,
3263 u8 selector,
3264 u8 *desc_buf, int *buf_len)
3265 {
3266 int err;
3267 int retries;
3268
3269 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3270 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3271 selector, desc_buf, buf_len);
3272 if (!err || err == -EINVAL)
3273 break;
3274 }
3275
3276 return err;
3277 }
3278 EXPORT_SYMBOL_GPL(ufshcd_query_descriptor_retry);
3279
3280 /**
3281 * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3282 * @hba: Pointer to adapter instance
3283 * @desc_id: descriptor idn value
3284 * @desc_len: mapped desc length (out)
3285 */
ufshcd_map_desc_id_to_length(struct ufs_hba * hba,enum desc_idn desc_id,int * desc_len)3286 void ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
3287 int *desc_len)
3288 {
3289 if (desc_id >= QUERY_DESC_IDN_MAX || desc_id == QUERY_DESC_IDN_RFU_0 ||
3290 desc_id == QUERY_DESC_IDN_RFU_1)
3291 *desc_len = 0;
3292 else
3293 *desc_len = hba->desc_size[desc_id];
3294 }
3295 EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3296
ufshcd_update_desc_length(struct ufs_hba * hba,enum desc_idn desc_id,int desc_index,unsigned char desc_len)3297 static void ufshcd_update_desc_length(struct ufs_hba *hba,
3298 enum desc_idn desc_id, int desc_index,
3299 unsigned char desc_len)
3300 {
3301 if (hba->desc_size[desc_id] == QUERY_DESC_MAX_SIZE &&
3302 desc_id != QUERY_DESC_IDN_STRING && desc_index != UFS_RPMB_UNIT)
3303 /* For UFS 3.1, the normal unit descriptor is 10 bytes larger
3304 * than the RPMB unit, however, both descriptors share the same
3305 * desc_idn, to cover both unit descriptors with one length, we
3306 * choose the normal unit descriptor length by desc_index.
3307 */
3308 hba->desc_size[desc_id] = desc_len;
3309 }
3310
3311 /**
3312 * ufshcd_read_desc_param - read the specified descriptor parameter
3313 * @hba: Pointer to adapter instance
3314 * @desc_id: descriptor idn value
3315 * @desc_index: descriptor index
3316 * @param_offset: offset of the parameter to read
3317 * @param_read_buf: pointer to buffer where parameter would be read
3318 * @param_size: sizeof(param_read_buf)
3319 *
3320 * Return 0 in case of success, non-zero otherwise
3321 */
ufshcd_read_desc_param(struct ufs_hba * hba,enum desc_idn desc_id,int desc_index,u8 param_offset,u8 * param_read_buf,u8 param_size)3322 int ufshcd_read_desc_param(struct ufs_hba *hba,
3323 enum desc_idn desc_id,
3324 int desc_index,
3325 u8 param_offset,
3326 u8 *param_read_buf,
3327 u8 param_size)
3328 {
3329 int ret;
3330 u8 *desc_buf;
3331 int buff_len;
3332 bool is_kmalloc = true;
3333
3334 /* Safety check */
3335 if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
3336 return -EINVAL;
3337
3338 /* Get the length of descriptor */
3339 ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
3340 if (!buff_len) {
3341 dev_err(hba->dev, "%s: Failed to get desc length\n", __func__);
3342 return -EINVAL;
3343 }
3344
3345 if (param_offset >= buff_len) {
3346 dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n",
3347 __func__, param_offset, desc_id, buff_len);
3348 return -EINVAL;
3349 }
3350
3351 /* Check whether we need temp memory */
3352 if (param_offset != 0 || param_size < buff_len) {
3353 desc_buf = kzalloc(buff_len, GFP_KERNEL);
3354 if (!desc_buf)
3355 return -ENOMEM;
3356 } else {
3357 desc_buf = param_read_buf;
3358 is_kmalloc = false;
3359 }
3360
3361 /* Request for full descriptor */
3362 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3363 desc_id, desc_index, 0,
3364 desc_buf, &buff_len);
3365
3366 if (ret) {
3367 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n",
3368 __func__, desc_id, desc_index, param_offset, ret);
3369 goto out;
3370 }
3371
3372 /* Sanity check */
3373 if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3374 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n",
3375 __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3376 ret = -EINVAL;
3377 goto out;
3378 }
3379
3380 /* Update descriptor length */
3381 buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET];
3382 ufshcd_update_desc_length(hba, desc_id, desc_index, buff_len);
3383
3384 if (is_kmalloc) {
3385 /* Make sure we don't copy more data than available */
3386 if (param_offset >= buff_len)
3387 ret = -EINVAL;
3388 else
3389 memcpy(param_read_buf, &desc_buf[param_offset],
3390 min_t(u32, param_size, buff_len - param_offset));
3391 }
3392 out:
3393 if (is_kmalloc)
3394 kfree(desc_buf);
3395 return ret;
3396 }
3397 EXPORT_SYMBOL_GPL(ufshcd_read_desc_param);
3398
3399 /**
3400 * struct uc_string_id - unicode string
3401 *
3402 * @len: size of this descriptor inclusive
3403 * @type: descriptor type
3404 * @uc: unicode string character
3405 */
3406 struct uc_string_id {
3407 u8 len;
3408 u8 type;
3409 wchar_t uc[];
3410 } __packed;
3411
3412 /* replace non-printable or non-ASCII characters with spaces */
ufshcd_remove_non_printable(u8 ch)3413 static inline char ufshcd_remove_non_printable(u8 ch)
3414 {
3415 return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3416 }
3417
3418 /**
3419 * ufshcd_read_string_desc - read string descriptor
3420 * @hba: pointer to adapter instance
3421 * @desc_index: descriptor index
3422 * @buf: pointer to buffer where descriptor would be read,
3423 * the caller should free the memory.
3424 * @ascii: if true convert from unicode to ascii characters
3425 * null terminated string.
3426 *
3427 * Return:
3428 * * string size on success.
3429 * * -ENOMEM: on allocation failure
3430 * * -EINVAL: on a wrong parameter
3431 */
ufshcd_read_string_desc(struct ufs_hba * hba,u8 desc_index,u8 ** buf,bool ascii)3432 int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3433 u8 **buf, bool ascii)
3434 {
3435 struct uc_string_id *uc_str;
3436 u8 *str;
3437 int ret;
3438
3439 if (!buf)
3440 return -EINVAL;
3441
3442 uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3443 if (!uc_str)
3444 return -ENOMEM;
3445
3446 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0,
3447 (u8 *)uc_str, QUERY_DESC_MAX_SIZE);
3448 if (ret < 0) {
3449 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3450 QUERY_REQ_RETRIES, ret);
3451 str = NULL;
3452 goto out;
3453 }
3454
3455 if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3456 dev_dbg(hba->dev, "String Desc is of zero length\n");
3457 str = NULL;
3458 ret = 0;
3459 goto out;
3460 }
3461
3462 if (ascii) {
3463 ssize_t ascii_len;
3464 int i;
3465 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3466 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3467 str = kzalloc(ascii_len, GFP_KERNEL);
3468 if (!str) {
3469 ret = -ENOMEM;
3470 goto out;
3471 }
3472
3473 /*
3474 * the descriptor contains string in UTF16 format
3475 * we need to convert to utf-8 so it can be displayed
3476 */
3477 ret = utf16s_to_utf8s(uc_str->uc,
3478 uc_str->len - QUERY_DESC_HDR_SIZE,
3479 UTF16_BIG_ENDIAN, str, ascii_len);
3480
3481 /* replace non-printable or non-ASCII characters with spaces */
3482 for (i = 0; i < ret; i++)
3483 str[i] = ufshcd_remove_non_printable(str[i]);
3484
3485 str[ret++] = '\0';
3486
3487 } else {
3488 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
3489 if (!str) {
3490 ret = -ENOMEM;
3491 goto out;
3492 }
3493 ret = uc_str->len;
3494 }
3495 out:
3496 *buf = str;
3497 kfree(uc_str);
3498 return ret;
3499 }
3500
3501 /**
3502 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3503 * @hba: Pointer to adapter instance
3504 * @lun: lun id
3505 * @param_offset: offset of the parameter to read
3506 * @param_read_buf: pointer to buffer where parameter would be read
3507 * @param_size: sizeof(param_read_buf)
3508 *
3509 * Return 0 in case of success, non-zero otherwise
3510 */
ufshcd_read_unit_desc_param(struct ufs_hba * hba,int lun,enum unit_desc_param param_offset,u8 * param_read_buf,u32 param_size)3511 static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3512 int lun,
3513 enum unit_desc_param param_offset,
3514 u8 *param_read_buf,
3515 u32 param_size)
3516 {
3517 /*
3518 * Unit descriptors are only available for general purpose LUs (LUN id
3519 * from 0 to 7) and RPMB Well known LU.
3520 */
3521 if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun, param_offset))
3522 return -EOPNOTSUPP;
3523
3524 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3525 param_offset, param_read_buf, param_size);
3526 }
3527
ufshcd_get_ref_clk_gating_wait(struct ufs_hba * hba)3528 static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba)
3529 {
3530 int err = 0;
3531 u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3532
3533 if (hba->dev_info.wspecversion >= 0x300) {
3534 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3535 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0,
3536 &gating_wait);
3537 if (err)
3538 dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n",
3539 err, gating_wait);
3540
3541 if (gating_wait == 0) {
3542 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3543 dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n",
3544 gating_wait);
3545 }
3546
3547 hba->dev_info.clk_gating_wait_us = gating_wait;
3548 }
3549
3550 return err;
3551 }
3552
3553 /**
3554 * ufshcd_memory_alloc - allocate memory for host memory space data structures
3555 * @hba: per adapter instance
3556 *
3557 * 1. Allocate DMA memory for Command Descriptor array
3558 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3559 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3560 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3561 * (UTMRDL)
3562 * 4. Allocate memory for local reference block(lrb).
3563 *
3564 * Returns 0 for success, non-zero in case of failure
3565 */
ufshcd_memory_alloc(struct ufs_hba * hba)3566 static int ufshcd_memory_alloc(struct ufs_hba *hba)
3567 {
3568 size_t utmrdl_size, utrdl_size, ucdl_size;
3569
3570 /* Allocate memory for UTP command descriptors */
3571 ucdl_size = (sizeof_utp_transfer_cmd_desc(hba) * hba->nutrs);
3572 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3573 ucdl_size,
3574 &hba->ucdl_dma_addr,
3575 GFP_KERNEL);
3576
3577 /*
3578 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3579 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3580 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3581 * be aligned to 128 bytes as well
3582 */
3583 if (!hba->ucdl_base_addr ||
3584 WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
3585 dev_err(hba->dev,
3586 "Command Descriptor Memory allocation failed\n");
3587 goto out;
3588 }
3589
3590 /*
3591 * Allocate memory for UTP Transfer descriptors
3592 * UFSHCI requires 1024 byte alignment of UTRD
3593 */
3594 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
3595 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3596 utrdl_size,
3597 &hba->utrdl_dma_addr,
3598 GFP_KERNEL);
3599 if (!hba->utrdl_base_addr ||
3600 WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
3601 dev_err(hba->dev,
3602 "Transfer Descriptor Memory allocation failed\n");
3603 goto out;
3604 }
3605
3606 /*
3607 * Allocate memory for UTP Task Management descriptors
3608 * UFSHCI requires 1024 byte alignment of UTMRD
3609 */
3610 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
3611 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3612 utmrdl_size,
3613 &hba->utmrdl_dma_addr,
3614 GFP_KERNEL);
3615 if (!hba->utmrdl_base_addr ||
3616 WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
3617 dev_err(hba->dev,
3618 "Task Management Descriptor Memory allocation failed\n");
3619 goto out;
3620 }
3621
3622 /* Allocate memory for local reference block */
3623 hba->lrb = devm_kcalloc(hba->dev,
3624 hba->nutrs, sizeof(struct ufshcd_lrb),
3625 GFP_KERNEL);
3626 if (!hba->lrb) {
3627 dev_err(hba->dev, "LRB Memory allocation failed\n");
3628 goto out;
3629 }
3630 return 0;
3631 out:
3632 return -ENOMEM;
3633 }
3634
3635 /**
3636 * ufshcd_host_memory_configure - configure local reference block with
3637 * memory offsets
3638 * @hba: per adapter instance
3639 *
3640 * Configure Host memory space
3641 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3642 * address.
3643 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3644 * and PRDT offset.
3645 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3646 * into local reference block.
3647 */
ufshcd_host_memory_configure(struct ufs_hba * hba)3648 static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3649 {
3650 struct utp_transfer_req_desc *utrdlp;
3651 dma_addr_t cmd_desc_dma_addr;
3652 dma_addr_t cmd_desc_element_addr;
3653 u16 response_offset;
3654 u16 prdt_offset;
3655 int cmd_desc_size;
3656 int i;
3657
3658 utrdlp = hba->utrdl_base_addr;
3659
3660 response_offset =
3661 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3662 prdt_offset =
3663 offsetof(struct utp_transfer_cmd_desc, prd_table);
3664
3665 cmd_desc_size = sizeof_utp_transfer_cmd_desc(hba);
3666 cmd_desc_dma_addr = hba->ucdl_dma_addr;
3667
3668 for (i = 0; i < hba->nutrs; i++) {
3669 /* Configure UTRD with command descriptor base address */
3670 cmd_desc_element_addr =
3671 (cmd_desc_dma_addr + (cmd_desc_size * i));
3672 utrdlp[i].command_desc_base_addr_lo =
3673 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
3674 utrdlp[i].command_desc_base_addr_hi =
3675 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
3676
3677 /* Response upiu and prdt offset should be in double words */
3678 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3679 utrdlp[i].response_upiu_offset =
3680 cpu_to_le16(response_offset);
3681 utrdlp[i].prd_table_offset =
3682 cpu_to_le16(prdt_offset);
3683 utrdlp[i].response_upiu_length =
3684 cpu_to_le16(ALIGNED_UPIU_SIZE);
3685 } else {
3686 utrdlp[i].response_upiu_offset =
3687 cpu_to_le16(response_offset >> 2);
3688 utrdlp[i].prd_table_offset =
3689 cpu_to_le16(prdt_offset >> 2);
3690 utrdlp[i].response_upiu_length =
3691 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
3692 }
3693
3694 ufshcd_init_lrb(hba, &hba->lrb[i], i);
3695 }
3696 }
3697
3698 /**
3699 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3700 * @hba: per adapter instance
3701 *
3702 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3703 * in order to initialize the Unipro link startup procedure.
3704 * Once the Unipro links are up, the device connected to the controller
3705 * is detected.
3706 *
3707 * Returns 0 on success, non-zero value on failure
3708 */
ufshcd_dme_link_startup(struct ufs_hba * hba)3709 static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3710 {
3711 struct uic_command uic_cmd = {0};
3712 int ret;
3713
3714 uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
3715
3716 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3717 if (ret)
3718 dev_dbg(hba->dev,
3719 "dme-link-startup: error code %d\n", ret);
3720 return ret;
3721 }
3722 /**
3723 * ufshcd_dme_reset - UIC command for DME_RESET
3724 * @hba: per adapter instance
3725 *
3726 * DME_RESET command is issued in order to reset UniPro stack.
3727 * This function now deals with cold reset.
3728 *
3729 * Returns 0 on success, non-zero value on failure
3730 */
ufshcd_dme_reset(struct ufs_hba * hba)3731 static int ufshcd_dme_reset(struct ufs_hba *hba)
3732 {
3733 struct uic_command uic_cmd = {0};
3734 int ret;
3735
3736 uic_cmd.command = UIC_CMD_DME_RESET;
3737
3738 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3739 if (ret)
3740 dev_err(hba->dev,
3741 "dme-reset: error code %d\n", ret);
3742
3743 return ret;
3744 }
3745
3746 /**
3747 * ufshcd_dme_enable - UIC command for DME_ENABLE
3748 * @hba: per adapter instance
3749 *
3750 * DME_ENABLE command is issued in order to enable UniPro stack.
3751 *
3752 * Returns 0 on success, non-zero value on failure
3753 */
ufshcd_dme_enable(struct ufs_hba * hba)3754 static int ufshcd_dme_enable(struct ufs_hba *hba)
3755 {
3756 struct uic_command uic_cmd = {0};
3757 int ret;
3758
3759 uic_cmd.command = UIC_CMD_DME_ENABLE;
3760
3761 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3762 if (ret)
3763 dev_err(hba->dev,
3764 "dme-enable: error code %d\n", ret);
3765
3766 return ret;
3767 }
3768
ufshcd_add_delay_before_dme_cmd(struct ufs_hba * hba)3769 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
3770 {
3771 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
3772 unsigned long min_sleep_time_us;
3773
3774 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3775 return;
3776
3777 /*
3778 * last_dme_cmd_tstamp will be 0 only for 1st call to
3779 * this function
3780 */
3781 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3782 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
3783 } else {
3784 unsigned long delta =
3785 (unsigned long) ktime_to_us(
3786 ktime_sub(ktime_get(),
3787 hba->last_dme_cmd_tstamp));
3788
3789 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
3790 min_sleep_time_us =
3791 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3792 else
3793 return; /* no more delay required */
3794 }
3795
3796 /* allow sleep for extra 50us if needed */
3797 usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
3798 }
3799
3800 /**
3801 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3802 * @hba: per adapter instance
3803 * @attr_sel: uic command argument1
3804 * @attr_set: attribute set type as uic command argument2
3805 * @mib_val: setting value as uic command argument3
3806 * @peer: indicate whether peer or local
3807 *
3808 * Returns 0 on success, non-zero value on failure
3809 */
ufshcd_dme_set_attr(struct ufs_hba * hba,u32 attr_sel,u8 attr_set,u32 mib_val,u8 peer)3810 int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
3811 u8 attr_set, u32 mib_val, u8 peer)
3812 {
3813 struct uic_command uic_cmd = {0};
3814 static const char *const action[] = {
3815 "dme-set",
3816 "dme-peer-set"
3817 };
3818 const char *set = action[!!peer];
3819 int ret;
3820 int retries = UFS_UIC_COMMAND_RETRIES;
3821
3822 uic_cmd.command = peer ?
3823 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
3824 uic_cmd.argument1 = attr_sel;
3825 uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
3826 uic_cmd.argument3 = mib_val;
3827
3828 do {
3829 /* for peer attributes we retry upon failure */
3830 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3831 if (ret)
3832 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
3833 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
3834 } while (ret && peer && --retries);
3835
3836 if (ret)
3837 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
3838 set, UIC_GET_ATTR_ID(attr_sel), mib_val,
3839 UFS_UIC_COMMAND_RETRIES - retries);
3840
3841 return ret;
3842 }
3843 EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
3844
3845 /**
3846 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3847 * @hba: per adapter instance
3848 * @attr_sel: uic command argument1
3849 * @mib_val: the value of the attribute as returned by the UIC command
3850 * @peer: indicate whether peer or local
3851 *
3852 * Returns 0 on success, non-zero value on failure
3853 */
ufshcd_dme_get_attr(struct ufs_hba * hba,u32 attr_sel,u32 * mib_val,u8 peer)3854 int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
3855 u32 *mib_val, u8 peer)
3856 {
3857 struct uic_command uic_cmd = {0};
3858 static const char *const action[] = {
3859 "dme-get",
3860 "dme-peer-get"
3861 };
3862 const char *get = action[!!peer];
3863 int ret;
3864 int retries = UFS_UIC_COMMAND_RETRIES;
3865 struct ufs_pa_layer_attr orig_pwr_info;
3866 struct ufs_pa_layer_attr temp_pwr_info;
3867 bool pwr_mode_change = false;
3868
3869 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
3870 orig_pwr_info = hba->pwr_info;
3871 temp_pwr_info = orig_pwr_info;
3872
3873 if (orig_pwr_info.pwr_tx == FAST_MODE ||
3874 orig_pwr_info.pwr_rx == FAST_MODE) {
3875 temp_pwr_info.pwr_tx = FASTAUTO_MODE;
3876 temp_pwr_info.pwr_rx = FASTAUTO_MODE;
3877 pwr_mode_change = true;
3878 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
3879 orig_pwr_info.pwr_rx == SLOW_MODE) {
3880 temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
3881 temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
3882 pwr_mode_change = true;
3883 }
3884 if (pwr_mode_change) {
3885 ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
3886 if (ret)
3887 goto out;
3888 }
3889 }
3890
3891 uic_cmd.command = peer ?
3892 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
3893 uic_cmd.argument1 = attr_sel;
3894
3895 do {
3896 /* for peer attributes we retry upon failure */
3897 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3898 if (ret)
3899 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
3900 get, UIC_GET_ATTR_ID(attr_sel), ret);
3901 } while (ret && peer && --retries);
3902
3903 if (ret)
3904 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
3905 get, UIC_GET_ATTR_ID(attr_sel),
3906 UFS_UIC_COMMAND_RETRIES - retries);
3907
3908 if (mib_val && !ret)
3909 *mib_val = uic_cmd.argument3;
3910
3911 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
3912 && pwr_mode_change)
3913 ufshcd_change_power_mode(hba, &orig_pwr_info);
3914 out:
3915 return ret;
3916 }
3917 EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
3918
3919 /**
3920 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3921 * state) and waits for it to take effect.
3922 *
3923 * @hba: per adapter instance
3924 * @cmd: UIC command to execute
3925 *
3926 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3927 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3928 * and device UniPro link and hence it's final completion would be indicated by
3929 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3930 * addition to normal UIC command completion Status (UCCS). This function only
3931 * returns after the relevant status bits indicate the completion.
3932 *
3933 * Returns 0 on success, non-zero value on failure
3934 */
ufshcd_uic_pwr_ctrl(struct ufs_hba * hba,struct uic_command * cmd)3935 static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
3936 {
3937 DECLARE_COMPLETION_ONSTACK(uic_async_done);
3938 unsigned long flags;
3939 u8 status;
3940 int ret;
3941 bool reenable_intr = false;
3942
3943 mutex_lock(&hba->uic_cmd_mutex);
3944 ufshcd_add_delay_before_dme_cmd(hba);
3945
3946 spin_lock_irqsave(hba->host->host_lock, flags);
3947 if (ufshcd_is_link_broken(hba)) {
3948 ret = -ENOLINK;
3949 goto out_unlock;
3950 }
3951 hba->uic_async_done = &uic_async_done;
3952 if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
3953 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
3954 /*
3955 * Make sure UIC command completion interrupt is disabled before
3956 * issuing UIC command.
3957 */
3958 wmb();
3959 reenable_intr = true;
3960 }
3961 ret = __ufshcd_send_uic_cmd(hba, cmd, false);
3962 spin_unlock_irqrestore(hba->host->host_lock, flags);
3963 if (ret) {
3964 dev_err(hba->dev,
3965 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
3966 cmd->command, cmd->argument3, ret);
3967 goto out;
3968 }
3969
3970 if (!wait_for_completion_timeout(hba->uic_async_done,
3971 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
3972 dev_err(hba->dev,
3973 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
3974 cmd->command, cmd->argument3);
3975
3976 if (!cmd->cmd_active) {
3977 dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n",
3978 __func__);
3979 goto check_upmcrs;
3980 }
3981
3982 ret = -ETIMEDOUT;
3983 goto out;
3984 }
3985
3986 check_upmcrs:
3987 status = ufshcd_get_upmcrs(hba);
3988 if (status != PWR_LOCAL) {
3989 dev_err(hba->dev,
3990 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
3991 cmd->command, status);
3992 ret = (status != PWR_OK) ? status : -1;
3993 }
3994 out:
3995 if (ret) {
3996 ufshcd_print_host_state(hba);
3997 ufshcd_print_pwr_info(hba);
3998 ufshcd_print_evt_hist(hba);
3999 }
4000
4001 spin_lock_irqsave(hba->host->host_lock, flags);
4002 hba->active_uic_cmd = NULL;
4003 hba->uic_async_done = NULL;
4004 if (reenable_intr)
4005 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
4006 if (ret) {
4007 dev_err(hba->dev,
4008 "%s: Changing link power status failed (%d). Scheduling error handler\n",
4009 __func__, ret);
4010 ufshcd_set_link_broken(hba);
4011 ufshcd_schedule_eh_work(hba);
4012 }
4013 out_unlock:
4014 spin_unlock_irqrestore(hba->host->host_lock, flags);
4015 mutex_unlock(&hba->uic_cmd_mutex);
4016
4017 return ret;
4018 }
4019
4020 /**
4021 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
4022 * using DME_SET primitives.
4023 * @hba: per adapter instance
4024 * @mode: powr mode value
4025 *
4026 * Returns 0 on success, non-zero value on failure
4027 */
ufshcd_uic_change_pwr_mode(struct ufs_hba * hba,u8 mode)4028 static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
4029 {
4030 struct uic_command uic_cmd = {0};
4031 int ret;
4032
4033 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
4034 ret = ufshcd_dme_set(hba,
4035 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
4036 if (ret) {
4037 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
4038 __func__, ret);
4039 goto out;
4040 }
4041 }
4042
4043 uic_cmd.command = UIC_CMD_DME_SET;
4044 uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
4045 uic_cmd.argument3 = mode;
4046 ufshcd_hold(hba, false);
4047 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4048 ufshcd_release(hba);
4049
4050 out:
4051 return ret;
4052 }
4053
ufshcd_link_recovery(struct ufs_hba * hba)4054 int ufshcd_link_recovery(struct ufs_hba *hba)
4055 {
4056 int ret;
4057 unsigned long flags;
4058
4059 spin_lock_irqsave(hba->host->host_lock, flags);
4060 hba->ufshcd_state = UFSHCD_STATE_RESET;
4061 ufshcd_set_eh_in_progress(hba);
4062 spin_unlock_irqrestore(hba->host->host_lock, flags);
4063
4064 /* Reset the attached device */
4065 ufshcd_vops_device_reset(hba);
4066
4067 ret = ufshcd_host_reset_and_restore(hba);
4068
4069 spin_lock_irqsave(hba->host->host_lock, flags);
4070 if (ret)
4071 hba->ufshcd_state = UFSHCD_STATE_ERROR;
4072 ufshcd_clear_eh_in_progress(hba);
4073 spin_unlock_irqrestore(hba->host->host_lock, flags);
4074
4075 if (ret)
4076 dev_err(hba->dev, "%s: link recovery failed, err %d",
4077 __func__, ret);
4078
4079 return ret;
4080 }
4081 EXPORT_SYMBOL_GPL(ufshcd_link_recovery);
4082
ufshcd_uic_hibern8_enter(struct ufs_hba * hba)4083 int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
4084 {
4085 int ret;
4086 struct uic_command uic_cmd = {0};
4087 ktime_t start = ktime_get();
4088
4089 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
4090
4091 uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
4092 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4093 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
4094 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4095
4096 if (ret)
4097 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
4098 __func__, ret);
4099 else
4100 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
4101 POST_CHANGE);
4102
4103 return ret;
4104 }
4105 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_enter);
4106
ufshcd_uic_hibern8_exit(struct ufs_hba * hba)4107 int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
4108 {
4109 struct uic_command uic_cmd = {0};
4110 int ret;
4111 ktime_t start = ktime_get();
4112
4113 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
4114
4115 uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
4116 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4117 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
4118 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4119
4120 if (ret) {
4121 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
4122 __func__, ret);
4123 } else {
4124 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
4125 POST_CHANGE);
4126 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_get();
4127 hba->ufs_stats.hibern8_exit_cnt++;
4128 }
4129
4130 return ret;
4131 }
4132 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit);
4133
ufshcd_auto_hibern8_update(struct ufs_hba * hba,u32 ahit)4134 void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit)
4135 {
4136 unsigned long flags;
4137 bool update = false;
4138
4139 if (!ufshcd_is_auto_hibern8_supported(hba))
4140 return;
4141
4142 spin_lock_irqsave(hba->host->host_lock, flags);
4143 if (hba->ahit != ahit) {
4144 hba->ahit = ahit;
4145 update = true;
4146 }
4147 spin_unlock_irqrestore(hba->host->host_lock, flags);
4148
4149 if (update && !pm_runtime_suspended(hba->dev)) {
4150 pm_runtime_get_sync(hba->dev);
4151 ufshcd_hold(hba, false);
4152 ufshcd_auto_hibern8_enable(hba);
4153 ufshcd_release(hba);
4154 pm_runtime_put(hba->dev);
4155 }
4156 }
4157 EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update);
4158
ufshcd_auto_hibern8_enable(struct ufs_hba * hba)4159 void ufshcd_auto_hibern8_enable(struct ufs_hba *hba)
4160 {
4161 unsigned long flags;
4162
4163 if (!ufshcd_is_auto_hibern8_supported(hba))
4164 return;
4165
4166 spin_lock_irqsave(hba->host->host_lock, flags);
4167 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
4168 spin_unlock_irqrestore(hba->host->host_lock, flags);
4169 }
4170
4171 /**
4172 * ufshcd_init_pwr_info - setting the POR (power on reset)
4173 * values in hba power info
4174 * @hba: per-adapter instance
4175 */
ufshcd_init_pwr_info(struct ufs_hba * hba)4176 static void ufshcd_init_pwr_info(struct ufs_hba *hba)
4177 {
4178 hba->pwr_info.gear_rx = UFS_PWM_G1;
4179 hba->pwr_info.gear_tx = UFS_PWM_G1;
4180 hba->pwr_info.lane_rx = 1;
4181 hba->pwr_info.lane_tx = 1;
4182 hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
4183 hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
4184 hba->pwr_info.hs_rate = 0;
4185 }
4186
4187 /**
4188 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
4189 * @hba: per-adapter instance
4190 */
ufshcd_get_max_pwr_mode(struct ufs_hba * hba)4191 static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
4192 {
4193 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
4194
4195 if (hba->max_pwr_info.is_valid)
4196 return 0;
4197
4198 pwr_info->pwr_tx = FAST_MODE;
4199 pwr_info->pwr_rx = FAST_MODE;
4200 pwr_info->hs_rate = PA_HS_MODE_B;
4201
4202 /* Get the connected lane count */
4203 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
4204 &pwr_info->lane_rx);
4205 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4206 &pwr_info->lane_tx);
4207
4208 if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
4209 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
4210 __func__,
4211 pwr_info->lane_rx,
4212 pwr_info->lane_tx);
4213 return -EINVAL;
4214 }
4215
4216 /*
4217 * First, get the maximum gears of HS speed.
4218 * If a zero value, it means there is no HSGEAR capability.
4219 * Then, get the maximum gears of PWM speed.
4220 */
4221 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
4222 if (!pwr_info->gear_rx) {
4223 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4224 &pwr_info->gear_rx);
4225 if (!pwr_info->gear_rx) {
4226 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
4227 __func__, pwr_info->gear_rx);
4228 return -EINVAL;
4229 }
4230 pwr_info->pwr_rx = SLOW_MODE;
4231 }
4232
4233 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
4234 &pwr_info->gear_tx);
4235 if (!pwr_info->gear_tx) {
4236 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4237 &pwr_info->gear_tx);
4238 if (!pwr_info->gear_tx) {
4239 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
4240 __func__, pwr_info->gear_tx);
4241 return -EINVAL;
4242 }
4243 pwr_info->pwr_tx = SLOW_MODE;
4244 }
4245
4246 hba->max_pwr_info.is_valid = true;
4247 return 0;
4248 }
4249
ufshcd_change_power_mode(struct ufs_hba * hba,struct ufs_pa_layer_attr * pwr_mode)4250 static int ufshcd_change_power_mode(struct ufs_hba *hba,
4251 struct ufs_pa_layer_attr *pwr_mode)
4252 {
4253 int ret;
4254
4255 /* if already configured to the requested pwr_mode */
4256 if (!hba->force_pmc &&
4257 pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
4258 pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4259 pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4260 pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4261 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4262 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4263 pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4264 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4265 return 0;
4266 }
4267
4268 /*
4269 * Configure attributes for power mode change with below.
4270 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4271 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4272 * - PA_HSSERIES
4273 */
4274 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4275 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4276 pwr_mode->lane_rx);
4277 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4278 pwr_mode->pwr_rx == FAST_MODE)
4279 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
4280 else
4281 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
4282
4283 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4284 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4285 pwr_mode->lane_tx);
4286 if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4287 pwr_mode->pwr_tx == FAST_MODE)
4288 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
4289 else
4290 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
4291
4292 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4293 pwr_mode->pwr_tx == FASTAUTO_MODE ||
4294 pwr_mode->pwr_rx == FAST_MODE ||
4295 pwr_mode->pwr_tx == FAST_MODE)
4296 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4297 pwr_mode->hs_rate);
4298
4299 if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) {
4300 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0),
4301 DL_FC0ProtectionTimeOutVal_Default);
4302 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1),
4303 DL_TC0ReplayTimeOutVal_Default);
4304 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2),
4305 DL_AFC0ReqTimeOutVal_Default);
4306 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3),
4307 DL_FC1ProtectionTimeOutVal_Default);
4308 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4),
4309 DL_TC1ReplayTimeOutVal_Default);
4310 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5),
4311 DL_AFC1ReqTimeOutVal_Default);
4312
4313 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal),
4314 DL_FC0ProtectionTimeOutVal_Default);
4315 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal),
4316 DL_TC0ReplayTimeOutVal_Default);
4317 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal),
4318 DL_AFC0ReqTimeOutVal_Default);
4319 }
4320
4321 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4322 | pwr_mode->pwr_tx);
4323
4324 if (ret) {
4325 dev_err(hba->dev,
4326 "%s: power mode change failed %d\n", __func__, ret);
4327 } else {
4328 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4329 pwr_mode);
4330
4331 memcpy(&hba->pwr_info, pwr_mode,
4332 sizeof(struct ufs_pa_layer_attr));
4333 }
4334
4335 return ret;
4336 }
4337
4338 /**
4339 * ufshcd_config_pwr_mode - configure a new power mode
4340 * @hba: per-adapter instance
4341 * @desired_pwr_mode: desired power configuration
4342 */
ufshcd_config_pwr_mode(struct ufs_hba * hba,struct ufs_pa_layer_attr * desired_pwr_mode)4343 int ufshcd_config_pwr_mode(struct ufs_hba *hba,
4344 struct ufs_pa_layer_attr *desired_pwr_mode)
4345 {
4346 struct ufs_pa_layer_attr final_params = { 0 };
4347 int ret;
4348
4349 ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4350 desired_pwr_mode, &final_params);
4351
4352 if (ret)
4353 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4354
4355 ret = ufshcd_change_power_mode(hba, &final_params);
4356
4357 return ret;
4358 }
4359 EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
4360
4361 /**
4362 * ufshcd_complete_dev_init() - checks device readiness
4363 * @hba: per-adapter instance
4364 *
4365 * Set fDeviceInit flag and poll until device toggles it.
4366 */
ufshcd_complete_dev_init(struct ufs_hba * hba)4367 static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4368 {
4369 int err;
4370 bool flag_res = true;
4371 ktime_t timeout;
4372
4373 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
4374 QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL);
4375 if (err) {
4376 dev_err(hba->dev,
4377 "%s setting fDeviceInit flag failed with error %d\n",
4378 __func__, err);
4379 goto out;
4380 }
4381
4382 /* Poll fDeviceInit flag to be cleared */
4383 timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT);
4384 do {
4385 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4386 QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res);
4387 if (!flag_res)
4388 break;
4389 usleep_range(500, 1000);
4390 } while (ktime_before(ktime_get(), timeout));
4391
4392 if (err) {
4393 dev_err(hba->dev,
4394 "%s reading fDeviceInit flag failed with error %d\n",
4395 __func__, err);
4396 } else if (flag_res) {
4397 dev_err(hba->dev,
4398 "%s fDeviceInit was not cleared by the device\n",
4399 __func__);
4400 err = -EBUSY;
4401 }
4402 out:
4403 return err;
4404 }
4405
4406 /**
4407 * ufshcd_make_hba_operational - Make UFS controller operational
4408 * @hba: per adapter instance
4409 *
4410 * To bring UFS host controller to operational state,
4411 * 1. Enable required interrupts
4412 * 2. Configure interrupt aggregation
4413 * 3. Program UTRL and UTMRL base address
4414 * 4. Configure run-stop-registers
4415 *
4416 * Returns 0 on success, non-zero value on failure
4417 */
ufshcd_make_hba_operational(struct ufs_hba * hba)4418 int ufshcd_make_hba_operational(struct ufs_hba *hba)
4419 {
4420 int err = 0;
4421 u32 reg;
4422
4423 /* Enable required interrupts */
4424 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4425
4426 /* Configure interrupt aggregation */
4427 if (ufshcd_is_intr_aggr_allowed(hba))
4428 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4429 else
4430 ufshcd_disable_intr_aggr(hba);
4431
4432 /* Configure UTRL and UTMRL base address registers */
4433 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4434 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4435 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4436 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4437 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4438 REG_UTP_TASK_REQ_LIST_BASE_L);
4439 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4440 REG_UTP_TASK_REQ_LIST_BASE_H);
4441
4442 /*
4443 * Make sure base address and interrupt setup are updated before
4444 * enabling the run/stop registers below.
4445 */
4446 wmb();
4447
4448 /*
4449 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
4450 */
4451 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
4452 if (!(ufshcd_get_lists_status(reg))) {
4453 ufshcd_enable_run_stop_reg(hba);
4454 } else {
4455 dev_err(hba->dev,
4456 "Host controller not ready to process requests");
4457 err = -EIO;
4458 }
4459
4460 return err;
4461 }
4462 EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
4463
4464 /**
4465 * ufshcd_hba_stop - Send controller to reset state
4466 * @hba: per adapter instance
4467 */
ufshcd_hba_stop(struct ufs_hba * hba)4468 void ufshcd_hba_stop(struct ufs_hba *hba)
4469 {
4470 unsigned long flags;
4471 int err;
4472
4473 /*
4474 * Obtain the host lock to prevent that the controller is disabled
4475 * while the UFS interrupt handler is active on another CPU.
4476 */
4477 spin_lock_irqsave(hba->host->host_lock, flags);
4478 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE);
4479 spin_unlock_irqrestore(hba->host->host_lock, flags);
4480
4481 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4482 CONTROLLER_ENABLE, CONTROLLER_DISABLE,
4483 10, 1);
4484 if (err)
4485 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4486 }
4487 EXPORT_SYMBOL_GPL(ufshcd_hba_stop);
4488
4489 /**
4490 * ufshcd_hba_execute_hce - initialize the controller
4491 * @hba: per adapter instance
4492 *
4493 * The controller resets itself and controller firmware initialization
4494 * sequence kicks off. When controller is ready it will set
4495 * the Host Controller Enable bit to 1.
4496 *
4497 * Returns 0 on success, non-zero value on failure
4498 */
ufshcd_hba_execute_hce(struct ufs_hba * hba)4499 static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
4500 {
4501 int retry_outer = 3;
4502 int retry_inner;
4503
4504 start:
4505 if (!ufshcd_is_hba_active(hba))
4506 /* change controller state to "reset state" */
4507 ufshcd_hba_stop(hba);
4508
4509 /* UniPro link is disabled at this point */
4510 ufshcd_set_link_off(hba);
4511
4512 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4513
4514 /* start controller initialization sequence */
4515 ufshcd_hba_start(hba);
4516
4517 /*
4518 * To initialize a UFS host controller HCE bit must be set to 1.
4519 * During initialization the HCE bit value changes from 1->0->1.
4520 * When the host controller completes initialization sequence
4521 * it sets the value of HCE bit to 1. The same HCE bit is read back
4522 * to check if the controller has completed initialization sequence.
4523 * So without this delay the value HCE = 1, set in the previous
4524 * instruction might be read back.
4525 * This delay can be changed based on the controller.
4526 */
4527 ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100);
4528
4529 /* wait for the host controller to complete initialization */
4530 retry_inner = 50;
4531 while (ufshcd_is_hba_active(hba)) {
4532 if (retry_inner) {
4533 retry_inner--;
4534 } else {
4535 dev_err(hba->dev,
4536 "Controller enable failed\n");
4537 if (retry_outer) {
4538 retry_outer--;
4539 goto start;
4540 }
4541 return -EIO;
4542 }
4543 usleep_range(1000, 1100);
4544 }
4545
4546 /* enable UIC related interrupts */
4547 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4548
4549 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4550
4551 return 0;
4552 }
4553
ufshcd_hba_enable(struct ufs_hba * hba)4554 int ufshcd_hba_enable(struct ufs_hba *hba)
4555 {
4556 int ret;
4557
4558 if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4559 ufshcd_set_link_off(hba);
4560 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4561
4562 /* enable UIC related interrupts */
4563 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4564 ret = ufshcd_dme_reset(hba);
4565 if (!ret) {
4566 ret = ufshcd_dme_enable(hba);
4567 if (!ret)
4568 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4569 if (ret)
4570 dev_err(hba->dev,
4571 "Host controller enable failed with non-hce\n");
4572 }
4573 } else {
4574 ret = ufshcd_hba_execute_hce(hba);
4575 }
4576
4577 return ret;
4578 }
4579 EXPORT_SYMBOL_GPL(ufshcd_hba_enable);
4580
ufshcd_disable_tx_lcc(struct ufs_hba * hba,bool peer)4581 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4582 {
4583 int tx_lanes = 0, i, err = 0;
4584
4585 if (!peer)
4586 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4587 &tx_lanes);
4588 else
4589 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4590 &tx_lanes);
4591 for (i = 0; i < tx_lanes; i++) {
4592 if (!peer)
4593 err = ufshcd_dme_set(hba,
4594 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4595 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4596 0);
4597 else
4598 err = ufshcd_dme_peer_set(hba,
4599 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4600 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4601 0);
4602 if (err) {
4603 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4604 __func__, peer, i, err);
4605 break;
4606 }
4607 }
4608
4609 return err;
4610 }
4611
ufshcd_disable_device_tx_lcc(struct ufs_hba * hba)4612 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4613 {
4614 return ufshcd_disable_tx_lcc(hba, true);
4615 }
4616
ufshcd_update_evt_hist(struct ufs_hba * hba,u32 id,u32 val)4617 void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val)
4618 {
4619 struct ufs_event_hist *e;
4620
4621 if (id >= UFS_EVT_CNT)
4622 return;
4623
4624 e = &hba->ufs_stats.event[id];
4625 e->val[e->pos] = val;
4626 e->tstamp[e->pos] = ktime_get();
4627 e->cnt += 1;
4628 e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH;
4629
4630 ufshcd_vops_event_notify(hba, id, &val);
4631 }
4632 EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist);
4633
4634 /**
4635 * ufshcd_link_startup - Initialize unipro link startup
4636 * @hba: per adapter instance
4637 *
4638 * Returns 0 for success, non-zero in case of failure
4639 */
ufshcd_link_startup(struct ufs_hba * hba)4640 static int ufshcd_link_startup(struct ufs_hba *hba)
4641 {
4642 int ret;
4643 int retries = DME_LINKSTARTUP_RETRIES;
4644 bool link_startup_again = false;
4645
4646 /*
4647 * If UFS device isn't active then we will have to issue link startup
4648 * 2 times to make sure the device state move to active.
4649 */
4650 if (!ufshcd_is_ufs_dev_active(hba))
4651 link_startup_again = true;
4652
4653 link_startup:
4654 do {
4655 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
4656
4657 ret = ufshcd_dme_link_startup(hba);
4658
4659 /* check if device is detected by inter-connect layer */
4660 if (!ret && !ufshcd_is_device_present(hba)) {
4661 ufshcd_update_evt_hist(hba,
4662 UFS_EVT_LINK_STARTUP_FAIL,
4663 0);
4664 dev_err(hba->dev, "%s: Device not present\n", __func__);
4665 ret = -ENXIO;
4666 goto out;
4667 }
4668
4669 /*
4670 * DME link lost indication is only received when link is up,
4671 * but we can't be sure if the link is up until link startup
4672 * succeeds. So reset the local Uni-Pro and try again.
4673 */
4674 if (ret && ufshcd_hba_enable(hba)) {
4675 ufshcd_update_evt_hist(hba,
4676 UFS_EVT_LINK_STARTUP_FAIL,
4677 (u32)ret);
4678 goto out;
4679 }
4680 } while (ret && retries--);
4681
4682 if (ret) {
4683 /* failed to get the link up... retire */
4684 ufshcd_update_evt_hist(hba,
4685 UFS_EVT_LINK_STARTUP_FAIL,
4686 (u32)ret);
4687 goto out;
4688 }
4689
4690 if (link_startup_again) {
4691 link_startup_again = false;
4692 retries = DME_LINKSTARTUP_RETRIES;
4693 goto link_startup;
4694 }
4695
4696 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4697 ufshcd_init_pwr_info(hba);
4698 ufshcd_print_pwr_info(hba);
4699
4700 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4701 ret = ufshcd_disable_device_tx_lcc(hba);
4702 if (ret)
4703 goto out;
4704 }
4705
4706 /* Include any host controller configuration via UIC commands */
4707 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
4708 if (ret)
4709 goto out;
4710
4711 /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */
4712 ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
4713 ret = ufshcd_make_hba_operational(hba);
4714 out:
4715 if (ret) {
4716 dev_err(hba->dev, "link startup failed %d\n", ret);
4717 ufshcd_print_host_state(hba);
4718 ufshcd_print_pwr_info(hba);
4719 ufshcd_print_evt_hist(hba);
4720 }
4721 return ret;
4722 }
4723
4724 /**
4725 * ufshcd_verify_dev_init() - Verify device initialization
4726 * @hba: per-adapter instance
4727 *
4728 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4729 * device Transport Protocol (UTP) layer is ready after a reset.
4730 * If the UTP layer at the device side is not initialized, it may
4731 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4732 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4733 */
ufshcd_verify_dev_init(struct ufs_hba * hba)4734 static int ufshcd_verify_dev_init(struct ufs_hba *hba)
4735 {
4736 int err = 0;
4737 int retries;
4738
4739 ufshcd_hold(hba, false);
4740 mutex_lock(&hba->dev_cmd.lock);
4741 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
4742 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
4743 NOP_OUT_TIMEOUT);
4744
4745 if (!err || err == -ETIMEDOUT)
4746 break;
4747
4748 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
4749 }
4750 mutex_unlock(&hba->dev_cmd.lock);
4751 ufshcd_release(hba);
4752
4753 if (err)
4754 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
4755 return err;
4756 }
4757
4758 /**
4759 * ufshcd_set_queue_depth - set lun queue depth
4760 * @sdev: pointer to SCSI device
4761 *
4762 * Read bLUQueueDepth value and activate scsi tagged command
4763 * queueing. For WLUN, queue depth is set to 1. For best-effort
4764 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4765 * value that host can queue.
4766 */
ufshcd_set_queue_depth(struct scsi_device * sdev)4767 static void ufshcd_set_queue_depth(struct scsi_device *sdev)
4768 {
4769 int ret = 0;
4770 u8 lun_qdepth;
4771 struct ufs_hba *hba;
4772
4773 hba = shost_priv(sdev->host);
4774
4775 lun_qdepth = hba->nutrs;
4776 ret = ufshcd_read_unit_desc_param(hba,
4777 ufshcd_scsi_to_upiu_lun(sdev->lun),
4778 UNIT_DESC_PARAM_LU_Q_DEPTH,
4779 &lun_qdepth,
4780 sizeof(lun_qdepth));
4781
4782 /* Some WLUN doesn't support unit descriptor */
4783 if (ret == -EOPNOTSUPP)
4784 lun_qdepth = 1;
4785 else if (!lun_qdepth)
4786 /* eventually, we can figure out the real queue depth */
4787 lun_qdepth = hba->nutrs;
4788 else
4789 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
4790
4791 dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
4792 __func__, lun_qdepth);
4793 scsi_change_queue_depth(sdev, lun_qdepth);
4794 }
4795
4796 /*
4797 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4798 * @hba: per-adapter instance
4799 * @lun: UFS device lun id
4800 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4801 *
4802 * Returns 0 in case of success and b_lu_write_protect status would be returned
4803 * @b_lu_write_protect parameter.
4804 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4805 * Returns -EINVAL in case of invalid parameters passed to this function.
4806 */
ufshcd_get_lu_wp(struct ufs_hba * hba,u8 lun,u8 * b_lu_write_protect)4807 static int ufshcd_get_lu_wp(struct ufs_hba *hba,
4808 u8 lun,
4809 u8 *b_lu_write_protect)
4810 {
4811 int ret;
4812
4813 if (!b_lu_write_protect)
4814 ret = -EINVAL;
4815 /*
4816 * According to UFS device spec, RPMB LU can't be write
4817 * protected so skip reading bLUWriteProtect parameter for
4818 * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4819 */
4820 else if (lun >= hba->dev_info.max_lu_supported)
4821 ret = -ENOTSUPP;
4822 else
4823 ret = ufshcd_read_unit_desc_param(hba,
4824 lun,
4825 UNIT_DESC_PARAM_LU_WR_PROTECT,
4826 b_lu_write_protect,
4827 sizeof(*b_lu_write_protect));
4828 return ret;
4829 }
4830
4831 /**
4832 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4833 * status
4834 * @hba: per-adapter instance
4835 * @sdev: pointer to SCSI device
4836 *
4837 */
ufshcd_get_lu_power_on_wp_status(struct ufs_hba * hba,struct scsi_device * sdev)4838 static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
4839 struct scsi_device *sdev)
4840 {
4841 if (hba->dev_info.f_power_on_wp_en &&
4842 !hba->dev_info.is_lu_power_on_wp) {
4843 u8 b_lu_write_protect;
4844
4845 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
4846 &b_lu_write_protect) &&
4847 (b_lu_write_protect == UFS_LU_POWER_ON_WP))
4848 hba->dev_info.is_lu_power_on_wp = true;
4849 }
4850 }
4851
4852 /**
4853 * ufshcd_slave_alloc - handle initial SCSI device configurations
4854 * @sdev: pointer to SCSI device
4855 *
4856 * Returns success
4857 */
ufshcd_slave_alloc(struct scsi_device * sdev)4858 static int ufshcd_slave_alloc(struct scsi_device *sdev)
4859 {
4860 struct ufs_hba *hba;
4861
4862 hba = shost_priv(sdev->host);
4863
4864 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4865 sdev->use_10_for_ms = 1;
4866
4867 /* DBD field should be set to 1 in mode sense(10) */
4868 sdev->set_dbd_for_ms = 1;
4869
4870 /* allow SCSI layer to restart the device in case of errors */
4871 sdev->allow_restart = 1;
4872
4873 /* REPORT SUPPORTED OPERATION CODES is not supported */
4874 sdev->no_report_opcodes = 1;
4875
4876 /* WRITE_SAME command is not supported */
4877 sdev->no_write_same = 1;
4878
4879 ufshcd_set_queue_depth(sdev);
4880
4881 ufshcd_get_lu_power_on_wp_status(hba, sdev);
4882
4883 return 0;
4884 }
4885
4886 /**
4887 * ufshcd_change_queue_depth - change queue depth
4888 * @sdev: pointer to SCSI device
4889 * @depth: required depth to set
4890 *
4891 * Change queue depth and make sure the max. limits are not crossed.
4892 */
ufshcd_change_queue_depth(struct scsi_device * sdev,int depth)4893 static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
4894 {
4895 return scsi_change_queue_depth(sdev, min(depth, sdev->host->can_queue));
4896 }
4897
ufshcd_hpb_destroy(struct ufs_hba * hba,struct scsi_device * sdev)4898 static void ufshcd_hpb_destroy(struct ufs_hba *hba, struct scsi_device *sdev)
4899 {
4900 /* skip well-known LU */
4901 if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) ||
4902 !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba))
4903 return;
4904
4905 ufshpb_destroy_lu(hba, sdev);
4906 }
4907
ufshcd_hpb_configure(struct ufs_hba * hba,struct scsi_device * sdev)4908 static void ufshcd_hpb_configure(struct ufs_hba *hba, struct scsi_device *sdev)
4909 {
4910 /* skip well-known LU */
4911 if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) ||
4912 !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba))
4913 return;
4914
4915 ufshpb_init_hpb_lu(hba, sdev);
4916 }
4917
4918 /**
4919 * ufshcd_slave_configure - adjust SCSI device configurations
4920 * @sdev: pointer to SCSI device
4921 */
ufshcd_slave_configure(struct scsi_device * sdev)4922 static int ufshcd_slave_configure(struct scsi_device *sdev)
4923 {
4924 struct ufs_hba *hba = shost_priv(sdev->host);
4925 struct request_queue *q = sdev->request_queue;
4926
4927 ufshcd_hpb_configure(hba, sdev);
4928
4929 blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
4930 if (hba->quirks & UFSHCD_QUIRK_ALIGN_SG_WITH_PAGE_SIZE)
4931 blk_queue_update_dma_alignment(q, PAGE_SIZE - 1);
4932
4933 if (ufshcd_is_rpm_autosuspend_allowed(hba))
4934 sdev->rpm_autosuspend = 1;
4935
4936 ufshcd_crypto_setup_rq_keyslot_manager(hba, q);
4937
4938 trace_android_vh_ufs_update_sdev(sdev);
4939
4940 return 0;
4941 }
4942
4943 /**
4944 * ufshcd_slave_destroy - remove SCSI device configurations
4945 * @sdev: pointer to SCSI device
4946 */
ufshcd_slave_destroy(struct scsi_device * sdev)4947 static void ufshcd_slave_destroy(struct scsi_device *sdev)
4948 {
4949 struct ufs_hba *hba;
4950
4951 hba = shost_priv(sdev->host);
4952
4953 ufshcd_hpb_destroy(hba, sdev);
4954
4955 /* Drop the reference as it won't be needed anymore */
4956 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
4957 unsigned long flags;
4958
4959 spin_lock_irqsave(hba->host->host_lock, flags);
4960 hba->sdev_ufs_device = NULL;
4961 spin_unlock_irqrestore(hba->host->host_lock, flags);
4962 }
4963 }
4964
4965 /**
4966 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
4967 * @lrbp: pointer to local reference block of completed command
4968 * @scsi_status: SCSI command status
4969 *
4970 * Returns value base on SCSI command status
4971 */
4972 static inline int
ufshcd_scsi_cmd_status(struct ufshcd_lrb * lrbp,int scsi_status)4973 ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
4974 {
4975 int result = 0;
4976
4977 switch (scsi_status) {
4978 case SAM_STAT_CHECK_CONDITION:
4979 ufshcd_copy_sense_data(lrbp);
4980 fallthrough;
4981 case SAM_STAT_GOOD:
4982 result |= DID_OK << 16 |
4983 COMMAND_COMPLETE << 8 |
4984 scsi_status;
4985 break;
4986 case SAM_STAT_TASK_SET_FULL:
4987 case SAM_STAT_BUSY:
4988 case SAM_STAT_TASK_ABORTED:
4989 ufshcd_copy_sense_data(lrbp);
4990 result |= scsi_status;
4991 break;
4992 default:
4993 result |= DID_ERROR << 16;
4994 break;
4995 } /* end of switch */
4996
4997 return result;
4998 }
4999
5000 /**
5001 * ufshcd_transfer_rsp_status - Get overall status of the response
5002 * @hba: per adapter instance
5003 * @lrbp: pointer to local reference block of completed command
5004 *
5005 * Returns result of the command to notify SCSI midlayer
5006 */
5007 static inline int
ufshcd_transfer_rsp_status(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)5008 ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
5009 {
5010 int result = 0;
5011 int scsi_status;
5012 int ocs;
5013
5014 /* overall command status of utrd */
5015 ocs = ufshcd_get_tr_ocs(lrbp);
5016
5017 if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) {
5018 if (be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_1) &
5019 MASK_RSP_UPIU_RESULT)
5020 ocs = OCS_SUCCESS;
5021 }
5022
5023 switch (ocs) {
5024 case OCS_SUCCESS:
5025 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
5026 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5027 switch (result) {
5028 case UPIU_TRANSACTION_RESPONSE:
5029 /*
5030 * get the response UPIU result to extract
5031 * the SCSI command status
5032 */
5033 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
5034
5035 /*
5036 * get the result based on SCSI status response
5037 * to notify the SCSI midlayer of the command status
5038 */
5039 scsi_status = result & MASK_SCSI_STATUS;
5040 result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
5041
5042 /*
5043 * Currently we are only supporting BKOPs exception
5044 * events hence we can ignore BKOPs exception event
5045 * during power management callbacks. BKOPs exception
5046 * event is not expected to be raised in runtime suspend
5047 * callback as it allows the urgent bkops.
5048 * During system suspend, we are anyway forcefully
5049 * disabling the bkops and if urgent bkops is needed
5050 * it will be enabled on system resume. Long term
5051 * solution could be to abort the system suspend if
5052 * UFS device needs urgent BKOPs.
5053 */
5054 if (!hba->pm_op_in_progress &&
5055 !ufshcd_eh_in_progress(hba) &&
5056 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr) &&
5057 schedule_work(&hba->eeh_work)) {
5058 /*
5059 * Prevent suspend once eeh_work is scheduled
5060 * to avoid deadlock between ufshcd_suspend
5061 * and exception event handler.
5062 */
5063 pm_runtime_get_noresume(hba->dev);
5064 }
5065
5066 if (scsi_status == SAM_STAT_GOOD)
5067 ufshpb_rsp_upiu(hba, lrbp);
5068 break;
5069 case UPIU_TRANSACTION_REJECT_UPIU:
5070 /* TODO: handle Reject UPIU Response */
5071 result = DID_ERROR << 16;
5072 dev_err(hba->dev,
5073 "Reject UPIU not fully implemented\n");
5074 break;
5075 default:
5076 dev_err(hba->dev,
5077 "Unexpected request response code = %x\n",
5078 result);
5079 result = DID_ERROR << 16;
5080 break;
5081 }
5082 break;
5083 case OCS_ABORTED:
5084 result |= DID_ABORT << 16;
5085 break;
5086 case OCS_INVALID_COMMAND_STATUS:
5087 dev_err_ratelimited(hba->dev,
5088 "Retrying request with tag %d / cdb %#02x because of invalid command status\n",
5089 lrbp->task_tag, lrbp->cmd && lrbp->cmd->cmnd ?
5090 lrbp->cmd->cmnd[0] : 0);
5091 result |= DID_REQUEUE << 16;
5092 break;
5093 case OCS_INVALID_CMD_TABLE_ATTR:
5094 case OCS_INVALID_PRDT_ATTR:
5095 case OCS_MISMATCH_DATA_BUF_SIZE:
5096 case OCS_MISMATCH_RESP_UPIU_SIZE:
5097 case OCS_PEER_COMM_FAILURE:
5098 case OCS_FATAL_ERROR:
5099 case OCS_DEVICE_FATAL_ERROR:
5100 case OCS_INVALID_CRYPTO_CONFIG:
5101 case OCS_GENERAL_CRYPTO_ERROR:
5102 default:
5103 result |= DID_ERROR << 16;
5104 dev_err(hba->dev,
5105 "OCS error from controller = %x for tag %d\n",
5106 ocs, lrbp->task_tag);
5107 ufshcd_print_evt_hist(hba);
5108 ufshcd_print_host_state(hba);
5109 break;
5110 } /* end of switch */
5111
5112 if ((host_byte(result) != DID_OK) &&
5113 (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs)
5114 ufshcd_print_trs(hba, 1 << lrbp->task_tag, true);
5115 return result;
5116 }
5117
ufshcd_is_auto_hibern8_error(struct ufs_hba * hba,u32 intr_mask)5118 static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
5119 u32 intr_mask)
5120 {
5121 if (!ufshcd_is_auto_hibern8_supported(hba) ||
5122 !ufshcd_is_auto_hibern8_enabled(hba))
5123 return false;
5124
5125 if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
5126 return false;
5127
5128 if (hba->active_uic_cmd &&
5129 (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
5130 hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
5131 return false;
5132
5133 return true;
5134 }
5135
5136 /**
5137 * ufshcd_uic_cmd_compl - handle completion of uic command
5138 * @hba: per adapter instance
5139 * @intr_status: interrupt status generated by the controller
5140 *
5141 * Returns
5142 * IRQ_HANDLED - If interrupt is valid
5143 * IRQ_NONE - If invalid interrupt
5144 */
ufshcd_uic_cmd_compl(struct ufs_hba * hba,u32 intr_status)5145 static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
5146 {
5147 irqreturn_t retval = IRQ_NONE;
5148
5149 spin_lock(hba->host->host_lock);
5150 if (ufshcd_is_auto_hibern8_error(hba, intr_status))
5151 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
5152
5153 if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
5154 hba->active_uic_cmd->argument2 |=
5155 ufshcd_get_uic_cmd_result(hba);
5156 hba->active_uic_cmd->argument3 =
5157 ufshcd_get_dme_attr_val(hba);
5158 if (!hba->uic_async_done)
5159 hba->active_uic_cmd->cmd_active = 0;
5160 complete(&hba->active_uic_cmd->done);
5161 retval = IRQ_HANDLED;
5162 }
5163
5164 if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
5165 hba->active_uic_cmd->cmd_active = 0;
5166 complete(hba->uic_async_done);
5167 retval = IRQ_HANDLED;
5168 }
5169
5170 if (retval == IRQ_HANDLED)
5171 ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd,
5172 "complete");
5173 spin_unlock(hba->host->host_lock);
5174 return retval;
5175 }
5176
5177 /* Release the resources allocated for processing a SCSI command. */
ufshcd_release_scsi_cmd(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)5178 static void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
5179 struct ufshcd_lrb *lrbp)
5180 {
5181 struct scsi_cmnd *cmd = lrbp->cmd;
5182
5183 scsi_dma_unmap(cmd);
5184 ufshcd_crypto_clear_prdt(hba, lrbp);
5185 lrbp->cmd = NULL; /* Mark the command as completed. */
5186 ufshcd_release(hba);
5187 ufshcd_clk_scaling_update_busy(hba);
5188 }
5189
5190 /**
5191 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
5192 * @hba: per adapter instance
5193 * @completed_reqs: requests to complete
5194 */
__ufshcd_transfer_req_compl(struct ufs_hba * hba,unsigned long completed_reqs)5195 static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
5196 unsigned long completed_reqs)
5197 {
5198 struct ufshcd_lrb *lrbp;
5199 struct scsi_cmnd *cmd;
5200 int index;
5201
5202 for_each_set_bit(index, &completed_reqs, hba->nutrs) {
5203 if (!test_and_clear_bit(index, &hba->outstanding_reqs))
5204 continue;
5205 lrbp = &hba->lrb[index];
5206 lrbp->compl_time_stamp = ktime_get();
5207 cmd = lrbp->cmd;
5208 if (cmd) {
5209 if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
5210 ufshcd_update_monitor(hba, lrbp);
5211 trace_android_vh_ufs_compl_command(hba, lrbp);
5212 ufshcd_add_command_trace(hba, index, "complete");
5213 cmd->result = ufshcd_transfer_rsp_status(hba, lrbp);
5214 ufshcd_release_scsi_cmd(hba, lrbp);
5215 /* Do not touch lrbp after scsi done */
5216 cmd->scsi_done(cmd);
5217 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
5218 lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
5219 if (hba->dev_cmd.complete) {
5220 trace_android_vh_ufs_compl_command(hba, lrbp);
5221 ufshcd_add_command_trace(hba, index,
5222 "dev_complete");
5223 complete(hba->dev_cmd.complete);
5224 ufshcd_clk_scaling_update_busy(hba);
5225 }
5226 }
5227 }
5228 }
5229
5230 /**
5231 * ufshcd_trc_handler - handle transfer requests completion
5232 * @hba: per adapter instance
5233 * @use_utrlcnr: get completed requests from UTRLCNR
5234 *
5235 * Returns
5236 * IRQ_HANDLED - If interrupt is valid
5237 * IRQ_NONE - If invalid interrupt
5238 */
ufshcd_trc_handler(struct ufs_hba * hba,bool use_utrlcnr)5239 static irqreturn_t ufshcd_trc_handler(struct ufs_hba *hba, bool use_utrlcnr)
5240 {
5241 unsigned long completed_reqs = 0;
5242
5243 /* Resetting interrupt aggregation counters first and reading the
5244 * DOOR_BELL afterward allows us to handle all the completed requests.
5245 * In order to prevent other interrupts starvation the DB is read once
5246 * after reset. The down side of this solution is the possibility of
5247 * false interrupt if device completes another request after resetting
5248 * aggregation and before reading the DB.
5249 */
5250 if (ufshcd_is_intr_aggr_allowed(hba) &&
5251 !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
5252 ufshcd_reset_intr_aggr(hba);
5253
5254 if (use_utrlcnr) {
5255 u32 utrlcnr;
5256
5257 utrlcnr = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_LIST_COMPL);
5258 if (utrlcnr) {
5259 ufshcd_writel(hba, utrlcnr,
5260 REG_UTP_TRANSFER_REQ_LIST_COMPL);
5261 completed_reqs = utrlcnr;
5262 }
5263 } else {
5264 unsigned long flags;
5265 u32 tr_doorbell;
5266
5267 spin_lock_irqsave(hba->host->host_lock, flags);
5268 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
5269 completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
5270 spin_unlock_irqrestore(hba->host->host_lock, flags);
5271 }
5272
5273 if (completed_reqs) {
5274 __ufshcd_transfer_req_compl(hba, completed_reqs);
5275 return IRQ_HANDLED;
5276 } else {
5277 return IRQ_NONE;
5278 }
5279 }
5280
5281 /**
5282 * ufshcd_disable_ee - disable exception event
5283 * @hba: per-adapter instance
5284 * @mask: exception event to disable
5285 *
5286 * Disables exception event in the device so that the EVENT_ALERT
5287 * bit is not set.
5288 *
5289 * Returns zero on success, non-zero error value on failure.
5290 */
ufshcd_disable_ee(struct ufs_hba * hba,u16 mask)5291 static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
5292 {
5293 int err = 0;
5294 u32 val;
5295
5296 if (!(hba->ee_ctrl_mask & mask))
5297 goto out;
5298
5299 val = hba->ee_ctrl_mask & ~mask;
5300 val &= MASK_EE_STATUS;
5301 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
5302 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
5303 if (!err)
5304 hba->ee_ctrl_mask &= ~mask;
5305 out:
5306 return err;
5307 }
5308
5309 /**
5310 * ufshcd_enable_ee - enable exception event
5311 * @hba: per-adapter instance
5312 * @mask: exception event to enable
5313 *
5314 * Enable corresponding exception event in the device to allow
5315 * device to alert host in critical scenarios.
5316 *
5317 * Returns zero on success, non-zero error value on failure.
5318 */
ufshcd_enable_ee(struct ufs_hba * hba,u16 mask)5319 static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
5320 {
5321 int err = 0;
5322 u32 val;
5323
5324 if (hba->ee_ctrl_mask & mask)
5325 goto out;
5326
5327 val = hba->ee_ctrl_mask | mask;
5328 val &= MASK_EE_STATUS;
5329 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
5330 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
5331 if (!err)
5332 hba->ee_ctrl_mask |= mask;
5333 out:
5334 return err;
5335 }
5336
5337 /**
5338 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
5339 * @hba: per-adapter instance
5340 *
5341 * Allow device to manage background operations on its own. Enabling
5342 * this might lead to inconsistent latencies during normal data transfers
5343 * as the device is allowed to manage its own way of handling background
5344 * operations.
5345 *
5346 * Returns zero on success, non-zero on failure.
5347 */
ufshcd_enable_auto_bkops(struct ufs_hba * hba)5348 static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
5349 {
5350 int err = 0;
5351
5352 if (hba->auto_bkops_enabled)
5353 goto out;
5354
5355 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
5356 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5357 if (err) {
5358 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
5359 __func__, err);
5360 goto out;
5361 }
5362
5363 hba->auto_bkops_enabled = true;
5364 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
5365
5366 /* No need of URGENT_BKOPS exception from the device */
5367 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5368 if (err)
5369 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
5370 __func__, err);
5371 out:
5372 return err;
5373 }
5374
5375 /**
5376 * ufshcd_disable_auto_bkops - block device in doing background operations
5377 * @hba: per-adapter instance
5378 *
5379 * Disabling background operations improves command response latency but
5380 * has drawback of device moving into critical state where the device is
5381 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
5382 * host is idle so that BKOPS are managed effectively without any negative
5383 * impacts.
5384 *
5385 * Returns zero on success, non-zero on failure.
5386 */
ufshcd_disable_auto_bkops(struct ufs_hba * hba)5387 static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
5388 {
5389 int err = 0;
5390
5391 if (!hba->auto_bkops_enabled)
5392 goto out;
5393
5394 /*
5395 * If host assisted BKOPs is to be enabled, make sure
5396 * urgent bkops exception is allowed.
5397 */
5398 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
5399 if (err) {
5400 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
5401 __func__, err);
5402 goto out;
5403 }
5404
5405 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
5406 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5407 if (err) {
5408 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
5409 __func__, err);
5410 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5411 goto out;
5412 }
5413
5414 hba->auto_bkops_enabled = false;
5415 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
5416 hba->is_urgent_bkops_lvl_checked = false;
5417 out:
5418 return err;
5419 }
5420
5421 /**
5422 * ufshcd_force_reset_auto_bkops - force reset auto bkops state
5423 * @hba: per adapter instance
5424 *
5425 * After a device reset the device may toggle the BKOPS_EN flag
5426 * to default value. The s/w tracking variables should be updated
5427 * as well. This function would change the auto-bkops state based on
5428 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
5429 */
ufshcd_force_reset_auto_bkops(struct ufs_hba * hba)5430 static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
5431 {
5432 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5433 hba->auto_bkops_enabled = false;
5434 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5435 ufshcd_enable_auto_bkops(hba);
5436 } else {
5437 hba->auto_bkops_enabled = true;
5438 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5439 ufshcd_disable_auto_bkops(hba);
5440 }
5441 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
5442 hba->is_urgent_bkops_lvl_checked = false;
5443 }
5444
ufshcd_get_bkops_status(struct ufs_hba * hba,u32 * status)5445 static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5446 {
5447 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5448 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5449 }
5450
5451 /**
5452 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
5453 * @hba: per-adapter instance
5454 * @status: bkops_status value
5455 *
5456 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5457 * flag in the device to permit background operations if the device
5458 * bkops_status is greater than or equal to "status" argument passed to
5459 * this function, disable otherwise.
5460 *
5461 * Returns 0 for success, non-zero in case of failure.
5462 *
5463 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5464 * to know whether auto bkops is enabled or disabled after this function
5465 * returns control to it.
5466 */
ufshcd_bkops_ctrl(struct ufs_hba * hba,enum bkops_status status)5467 int ufshcd_bkops_ctrl(struct ufs_hba *hba,
5468 enum bkops_status status)
5469 {
5470 int err;
5471 u32 curr_status = 0;
5472
5473 err = ufshcd_get_bkops_status(hba, &curr_status);
5474 if (err) {
5475 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5476 __func__, err);
5477 goto out;
5478 } else if (curr_status > BKOPS_STATUS_MAX) {
5479 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5480 __func__, curr_status);
5481 err = -EINVAL;
5482 goto out;
5483 }
5484
5485 if (curr_status >= status)
5486 err = ufshcd_enable_auto_bkops(hba);
5487 else
5488 err = ufshcd_disable_auto_bkops(hba);
5489 out:
5490 return err;
5491 }
5492 EXPORT_SYMBOL_GPL(ufshcd_bkops_ctrl);
5493
5494 /**
5495 * ufshcd_urgent_bkops - handle urgent bkops exception event
5496 * @hba: per-adapter instance
5497 *
5498 * Enable fBackgroundOpsEn flag in the device to permit background
5499 * operations.
5500 *
5501 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5502 * and negative error value for any other failure.
5503 */
ufshcd_urgent_bkops(struct ufs_hba * hba)5504 static int ufshcd_urgent_bkops(struct ufs_hba *hba)
5505 {
5506 return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
5507 }
5508
ufshcd_get_ee_status(struct ufs_hba * hba,u32 * status)5509 static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5510 {
5511 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5512 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5513 }
5514
ufshcd_bkops_exception_event_handler(struct ufs_hba * hba)5515 static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5516 {
5517 int err;
5518 u32 curr_status = 0;
5519
5520 if (hba->is_urgent_bkops_lvl_checked)
5521 goto enable_auto_bkops;
5522
5523 err = ufshcd_get_bkops_status(hba, &curr_status);
5524 if (err) {
5525 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5526 __func__, err);
5527 goto out;
5528 }
5529
5530 /*
5531 * We are seeing that some devices are raising the urgent bkops
5532 * exception events even when BKOPS status doesn't indicate performace
5533 * impacted or critical. Handle these device by determining their urgent
5534 * bkops status at runtime.
5535 */
5536 if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5537 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5538 __func__, curr_status);
5539 /* update the current status as the urgent bkops level */
5540 hba->urgent_bkops_lvl = curr_status;
5541 hba->is_urgent_bkops_lvl_checked = true;
5542 }
5543
5544 enable_auto_bkops:
5545 err = ufshcd_enable_auto_bkops(hba);
5546 out:
5547 if (err < 0)
5548 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5549 __func__, err);
5550 }
5551
ufshcd_wb_ctrl(struct ufs_hba * hba,bool enable)5552 static int ufshcd_wb_ctrl(struct ufs_hba *hba, bool enable)
5553 {
5554 int ret;
5555 u8 index;
5556 enum query_opcode opcode;
5557
5558 if (!ufshcd_is_wb_allowed(hba))
5559 return 0;
5560
5561 if (!(enable ^ hba->wb_enabled))
5562 return 0;
5563 if (enable)
5564 opcode = UPIU_QUERY_OPCODE_SET_FLAG;
5565 else
5566 opcode = UPIU_QUERY_OPCODE_CLEAR_FLAG;
5567
5568 index = ufshcd_wb_get_query_index(hba);
5569 ret = ufshcd_query_flag_retry(hba, opcode,
5570 QUERY_FLAG_IDN_WB_EN, index, NULL);
5571 if (ret) {
5572 dev_err(hba->dev, "%s write booster %s failed %d\n",
5573 __func__, enable ? "enable" : "disable", ret);
5574 return ret;
5575 }
5576
5577 hba->wb_enabled = enable;
5578 dev_dbg(hba->dev, "%s write booster %s %d\n",
5579 __func__, enable ? "enable" : "disable", ret);
5580
5581 return ret;
5582 }
5583
ufshcd_wb_toggle_flush_during_h8(struct ufs_hba * hba,bool set)5584 static int ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set)
5585 {
5586 int val;
5587 u8 index;
5588
5589 if (set)
5590 val = UPIU_QUERY_OPCODE_SET_FLAG;
5591 else
5592 val = UPIU_QUERY_OPCODE_CLEAR_FLAG;
5593
5594 index = ufshcd_wb_get_query_index(hba);
5595 return ufshcd_query_flag_retry(hba, val,
5596 QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8,
5597 index, NULL);
5598 }
5599
ufshcd_wb_toggle_flush(struct ufs_hba * hba,bool enable)5600 static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable)
5601 {
5602 if (enable)
5603 ufshcd_wb_buf_flush_enable(hba);
5604 else
5605 ufshcd_wb_buf_flush_disable(hba);
5606
5607 }
5608
ufshcd_wb_buf_flush_enable(struct ufs_hba * hba)5609 static int ufshcd_wb_buf_flush_enable(struct ufs_hba *hba)
5610 {
5611 int ret;
5612 u8 index;
5613
5614 if (!ufshcd_is_wb_allowed(hba) || hba->wb_buf_flush_enabled)
5615 return 0;
5616
5617 index = ufshcd_wb_get_query_index(hba);
5618 ret = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
5619 QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN,
5620 index, NULL);
5621 if (ret)
5622 dev_err(hba->dev, "%s WB - buf flush enable failed %d\n",
5623 __func__, ret);
5624 else
5625 hba->wb_buf_flush_enabled = true;
5626
5627 dev_dbg(hba->dev, "WB - Flush enabled: %d\n", ret);
5628 return ret;
5629 }
5630
ufshcd_wb_buf_flush_disable(struct ufs_hba * hba)5631 static int ufshcd_wb_buf_flush_disable(struct ufs_hba *hba)
5632 {
5633 int ret;
5634 u8 index;
5635
5636 if (!ufshcd_is_wb_allowed(hba) || !hba->wb_buf_flush_enabled)
5637 return 0;
5638
5639 index = ufshcd_wb_get_query_index(hba);
5640 ret = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
5641 QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN,
5642 index, NULL);
5643 if (ret) {
5644 dev_warn(hba->dev, "%s: WB - buf flush disable failed %d\n",
5645 __func__, ret);
5646 } else {
5647 hba->wb_buf_flush_enabled = false;
5648 dev_dbg(hba->dev, "WB - Flush disabled: %d\n", ret);
5649 }
5650
5651 return ret;
5652 }
5653
ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba * hba,u32 avail_buf)5654 static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba,
5655 u32 avail_buf)
5656 {
5657 u32 cur_buf;
5658 int ret;
5659 u8 index;
5660
5661 index = ufshcd_wb_get_query_index(hba);
5662 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5663 QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE,
5664 index, 0, &cur_buf);
5665 if (ret) {
5666 dev_err(hba->dev, "%s dCurWriteBoosterBufferSize read failed %d\n",
5667 __func__, ret);
5668 return false;
5669 }
5670
5671 if (!cur_buf) {
5672 dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n",
5673 cur_buf);
5674 return false;
5675 }
5676 /* Let it continue to flush when available buffer exceeds threshold */
5677 if (avail_buf < hba->vps->wb_flush_threshold)
5678 return true;
5679
5680 return false;
5681 }
5682
ufshcd_wb_need_flush(struct ufs_hba * hba)5683 static bool ufshcd_wb_need_flush(struct ufs_hba *hba)
5684 {
5685 int ret;
5686 u32 avail_buf;
5687 u8 index;
5688
5689 if (!ufshcd_is_wb_allowed(hba))
5690 return false;
5691 /*
5692 * The ufs device needs the vcc to be ON to flush.
5693 * With user-space reduction enabled, it's enough to enable flush
5694 * by checking only the available buffer. The threshold
5695 * defined here is > 90% full.
5696 * With user-space preserved enabled, the current-buffer
5697 * should be checked too because the wb buffer size can reduce
5698 * when disk tends to be full. This info is provided by current
5699 * buffer (dCurrentWriteBoosterBufferSize). There's no point in
5700 * keeping vcc on when current buffer is empty.
5701 */
5702 index = ufshcd_wb_get_query_index(hba);
5703 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5704 QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE,
5705 index, 0, &avail_buf);
5706 if (ret) {
5707 dev_warn(hba->dev, "%s dAvailableWriteBoosterBufferSize read failed %d\n",
5708 __func__, ret);
5709 return false;
5710 }
5711
5712 if (!hba->dev_info.b_presrv_uspc_en) {
5713 if (avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10))
5714 return true;
5715 return false;
5716 }
5717
5718 return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf);
5719 }
5720
ufshcd_rpm_dev_flush_recheck_work(struct work_struct * work)5721 static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work)
5722 {
5723 struct ufs_hba *hba = container_of(to_delayed_work(work),
5724 struct ufs_hba,
5725 rpm_dev_flush_recheck_work);
5726 /*
5727 * To prevent unnecessary VCC power drain after device finishes
5728 * WriteBooster buffer flush or Auto BKOPs, force runtime resume
5729 * after a certain delay to recheck the threshold by next runtime
5730 * suspend.
5731 */
5732 pm_runtime_get_sync(hba->dev);
5733 pm_runtime_put_sync(hba->dev);
5734 }
5735
5736 /**
5737 * ufshcd_exception_event_handler - handle exceptions raised by device
5738 * @work: pointer to work data
5739 *
5740 * Read bExceptionEventStatus attribute from the device and handle the
5741 * exception event accordingly.
5742 */
ufshcd_exception_event_handler(struct work_struct * work)5743 static void ufshcd_exception_event_handler(struct work_struct *work)
5744 {
5745 struct ufs_hba *hba;
5746 int err;
5747 u32 status = 0;
5748 hba = container_of(work, struct ufs_hba, eeh_work);
5749
5750 pm_runtime_get_sync(hba->dev);
5751 ufshcd_scsi_block_requests(hba);
5752 err = ufshcd_get_ee_status(hba, &status);
5753 if (err) {
5754 dev_err(hba->dev, "%s: failed to get exception status %d\n",
5755 __func__, err);
5756 goto out;
5757 }
5758
5759 status &= hba->ee_ctrl_mask;
5760
5761 if (status & MASK_EE_URGENT_BKOPS)
5762 ufshcd_bkops_exception_event_handler(hba);
5763
5764 out:
5765 ufshcd_scsi_unblock_requests(hba);
5766 /*
5767 * pm_runtime_get_noresume is called while scheduling
5768 * eeh_work to avoid suspend racing with exception work.
5769 * Hence decrement usage counter using pm_runtime_put_noidle
5770 * to allow suspend on completion of exception event handler.
5771 */
5772 pm_runtime_put_noidle(hba->dev);
5773 pm_runtime_put(hba->dev);
5774 return;
5775 }
5776
5777 /* Complete requests that have door-bell cleared */
ufshcd_complete_requests(struct ufs_hba * hba)5778 static void ufshcd_complete_requests(struct ufs_hba *hba)
5779 {
5780 ufshcd_trc_handler(hba, false);
5781 ufshcd_tmc_handler(hba);
5782 }
5783
5784 /**
5785 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
5786 * to recover from the DL NAC errors or not.
5787 * @hba: per-adapter instance
5788 *
5789 * Returns true if error handling is required, false otherwise
5790 */
ufshcd_quirk_dl_nac_errors(struct ufs_hba * hba)5791 static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
5792 {
5793 unsigned long flags;
5794 bool err_handling = true;
5795
5796 spin_lock_irqsave(hba->host->host_lock, flags);
5797 /*
5798 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
5799 * device fatal error and/or DL NAC & REPLAY timeout errors.
5800 */
5801 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
5802 goto out;
5803
5804 if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
5805 ((hba->saved_err & UIC_ERROR) &&
5806 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
5807 goto out;
5808
5809 if ((hba->saved_err & UIC_ERROR) &&
5810 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
5811 int err;
5812 /*
5813 * wait for 50ms to see if we can get any other errors or not.
5814 */
5815 spin_unlock_irqrestore(hba->host->host_lock, flags);
5816 msleep(50);
5817 spin_lock_irqsave(hba->host->host_lock, flags);
5818
5819 /*
5820 * now check if we have got any other severe errors other than
5821 * DL NAC error?
5822 */
5823 if ((hba->saved_err & INT_FATAL_ERRORS) ||
5824 ((hba->saved_err & UIC_ERROR) &&
5825 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
5826 goto out;
5827
5828 /*
5829 * As DL NAC is the only error received so far, send out NOP
5830 * command to confirm if link is still active or not.
5831 * - If we don't get any response then do error recovery.
5832 * - If we get response then clear the DL NAC error bit.
5833 */
5834
5835 spin_unlock_irqrestore(hba->host->host_lock, flags);
5836 err = ufshcd_verify_dev_init(hba);
5837 spin_lock_irqsave(hba->host->host_lock, flags);
5838
5839 if (err)
5840 goto out;
5841
5842 /* Link seems to be alive hence ignore the DL NAC errors */
5843 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
5844 hba->saved_err &= ~UIC_ERROR;
5845 /* clear NAC error */
5846 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5847 if (!hba->saved_uic_err)
5848 err_handling = false;
5849 }
5850 out:
5851 spin_unlock_irqrestore(hba->host->host_lock, flags);
5852 return err_handling;
5853 }
5854
5855 /* host lock must be held before calling this func */
ufshcd_is_saved_err_fatal(struct ufs_hba * hba)5856 static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba)
5857 {
5858 return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) ||
5859 (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK));
5860 }
5861
5862 /* host lock must be held before calling this func */
ufshcd_schedule_eh_work(struct ufs_hba * hba)5863 static inline void ufshcd_schedule_eh_work(struct ufs_hba *hba)
5864 {
5865 /* handle fatal errors only when link is not in error state */
5866 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) {
5867 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
5868 ufshcd_is_saved_err_fatal(hba))
5869 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL;
5870 else
5871 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL;
5872 queue_work(hba->eh_wq, &hba->eh_work);
5873 }
5874 }
5875
ufshcd_clk_scaling_allow(struct ufs_hba * hba,bool allow)5876 static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow)
5877 {
5878 down_write(&hba->clk_scaling_lock);
5879 hba->clk_scaling.is_allowed = allow;
5880 up_write(&hba->clk_scaling_lock);
5881 }
5882
ufshcd_clk_scaling_suspend(struct ufs_hba * hba,bool suspend)5883 static void ufshcd_clk_scaling_suspend(struct ufs_hba *hba, bool suspend)
5884 {
5885 if (suspend) {
5886 if (hba->clk_scaling.is_enabled)
5887 ufshcd_suspend_clkscaling(hba);
5888 ufshcd_clk_scaling_allow(hba, false);
5889 } else {
5890 ufshcd_clk_scaling_allow(hba, true);
5891 if (hba->clk_scaling.is_enabled)
5892 ufshcd_resume_clkscaling(hba);
5893 }
5894 }
5895
ufshcd_err_handling_prepare(struct ufs_hba * hba)5896 static void ufshcd_err_handling_prepare(struct ufs_hba *hba)
5897 {
5898 pm_runtime_get_sync(hba->dev);
5899 if (pm_runtime_status_suspended(hba->dev) || hba->is_sys_suspended) {
5900 enum ufs_pm_op pm_op;
5901
5902 /*
5903 * Don't assume anything of pm_runtime_get_sync(), if
5904 * resume fails, irq and clocks can be OFF, and powers
5905 * can be OFF or in LPM.
5906 */
5907 ufshcd_setup_hba_vreg(hba, true);
5908 ufshcd_enable_irq(hba);
5909 ufshcd_setup_vreg(hba, true);
5910 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
5911 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
5912 ufshcd_hold(hba, false);
5913 if (!ufshcd_is_clkgating_allowed(hba))
5914 ufshcd_setup_clocks(hba, true);
5915 ufshcd_release(hba);
5916 pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM;
5917 ufshcd_vops_resume(hba, pm_op);
5918 } else {
5919 ufshcd_hold(hba, false);
5920 if (ufshcd_is_clkscaling_supported(hba) &&
5921 hba->clk_scaling.is_enabled)
5922 ufshcd_suspend_clkscaling(hba);
5923 ufshcd_clk_scaling_allow(hba, false);
5924 }
5925 ufshcd_scsi_block_requests(hba);
5926 /* Drain ufshcd_queuecommand() */
5927 down_write(&hba->clk_scaling_lock);
5928 up_write(&hba->clk_scaling_lock);
5929 cancel_work_sync(&hba->eeh_work);
5930 }
5931
ufshcd_err_handling_unprepare(struct ufs_hba * hba)5932 static void ufshcd_err_handling_unprepare(struct ufs_hba *hba)
5933 {
5934 ufshcd_scsi_unblock_requests(hba);
5935 ufshcd_release(hba);
5936 if (ufshcd_is_clkscaling_supported(hba))
5937 ufshcd_clk_scaling_suspend(hba, false);
5938 pm_runtime_put(hba->dev);
5939 }
5940
ufshcd_err_handling_should_stop(struct ufs_hba * hba)5941 static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba)
5942 {
5943 return (!hba->is_powered || hba->shutting_down ||
5944 hba->ufshcd_state == UFSHCD_STATE_ERROR ||
5945 (!(hba->saved_err || hba->saved_uic_err || hba->force_reset ||
5946 ufshcd_is_link_broken(hba))));
5947 }
5948
5949 #ifdef CONFIG_PM
ufshcd_recover_pm_error(struct ufs_hba * hba)5950 static void ufshcd_recover_pm_error(struct ufs_hba *hba)
5951 {
5952 struct Scsi_Host *shost = hba->host;
5953 struct scsi_device *sdev;
5954 struct request_queue *q;
5955 int ret;
5956
5957 hba->is_sys_suspended = false;
5958 /*
5959 * Set RPM status of hba device to RPM_ACTIVE,
5960 * this also clears its runtime error.
5961 */
5962 ret = pm_runtime_set_active(hba->dev);
5963 /*
5964 * If hba device had runtime error, we also need to resume those
5965 * scsi devices under hba in case any of them has failed to be
5966 * resumed due to hba runtime resume failure. This is to unblock
5967 * blk_queue_enter in case there are bios waiting inside it.
5968 */
5969 if (!ret) {
5970 shost_for_each_device(sdev, shost) {
5971 q = sdev->request_queue;
5972 if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
5973 q->rpm_status == RPM_SUSPENDING))
5974 pm_request_resume(q->dev);
5975 }
5976 }
5977 }
5978 #else
ufshcd_recover_pm_error(struct ufs_hba * hba)5979 static inline void ufshcd_recover_pm_error(struct ufs_hba *hba)
5980 {
5981 }
5982 #endif
5983
ufshcd_is_pwr_mode_restore_needed(struct ufs_hba * hba)5984 static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba)
5985 {
5986 struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info;
5987 u32 mode;
5988
5989 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode);
5990
5991 if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK))
5992 return true;
5993
5994 if (pwr_info->pwr_tx != (mode & PWRMODE_MASK))
5995 return true;
5996
5997 return false;
5998 }
5999
6000 /**
6001 * ufshcd_err_handler - handle UFS errors that require s/w attention
6002 * @work: pointer to work structure
6003 */
ufshcd_err_handler(struct work_struct * work)6004 static void ufshcd_err_handler(struct work_struct *work)
6005 {
6006 struct ufs_hba *hba;
6007 unsigned long flags;
6008 bool err_xfer = false;
6009 bool err_tm = false;
6010 int err = 0, pmc_err;
6011 int tag;
6012 bool needs_reset = false, needs_restore = false;
6013
6014 hba = container_of(work, struct ufs_hba, eh_work);
6015
6016 down(&hba->host_sem);
6017 spin_lock_irqsave(hba->host->host_lock, flags);
6018 if (ufshcd_err_handling_should_stop(hba)) {
6019 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6020 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6021 spin_unlock_irqrestore(hba->host->host_lock, flags);
6022 up(&hba->host_sem);
6023 return;
6024 }
6025 ufshcd_set_eh_in_progress(hba);
6026 spin_unlock_irqrestore(hba->host->host_lock, flags);
6027 ufshcd_err_handling_prepare(hba);
6028 /* Complete requests that have door-bell cleared by h/w */
6029 ufshcd_complete_requests(hba);
6030 spin_lock_irqsave(hba->host->host_lock, flags);
6031 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6032 hba->ufshcd_state = UFSHCD_STATE_RESET;
6033 /*
6034 * A full reset and restore might have happened after preparation
6035 * is finished, double check whether we should stop.
6036 */
6037 if (ufshcd_err_handling_should_stop(hba))
6038 goto skip_err_handling;
6039
6040 if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
6041 bool ret;
6042
6043 spin_unlock_irqrestore(hba->host->host_lock, flags);
6044 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
6045 ret = ufshcd_quirk_dl_nac_errors(hba);
6046 spin_lock_irqsave(hba->host->host_lock, flags);
6047 if (!ret && ufshcd_err_handling_should_stop(hba))
6048 goto skip_err_handling;
6049 }
6050
6051 if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6052 (hba->saved_uic_err &&
6053 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6054 bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR);
6055
6056 spin_unlock_irqrestore(hba->host->host_lock, flags);
6057 ufshcd_print_host_state(hba);
6058 ufshcd_print_pwr_info(hba);
6059 ufshcd_print_evt_hist(hba);
6060 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
6061 ufshcd_print_trs(hba, hba->outstanding_reqs, pr_prdt);
6062 spin_lock_irqsave(hba->host->host_lock, flags);
6063 }
6064
6065 /*
6066 * if host reset is required then skip clearing the pending
6067 * transfers forcefully because they will get cleared during
6068 * host reset and restore
6069 */
6070 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6071 ufshcd_is_saved_err_fatal(hba) ||
6072 ((hba->saved_err & UIC_ERROR) &&
6073 (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
6074 UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))) {
6075 needs_reset = true;
6076 goto do_reset;
6077 }
6078
6079 /*
6080 * If LINERESET was caught, UFS might have been put to PWM mode,
6081 * check if power mode restore is needed.
6082 */
6083 if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) {
6084 hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6085 if (!hba->saved_uic_err)
6086 hba->saved_err &= ~UIC_ERROR;
6087 spin_unlock_irqrestore(hba->host->host_lock, flags);
6088 if (ufshcd_is_pwr_mode_restore_needed(hba))
6089 needs_restore = true;
6090 spin_lock_irqsave(hba->host->host_lock, flags);
6091 if (!hba->saved_err && !needs_restore)
6092 goto skip_err_handling;
6093 }
6094
6095 hba->silence_err_logs = true;
6096 /* release lock as clear command might sleep */
6097 spin_unlock_irqrestore(hba->host->host_lock, flags);
6098 /* Clear pending transfer requests */
6099 for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
6100 if (ufshcd_try_to_abort_task(hba, tag)) {
6101 err_xfer = true;
6102 goto lock_skip_pending_xfer_clear;
6103 }
6104 }
6105
6106 /* Clear pending task management requests */
6107 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
6108 if (ufshcd_clear_tm_cmd(hba, tag)) {
6109 err_tm = true;
6110 goto lock_skip_pending_xfer_clear;
6111 }
6112 }
6113
6114 lock_skip_pending_xfer_clear:
6115 /* Complete the requests that are cleared by s/w */
6116 ufshcd_complete_requests(hba);
6117
6118 spin_lock_irqsave(hba->host->host_lock, flags);
6119 hba->silence_err_logs = false;
6120 if (err_xfer || err_tm) {
6121 needs_reset = true;
6122 goto do_reset;
6123 }
6124
6125 /*
6126 * After all reqs and tasks are cleared from doorbell,
6127 * now it is safe to retore power mode.
6128 */
6129 if (needs_restore) {
6130 spin_unlock_irqrestore(hba->host->host_lock, flags);
6131 /*
6132 * Hold the scaling lock just in case dev cmds
6133 * are sent via bsg and/or sysfs.
6134 */
6135 down_write(&hba->clk_scaling_lock);
6136 hba->force_pmc = true;
6137 pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info));
6138 if (pmc_err) {
6139 needs_reset = true;
6140 dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n",
6141 __func__, pmc_err);
6142 }
6143 hba->force_pmc = false;
6144 ufshcd_print_pwr_info(hba);
6145 up_write(&hba->clk_scaling_lock);
6146 spin_lock_irqsave(hba->host->host_lock, flags);
6147 }
6148
6149 do_reset:
6150 /* Fatal errors need reset */
6151 if (needs_reset) {
6152 hba->force_reset = false;
6153 spin_unlock_irqrestore(hba->host->host_lock, flags);
6154 err = ufshcd_reset_and_restore(hba);
6155 if (err)
6156 dev_err(hba->dev, "%s: reset and restore failed with err %d\n",
6157 __func__, err);
6158 else
6159 ufshcd_recover_pm_error(hba);
6160 spin_lock_irqsave(hba->host->host_lock, flags);
6161 }
6162
6163 skip_err_handling:
6164 if (!needs_reset) {
6165 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
6166 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6167 if (hba->saved_err || hba->saved_uic_err)
6168 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
6169 __func__, hba->saved_err, hba->saved_uic_err);
6170 }
6171 ufshcd_clear_eh_in_progress(hba);
6172 spin_unlock_irqrestore(hba->host->host_lock, flags);
6173 ufshcd_err_handling_unprepare(hba);
6174 up(&hba->host_sem);
6175 }
6176
6177 /**
6178 * ufshcd_update_uic_error - check and set fatal UIC error flags.
6179 * @hba: per-adapter instance
6180 *
6181 * Returns
6182 * IRQ_HANDLED - If interrupt is valid
6183 * IRQ_NONE - If invalid interrupt
6184 */
ufshcd_update_uic_error(struct ufs_hba * hba)6185 static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
6186 {
6187 u32 reg;
6188 irqreturn_t retval = IRQ_NONE;
6189
6190 /* PHY layer error */
6191 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
6192 if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
6193 (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) {
6194 ufshcd_update_evt_hist(hba, UFS_EVT_PA_ERR, reg);
6195 /*
6196 * To know whether this error is fatal or not, DB timeout
6197 * must be checked but this error is handled separately.
6198 */
6199 if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)
6200 dev_dbg(hba->dev, "%s: UIC Lane error reported\n",
6201 __func__);
6202
6203 /* Got a LINERESET indication. */
6204 if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) {
6205 struct uic_command *cmd = NULL;
6206
6207 hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR;
6208 if (hba->uic_async_done && hba->active_uic_cmd)
6209 cmd = hba->active_uic_cmd;
6210 /*
6211 * Ignore the LINERESET during power mode change
6212 * operation via DME_SET command.
6213 */
6214 if (cmd && (cmd->command == UIC_CMD_DME_SET))
6215 hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6216 }
6217 retval |= IRQ_HANDLED;
6218 }
6219
6220 /* PA_INIT_ERROR is fatal and needs UIC reset */
6221 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
6222 if ((reg & UIC_DATA_LINK_LAYER_ERROR) &&
6223 (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) {
6224 ufshcd_update_evt_hist(hba, UFS_EVT_DL_ERR, reg);
6225
6226 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
6227 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
6228 else if (hba->dev_quirks &
6229 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
6230 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
6231 hba->uic_error |=
6232 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
6233 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
6234 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
6235 }
6236 retval |= IRQ_HANDLED;
6237 }
6238
6239 /* UIC NL/TL/DME errors needs software retry */
6240 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
6241 if ((reg & UIC_NETWORK_LAYER_ERROR) &&
6242 (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) {
6243 ufshcd_update_evt_hist(hba, UFS_EVT_NL_ERR, reg);
6244 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
6245 retval |= IRQ_HANDLED;
6246 }
6247
6248 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
6249 if ((reg & UIC_TRANSPORT_LAYER_ERROR) &&
6250 (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) {
6251 ufshcd_update_evt_hist(hba, UFS_EVT_TL_ERR, reg);
6252 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
6253 retval |= IRQ_HANDLED;
6254 }
6255
6256 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
6257 if ((reg & UIC_DME_ERROR) &&
6258 (reg & UIC_DME_ERROR_CODE_MASK)) {
6259 ufshcd_update_evt_hist(hba, UFS_EVT_DME_ERR, reg);
6260 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
6261 retval |= IRQ_HANDLED;
6262 }
6263
6264 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
6265 __func__, hba->uic_error);
6266 return retval;
6267 }
6268
6269 /**
6270 * ufshcd_check_errors - Check for errors that need s/w attention
6271 * @hba: per-adapter instance
6272 * @intr_status: interrupt status generated by the controller
6273 *
6274 * Returns
6275 * IRQ_HANDLED - If interrupt is valid
6276 * IRQ_NONE - If invalid interrupt
6277 */
ufshcd_check_errors(struct ufs_hba * hba,u32 intr_status)6278 static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status)
6279 {
6280 bool queue_eh_work = false;
6281 irqreturn_t retval = IRQ_NONE;
6282
6283 spin_lock(hba->host->host_lock);
6284 hba->errors |= UFSHCD_ERROR_MASK & intr_status;
6285
6286 if (hba->errors & INT_FATAL_ERRORS) {
6287 ufshcd_update_evt_hist(hba, UFS_EVT_FATAL_ERR,
6288 hba->errors);
6289 queue_eh_work = true;
6290 }
6291
6292 if (hba->errors & UIC_ERROR) {
6293 hba->uic_error = 0;
6294 retval = ufshcd_update_uic_error(hba);
6295 if (hba->uic_error) {
6296 dev_err(hba->dev,
6297 "Scheduling error handler because of an UIC error\n");
6298 queue_eh_work = true;
6299 }
6300 }
6301
6302 if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
6303 dev_err(hba->dev,
6304 "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
6305 __func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
6306 "Enter" : "Exit",
6307 hba->errors, ufshcd_get_upmcrs(hba));
6308 ufshcd_update_evt_hist(hba, UFS_EVT_AUTO_HIBERN8_ERR,
6309 hba->errors);
6310 ufshcd_set_link_broken(hba);
6311 queue_eh_work = true;
6312 }
6313
6314 trace_android_vh_ufs_check_int_errors(hba, queue_eh_work);
6315
6316 if (queue_eh_work) {
6317 /*
6318 * update the transfer error masks to sticky bits, let's do this
6319 * irrespective of current ufshcd_state.
6320 */
6321 hba->saved_err |= hba->errors;
6322 hba->saved_uic_err |= hba->uic_error;
6323
6324 /* dump controller state before resetting */
6325 if ((hba->saved_err &
6326 (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6327 (hba->saved_uic_err &&
6328 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6329 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
6330 __func__, hba->saved_err,
6331 hba->saved_uic_err);
6332 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE,
6333 "host_regs: ");
6334 ufshcd_print_pwr_info(hba);
6335 }
6336 ufshcd_schedule_eh_work(hba);
6337 retval |= IRQ_HANDLED;
6338 }
6339 /*
6340 * if (!queue_eh_work) -
6341 * Other errors are either non-fatal where host recovers
6342 * itself without s/w intervention or errors that will be
6343 * handled by the SCSI core layer.
6344 */
6345 hba->errors = 0;
6346 hba->uic_error = 0;
6347 spin_unlock(hba->host->host_lock);
6348 return retval;
6349 }
6350
6351 /**
6352 * ufshcd_tmc_handler - handle task management function completion
6353 * @hba: per adapter instance
6354 *
6355 * Returns
6356 * IRQ_HANDLED - If interrupt is valid
6357 * IRQ_NONE - If invalid interrupt
6358 */
ufshcd_tmc_handler(struct ufs_hba * hba)6359 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
6360 {
6361 struct request **tmf_rqs = ufs_hba_add_info(hba)->tmf_rqs;
6362 unsigned long flags, pending, issued;
6363 irqreturn_t ret = IRQ_NONE;
6364 int tag;
6365
6366 spin_lock_irqsave(hba->host->host_lock, flags);
6367 pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
6368 issued = hba->outstanding_tasks & ~pending;
6369 for_each_set_bit(tag, &issued, hba->nutmrs) {
6370 struct request *req = tmf_rqs[tag];
6371 struct completion *c = req->end_io_data;
6372
6373 complete(c);
6374 ret = IRQ_HANDLED;
6375 }
6376 spin_unlock_irqrestore(hba->host->host_lock, flags);
6377
6378 return ret;
6379 }
6380
6381 /**
6382 * ufshcd_sl_intr - Interrupt service routine
6383 * @hba: per adapter instance
6384 * @intr_status: contains interrupts generated by the controller
6385 *
6386 * Returns
6387 * IRQ_HANDLED - If interrupt is valid
6388 * IRQ_NONE - If invalid interrupt
6389 */
ufshcd_sl_intr(struct ufs_hba * hba,u32 intr_status)6390 static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
6391 {
6392 irqreturn_t retval = IRQ_NONE;
6393
6394 if (intr_status & UFSHCD_UIC_MASK)
6395 retval |= ufshcd_uic_cmd_compl(hba, intr_status);
6396
6397 if (intr_status & UFSHCD_ERROR_MASK || hba->errors)
6398 retval |= ufshcd_check_errors(hba, intr_status);
6399
6400 if (intr_status & UTP_TASK_REQ_COMPL)
6401 retval |= ufshcd_tmc_handler(hba);
6402
6403 if (intr_status & UTP_TRANSFER_REQ_COMPL)
6404 retval |= ufshcd_trc_handler(hba, ufshcd_has_utrlcnr(hba));
6405
6406 return retval;
6407 }
6408
6409 /**
6410 * ufshcd_intr - Main interrupt service routine
6411 * @irq: irq number
6412 * @__hba: pointer to adapter instance
6413 *
6414 * Returns
6415 * IRQ_HANDLED - If interrupt is valid
6416 * IRQ_NONE - If invalid interrupt
6417 */
ufshcd_intr(int irq,void * __hba)6418 static irqreturn_t ufshcd_intr(int irq, void *__hba)
6419 {
6420 u32 intr_status, enabled_intr_status = 0;
6421 irqreturn_t retval = IRQ_NONE;
6422 struct ufs_hba *hba = __hba;
6423 int retries = hba->nutrs;
6424
6425 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6426 hba->ufs_stats.last_intr_status = intr_status;
6427 hba->ufs_stats.last_intr_ts = ktime_get();
6428
6429 /*
6430 * There could be max of hba->nutrs reqs in flight and in worst case
6431 * if the reqs get finished 1 by 1 after the interrupt status is
6432 * read, make sure we handle them by checking the interrupt status
6433 * again in a loop until we process all of the reqs before returning.
6434 */
6435 while (intr_status && retries--) {
6436 enabled_intr_status =
6437 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
6438 if (intr_status)
6439 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
6440 if (enabled_intr_status)
6441 retval |= ufshcd_sl_intr(hba, enabled_intr_status);
6442
6443 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6444 }
6445
6446 if (enabled_intr_status && retval == IRQ_NONE &&
6447 !ufshcd_eh_in_progress(hba)) {
6448 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n",
6449 __func__,
6450 intr_status,
6451 hba->ufs_stats.last_intr_status,
6452 enabled_intr_status);
6453 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
6454 }
6455
6456 return retval;
6457 }
6458
ufshcd_clear_tm_cmd(struct ufs_hba * hba,int tag)6459 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
6460 {
6461 int err = 0;
6462 u32 mask = 1 << tag;
6463 unsigned long flags;
6464
6465 if (!test_bit(tag, &hba->outstanding_tasks))
6466 goto out;
6467
6468 spin_lock_irqsave(hba->host->host_lock, flags);
6469 ufshcd_utmrl_clear(hba, tag);
6470 spin_unlock_irqrestore(hba->host->host_lock, flags);
6471
6472 /* poll for max. 1 sec to clear door bell register by h/w */
6473 err = ufshcd_wait_for_register(hba,
6474 REG_UTP_TASK_REQ_DOOR_BELL,
6475 mask, 0, 1000, 1000);
6476 out:
6477 return err;
6478 }
6479
__ufshcd_issue_tm_cmd(struct ufs_hba * hba,struct utp_task_req_desc * treq,u8 tm_function)6480 static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
6481 struct utp_task_req_desc *treq, u8 tm_function)
6482 {
6483 struct request **tmf_rqs = ufs_hba_add_info(hba)->tmf_rqs;
6484 struct request_queue *q = hba->tmf_queue;
6485 struct Scsi_Host *host = hba->host;
6486 DECLARE_COMPLETION_ONSTACK(wait);
6487 struct request *req;
6488 unsigned long flags;
6489 int task_tag, err;
6490
6491 /*
6492 * blk_get_request() is used here only to get a free tag.
6493 */
6494 req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
6495 if (IS_ERR(req))
6496 return PTR_ERR(req);
6497
6498 req->end_io_data = &wait;
6499 ufshcd_hold(hba, false);
6500
6501 spin_lock_irqsave(host->host_lock, flags);
6502
6503 task_tag = req->tag;
6504 tmf_rqs[req->tag] = req;
6505 treq->req_header.dword_0 |= cpu_to_be32(task_tag);
6506
6507 memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq));
6508 ufshcd_vops_setup_task_mgmt(hba, task_tag, tm_function);
6509
6510 /* send command to the controller */
6511 __set_bit(task_tag, &hba->outstanding_tasks);
6512
6513 /* Make sure descriptors are ready before ringing the task doorbell */
6514 wmb();
6515
6516 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL);
6517 /* Make sure that doorbell is committed immediately */
6518 wmb();
6519
6520 spin_unlock_irqrestore(host->host_lock, flags);
6521
6522 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_send");
6523
6524 /* wait until the task management command is completed */
6525 err = wait_for_completion_io_timeout(&wait,
6526 msecs_to_jiffies(TM_CMD_TIMEOUT));
6527 if (!err) {
6528 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete_err");
6529 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
6530 __func__, tm_function);
6531 if (ufshcd_clear_tm_cmd(hba, task_tag))
6532 dev_WARN(hba->dev, "%s: unable to clear tm cmd (slot %d) after timeout\n",
6533 __func__, task_tag);
6534 err = -ETIMEDOUT;
6535 } else {
6536 err = 0;
6537 memcpy(treq, hba->utmrdl_base_addr + task_tag, sizeof(*treq));
6538
6539 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete");
6540 }
6541
6542 spin_lock_irqsave(hba->host->host_lock, flags);
6543 tmf_rqs[req->tag] = NULL;
6544 __clear_bit(task_tag, &hba->outstanding_tasks);
6545 spin_unlock_irqrestore(hba->host->host_lock, flags);
6546
6547 ufshcd_release(hba);
6548 blk_put_request(req);
6549
6550 return err;
6551 }
6552
6553 /**
6554 * ufshcd_issue_tm_cmd - issues task management commands to controller
6555 * @hba: per adapter instance
6556 * @lun_id: LUN ID to which TM command is sent
6557 * @task_id: task ID to which the TM command is applicable
6558 * @tm_function: task management function opcode
6559 * @tm_response: task management service response return value
6560 *
6561 * Returns non-zero value on error, zero on success.
6562 */
ufshcd_issue_tm_cmd(struct ufs_hba * hba,int lun_id,int task_id,u8 tm_function,u8 * tm_response)6563 static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
6564 u8 tm_function, u8 *tm_response)
6565 {
6566 struct utp_task_req_desc treq = { { 0 }, };
6567 int ocs_value, err;
6568
6569 /* Configure task request descriptor */
6570 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6571 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6572
6573 /* Configure task request UPIU */
6574 treq.req_header.dword_0 = cpu_to_be32(lun_id << 8) |
6575 cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24);
6576 treq.req_header.dword_1 = cpu_to_be32(tm_function << 16);
6577
6578 /*
6579 * The host shall provide the same value for LUN field in the basic
6580 * header and for Input Parameter.
6581 */
6582 treq.input_param1 = cpu_to_be32(lun_id);
6583 treq.input_param2 = cpu_to_be32(task_id);
6584
6585 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
6586 if (err == -ETIMEDOUT)
6587 return err;
6588
6589 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6590 if (ocs_value != OCS_SUCCESS)
6591 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
6592 __func__, ocs_value);
6593 else if (tm_response)
6594 *tm_response = be32_to_cpu(treq.output_param1) &
6595 MASK_TM_SERVICE_RESP;
6596 return err;
6597 }
6598
6599 /**
6600 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
6601 * @hba: per-adapter instance
6602 * @req_upiu: upiu request
6603 * @rsp_upiu: upiu reply
6604 * @desc_buff: pointer to descriptor buffer, NULL if NA
6605 * @buff_len: descriptor size, 0 if NA
6606 * @cmd_type: specifies the type (NOP, Query...)
6607 * @desc_op: descriptor operation
6608 *
6609 * Those type of requests uses UTP Transfer Request Descriptor - utrd.
6610 * Therefore, it "rides" the device management infrastructure: uses its tag and
6611 * tasks work queues.
6612 *
6613 * Since there is only one available tag for device management commands,
6614 * the caller is expected to hold the hba->dev_cmd.lock mutex.
6615 */
ufshcd_issue_devman_upiu_cmd(struct ufs_hba * hba,struct utp_upiu_req * req_upiu,struct utp_upiu_req * rsp_upiu,u8 * desc_buff,int * buff_len,enum dev_cmd_type cmd_type,enum query_opcode desc_op)6616 static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
6617 struct utp_upiu_req *req_upiu,
6618 struct utp_upiu_req *rsp_upiu,
6619 u8 *desc_buff, int *buff_len,
6620 enum dev_cmd_type cmd_type,
6621 enum query_opcode desc_op)
6622 {
6623 DECLARE_COMPLETION_ONSTACK(wait);
6624 const u32 tag = ufs_hba_add_info(hba)->reserved_slot;
6625 struct ufshcd_lrb *lrbp;
6626 int err = 0;
6627 u8 upiu_flags;
6628
6629 /* Protects use of ufs_hba_add_info(hba)->reserved_slot. */
6630 lockdep_assert_held(&hba->dev_cmd.lock);
6631
6632 down_read(&hba->clk_scaling_lock);
6633
6634 lrbp = &hba->lrb[tag];
6635 WARN_ON(lrbp->cmd);
6636 lrbp->cmd = NULL;
6637 lrbp->sense_bufflen = 0;
6638 lrbp->sense_buffer = NULL;
6639 lrbp->task_tag = tag;
6640 lrbp->lun = 0;
6641 lrbp->intr_cmd = true;
6642 ufshcd_prepare_lrbp_crypto(NULL, lrbp);
6643 hba->dev_cmd.type = cmd_type;
6644
6645 if (hba->ufs_version <= ufshci_version(1, 1))
6646 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
6647 else
6648 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
6649
6650 /* update the task tag in the request upiu */
6651 req_upiu->header.dword_0 |= cpu_to_be32(tag);
6652
6653 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
6654
6655 /* just copy the upiu request as it is */
6656 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
6657 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
6658 /* The Data Segment Area is optional depending upon the query
6659 * function value. for WRITE DESCRIPTOR, the data segment
6660 * follows right after the tsf.
6661 */
6662 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
6663 *buff_len = 0;
6664 }
6665
6666 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
6667
6668 hba->dev_cmd.complete = &wait;
6669
6670 /* Make sure descriptors are ready before ringing the doorbell */
6671 wmb();
6672
6673 ufshcd_send_command(hba, tag);
6674 /*
6675 * ignore the returning value here - ufshcd_check_query_response is
6676 * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
6677 * read the response directly ignoring all errors.
6678 */
6679 ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT);
6680
6681 /* just copy the upiu response as it is */
6682 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
6683 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
6684 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
6685 u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
6686 MASK_QUERY_DATA_SEG_LEN;
6687
6688 if (*buff_len >= resp_len) {
6689 memcpy(desc_buff, descp, resp_len);
6690 *buff_len = resp_len;
6691 } else {
6692 dev_warn(hba->dev,
6693 "%s: rsp size %d is bigger than buffer size %d",
6694 __func__, resp_len, *buff_len);
6695 *buff_len = 0;
6696 err = -EINVAL;
6697 }
6698 }
6699
6700 up_read(&hba->clk_scaling_lock);
6701 return err;
6702 }
6703
6704 /**
6705 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
6706 * @hba: per-adapter instance
6707 * @req_upiu: upiu request
6708 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands
6709 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target
6710 * @desc_buff: pointer to descriptor buffer, NULL if NA
6711 * @buff_len: descriptor size, 0 if NA
6712 * @desc_op: descriptor operation
6713 *
6714 * Supports UTP Transfer requests (nop and query), and UTP Task
6715 * Management requests.
6716 * It is up to the caller to fill the upiu conent properly, as it will
6717 * be copied without any further input validations.
6718 */
ufshcd_exec_raw_upiu_cmd(struct ufs_hba * hba,struct utp_upiu_req * req_upiu,struct utp_upiu_req * rsp_upiu,int msgcode,u8 * desc_buff,int * buff_len,enum query_opcode desc_op)6719 int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
6720 struct utp_upiu_req *req_upiu,
6721 struct utp_upiu_req *rsp_upiu,
6722 int msgcode,
6723 u8 *desc_buff, int *buff_len,
6724 enum query_opcode desc_op)
6725 {
6726 int err;
6727 enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY;
6728 struct utp_task_req_desc treq = { { 0 }, };
6729 int ocs_value;
6730 u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC;
6731
6732 switch (msgcode) {
6733 case UPIU_TRANSACTION_NOP_OUT:
6734 cmd_type = DEV_CMD_TYPE_NOP;
6735 fallthrough;
6736 case UPIU_TRANSACTION_QUERY_REQ:
6737 ufshcd_hold(hba, false);
6738 mutex_lock(&hba->dev_cmd.lock);
6739 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
6740 desc_buff, buff_len,
6741 cmd_type, desc_op);
6742 mutex_unlock(&hba->dev_cmd.lock);
6743 ufshcd_release(hba);
6744
6745 break;
6746 case UPIU_TRANSACTION_TASK_REQ:
6747 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6748 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6749
6750 memcpy(&treq.req_header, req_upiu, sizeof(*req_upiu));
6751
6752 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
6753 if (err == -ETIMEDOUT)
6754 break;
6755
6756 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6757 if (ocs_value != OCS_SUCCESS) {
6758 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
6759 ocs_value);
6760 break;
6761 }
6762
6763 memcpy(rsp_upiu, &treq.rsp_header, sizeof(*rsp_upiu));
6764
6765 break;
6766 default:
6767 err = -EINVAL;
6768
6769 break;
6770 }
6771
6772 return err;
6773 }
6774
6775 /**
6776 * ufshcd_eh_device_reset_handler - device reset handler registered to
6777 * scsi layer.
6778 * @cmd: SCSI command pointer
6779 *
6780 * Returns SUCCESS/FAILED
6781 */
ufshcd_eh_device_reset_handler(struct scsi_cmnd * cmd)6782 static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
6783 {
6784 struct Scsi_Host *host;
6785 struct ufs_hba *hba;
6786 u32 pos;
6787 int err;
6788 u8 resp = 0xF, lun;
6789
6790 host = cmd->device->host;
6791 hba = shost_priv(host);
6792
6793 lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
6794 err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp);
6795 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6796 if (!err)
6797 err = resp;
6798 goto out;
6799 }
6800
6801 /* clear the commands that were pending for corresponding LUN */
6802 for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) {
6803 if (hba->lrb[pos].lun == lun) {
6804 err = ufshcd_clear_cmd(hba, pos);
6805 if (err)
6806 break;
6807 __ufshcd_transfer_req_compl(hba, 1U << pos);
6808 }
6809 }
6810
6811 out:
6812 hba->req_abort_count = 0;
6813 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err);
6814 if (!err) {
6815 err = SUCCESS;
6816 } else {
6817 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
6818 err = FAILED;
6819 }
6820 return err;
6821 }
6822
ufshcd_set_req_abort_skip(struct ufs_hba * hba,unsigned long bitmap)6823 static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
6824 {
6825 struct ufshcd_lrb *lrbp;
6826 int tag;
6827
6828 for_each_set_bit(tag, &bitmap, hba->nutrs) {
6829 lrbp = &hba->lrb[tag];
6830 lrbp->req_abort_skip = true;
6831 }
6832 }
6833
6834 /**
6835 * ufshcd_try_to_abort_task - abort a specific task
6836 * @hba: Pointer to adapter instance
6837 * @tag: Task tag/index to be aborted
6838 *
6839 * Abort the pending command in device by sending UFS_ABORT_TASK task management
6840 * command, and in host controller by clearing the door-bell register. There can
6841 * be race between controller sending the command to the device while abort is
6842 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
6843 * really issued and then try to abort it.
6844 *
6845 * Returns zero on success, non-zero on failure
6846 */
ufshcd_try_to_abort_task(struct ufs_hba * hba,int tag)6847 static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
6848 {
6849 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
6850 int err = 0;
6851 int poll_cnt;
6852 u8 resp = 0xF;
6853 u32 reg;
6854
6855 for (poll_cnt = 100; poll_cnt; poll_cnt--) {
6856 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6857 UFS_QUERY_TASK, &resp);
6858 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
6859 /* cmd pending in the device */
6860 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
6861 __func__, tag);
6862 break;
6863 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6864 /*
6865 * cmd not pending in the device, check if it is
6866 * in transition.
6867 */
6868 dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
6869 __func__, tag);
6870 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
6871 if (reg & (1 << tag)) {
6872 /* sleep for max. 200us to stabilize */
6873 usleep_range(100, 200);
6874 continue;
6875 }
6876 /* command completed already */
6877 dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
6878 __func__, tag);
6879 goto out;
6880 } else {
6881 dev_err(hba->dev,
6882 "%s: no response from device. tag = %d, err %d\n",
6883 __func__, tag, err);
6884 if (!err)
6885 err = resp; /* service response error */
6886 goto out;
6887 }
6888 }
6889
6890 if (!poll_cnt) {
6891 err = -EBUSY;
6892 goto out;
6893 }
6894
6895 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6896 UFS_ABORT_TASK, &resp);
6897 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6898 if (!err) {
6899 err = resp; /* service response error */
6900 dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
6901 __func__, tag, err);
6902 }
6903 goto out;
6904 }
6905
6906 err = ufshcd_clear_cmd(hba, tag);
6907 if (err)
6908 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
6909 __func__, tag, err);
6910
6911 out:
6912 return err;
6913 }
6914
6915 /**
6916 * ufshcd_abort - scsi host template eh_abort_handler callback
6917 * @cmd: SCSI command pointer
6918 *
6919 * Returns SUCCESS/FAILED
6920 */
ufshcd_abort(struct scsi_cmnd * cmd)6921 static int ufshcd_abort(struct scsi_cmnd *cmd)
6922 {
6923 struct Scsi_Host *host;
6924 struct ufs_hba *hba;
6925 unsigned long flags;
6926 unsigned int tag;
6927 int err = FAILED, res;
6928 bool outstanding;
6929 struct ufshcd_lrb *lrbp;
6930 u32 reg;
6931
6932 host = cmd->device->host;
6933 hba = shost_priv(host);
6934 tag = cmd->request->tag;
6935 lrbp = &hba->lrb[tag];
6936 if (!ufshcd_valid_tag(hba, tag)) {
6937 dev_err(hba->dev,
6938 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
6939 __func__, tag, cmd, cmd->request);
6940 BUG();
6941 }
6942
6943 ufshcd_hold(hba, false);
6944 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
6945 /* If command is already aborted/completed, return FAILED. */
6946 if (!(test_bit(tag, &hba->outstanding_reqs))) {
6947 dev_err(hba->dev,
6948 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
6949 __func__, tag, hba->outstanding_reqs, reg);
6950 goto release;
6951 }
6952
6953 /* Print Transfer Request of aborted task */
6954 dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
6955
6956 /*
6957 * Print detailed info about aborted request.
6958 * As more than one request might get aborted at the same time,
6959 * print full information only for the first aborted request in order
6960 * to reduce repeated printouts. For other aborted requests only print
6961 * basic details.
6962 */
6963 scsi_print_command(cmd);
6964 if (!hba->req_abort_count) {
6965 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag);
6966 ufshcd_print_evt_hist(hba);
6967 ufshcd_print_host_state(hba);
6968 ufshcd_print_pwr_info(hba);
6969 ufshcd_print_trs(hba, 1 << tag, true);
6970 } else {
6971 ufshcd_print_trs(hba, 1 << tag, false);
6972 }
6973 hba->req_abort_count++;
6974
6975 if (!(reg & (1 << tag))) {
6976 dev_err(hba->dev,
6977 "%s: cmd was completed, but without a notifying intr, tag = %d",
6978 __func__, tag);
6979 __ufshcd_transfer_req_compl(hba, 1UL << tag);
6980 goto release;
6981 }
6982
6983 /*
6984 * Task abort to the device W-LUN is illegal. When this command
6985 * will fail, due to spec violation, scsi err handling next step
6986 * will be to send LU reset which, again, is a spec violation.
6987 * To avoid these unnecessary/illegal steps, first we clean up
6988 * the lrb taken by this cmd and re-set it in outstanding_reqs,
6989 * then queue the eh_work and bail.
6990 */
6991 if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) {
6992 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun);
6993
6994 spin_lock_irqsave(host->host_lock, flags);
6995 hba->force_reset = true;
6996 ufshcd_schedule_eh_work(hba);
6997 spin_unlock_irqrestore(host->host_lock, flags);
6998 goto release;
6999 }
7000
7001 /* Skip task abort in case previous aborts failed and report failure */
7002 if (lrbp->req_abort_skip) {
7003 dev_err(hba->dev, "%s: skipping abort\n", __func__);
7004 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7005 goto release;
7006 }
7007
7008 res = ufshcd_try_to_abort_task(hba, tag);
7009 if (res) {
7010 dev_err(hba->dev, "%s: failed with err %d\n", __func__, res);
7011 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7012 goto release;
7013 }
7014
7015 /*
7016 * Clear the corresponding bit from outstanding_reqs since the command
7017 * has been aborted successfully.
7018 */
7019 spin_lock_irqsave(host->host_lock, flags);
7020 outstanding = __test_and_clear_bit(tag, &hba->outstanding_reqs);
7021 spin_unlock_irqrestore(host->host_lock, flags);
7022
7023 if (outstanding)
7024 ufshcd_release_scsi_cmd(hba, lrbp);
7025
7026 err = SUCCESS;
7027
7028 release:
7029 /* Matches the ufshcd_hold() call at the start of this function. */
7030 ufshcd_release(hba);
7031 return err;
7032 }
7033
7034 /**
7035 * ufshcd_host_reset_and_restore - reset and restore host controller
7036 * @hba: per-adapter instance
7037 *
7038 * Note that host controller reset may issue DME_RESET to
7039 * local and remote (device) Uni-Pro stack and the attributes
7040 * are reset to default state.
7041 *
7042 * Returns zero on success, non-zero on failure
7043 */
ufshcd_host_reset_and_restore(struct ufs_hba * hba)7044 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
7045 {
7046 int err;
7047
7048 ufshpb_reset_host(hba);
7049 /*
7050 * Stop the host controller and complete the requests
7051 * cleared by h/w
7052 */
7053 ufshcd_hba_stop(hba);
7054 hba->silence_err_logs = true;
7055 ufshcd_complete_requests(hba);
7056 hba->silence_err_logs = false;
7057
7058 /* scale up clocks to max frequency before full reinitialization */
7059 ufshcd_set_clk_freq(hba, true);
7060
7061 err = ufshcd_hba_enable(hba);
7062
7063 /* Establish the link again and restore the device */
7064 if (!err)
7065 err = ufshcd_probe_hba(hba, false);
7066
7067 if (err)
7068 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
7069 ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err);
7070 return err;
7071 }
7072
7073 /**
7074 * ufshcd_reset_and_restore - reset and re-initialize host/device
7075 * @hba: per-adapter instance
7076 *
7077 * Reset and recover device, host and re-establish link. This
7078 * is helpful to recover the communication in fatal error conditions.
7079 *
7080 * Returns zero on success, non-zero on failure
7081 */
ufshcd_reset_and_restore(struct ufs_hba * hba)7082 static int ufshcd_reset_and_restore(struct ufs_hba *hba)
7083 {
7084 u32 saved_err;
7085 u32 saved_uic_err;
7086 int err = 0;
7087 unsigned long flags;
7088 int retries = MAX_HOST_RESET_RETRIES;
7089
7090 /*
7091 * This is a fresh start, cache and clear saved error first,
7092 * in case new error generated during reset and restore.
7093 */
7094 spin_lock_irqsave(hba->host->host_lock, flags);
7095 saved_err = hba->saved_err;
7096 saved_uic_err = hba->saved_uic_err;
7097 hba->saved_err = 0;
7098 hba->saved_uic_err = 0;
7099 spin_unlock_irqrestore(hba->host->host_lock, flags);
7100
7101 do {
7102 /* Reset the attached device */
7103 ufshcd_vops_device_reset(hba);
7104
7105 err = ufshcd_host_reset_and_restore(hba);
7106 } while (err && --retries);
7107
7108 spin_lock_irqsave(hba->host->host_lock, flags);
7109 /*
7110 * Inform scsi mid-layer that we did reset and allow to handle
7111 * Unit Attention properly.
7112 */
7113 scsi_report_bus_reset(hba->host, 0);
7114 if (err) {
7115 hba->ufshcd_state = UFSHCD_STATE_ERROR;
7116 hba->saved_err |= saved_err;
7117 hba->saved_uic_err |= saved_uic_err;
7118 }
7119 spin_unlock_irqrestore(hba->host->host_lock, flags);
7120
7121 return err;
7122 }
7123
7124 /**
7125 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
7126 * @cmd: SCSI command pointer
7127 *
7128 * Returns SUCCESS/FAILED
7129 */
ufshcd_eh_host_reset_handler(struct scsi_cmnd * cmd)7130 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
7131 {
7132 int err = SUCCESS;
7133 unsigned long flags;
7134 struct ufs_hba *hba;
7135
7136 hba = shost_priv(cmd->device->host);
7137
7138 spin_lock_irqsave(hba->host->host_lock, flags);
7139 hba->force_reset = true;
7140 ufshcd_schedule_eh_work(hba);
7141 dev_err(hba->dev, "%s: reset in progress - 1\n", __func__);
7142 spin_unlock_irqrestore(hba->host->host_lock, flags);
7143
7144 flush_work(&hba->eh_work);
7145
7146 spin_lock_irqsave(hba->host->host_lock, flags);
7147 if (hba->ufshcd_state == UFSHCD_STATE_ERROR)
7148 err = FAILED;
7149 spin_unlock_irqrestore(hba->host->host_lock, flags);
7150
7151 return err;
7152 }
7153
7154 /**
7155 * ufshcd_get_max_icc_level - calculate the ICC level
7156 * @sup_curr_uA: max. current supported by the regulator
7157 * @start_scan: row at the desc table to start scan from
7158 * @buff: power descriptor buffer
7159 *
7160 * Returns calculated max ICC level for specific regulator
7161 */
ufshcd_get_max_icc_level(int sup_curr_uA,u32 start_scan,char * buff)7162 static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff)
7163 {
7164 int i;
7165 int curr_uA;
7166 u16 data;
7167 u16 unit;
7168
7169 for (i = start_scan; i >= 0; i--) {
7170 data = be16_to_cpup((__be16 *)&buff[2 * i]);
7171 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
7172 ATTR_ICC_LVL_UNIT_OFFSET;
7173 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
7174 switch (unit) {
7175 case UFSHCD_NANO_AMP:
7176 curr_uA = curr_uA / 1000;
7177 break;
7178 case UFSHCD_MILI_AMP:
7179 curr_uA = curr_uA * 1000;
7180 break;
7181 case UFSHCD_AMP:
7182 curr_uA = curr_uA * 1000 * 1000;
7183 break;
7184 case UFSHCD_MICRO_AMP:
7185 default:
7186 break;
7187 }
7188 if (sup_curr_uA >= curr_uA)
7189 break;
7190 }
7191 if (i < 0) {
7192 i = 0;
7193 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
7194 }
7195
7196 return (u32)i;
7197 }
7198
7199 /**
7200 * ufshcd_calc_icc_level - calculate the max ICC level
7201 * In case regulators are not initialized we'll return 0
7202 * @hba: per-adapter instance
7203 * @desc_buf: power descriptor buffer to extract ICC levels from.
7204 * @len: length of desc_buff
7205 *
7206 * Returns calculated ICC level
7207 */
ufshcd_find_max_sup_active_icc_level(struct ufs_hba * hba,u8 * desc_buf,int len)7208 static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
7209 u8 *desc_buf, int len)
7210 {
7211 u32 icc_level = 0;
7212
7213 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
7214 !hba->vreg_info.vccq2) {
7215 dev_err(hba->dev,
7216 "%s: Regulator capability was not set, actvIccLevel=%d",
7217 __func__, icc_level);
7218 goto out;
7219 }
7220
7221 if (hba->vreg_info.vcc && hba->vreg_info.vcc->max_uA)
7222 icc_level = ufshcd_get_max_icc_level(
7223 hba->vreg_info.vcc->max_uA,
7224 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
7225 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
7226
7227 if (hba->vreg_info.vccq && hba->vreg_info.vccq->max_uA)
7228 icc_level = ufshcd_get_max_icc_level(
7229 hba->vreg_info.vccq->max_uA,
7230 icc_level,
7231 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
7232
7233 if (hba->vreg_info.vccq2 && hba->vreg_info.vccq2->max_uA)
7234 icc_level = ufshcd_get_max_icc_level(
7235 hba->vreg_info.vccq2->max_uA,
7236 icc_level,
7237 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
7238 out:
7239 return icc_level;
7240 }
7241
ufshcd_set_active_icc_lvl(struct ufs_hba * hba)7242 static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba)
7243 {
7244 int ret;
7245 int buff_len = hba->desc_size[QUERY_DESC_IDN_POWER];
7246 u8 *desc_buf;
7247 u32 icc_level;
7248
7249 desc_buf = kmalloc(buff_len, GFP_KERNEL);
7250 if (!desc_buf)
7251 return;
7252
7253 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0,
7254 desc_buf, buff_len);
7255 if (ret) {
7256 dev_err(hba->dev,
7257 "%s: Failed reading power descriptor.len = %d ret = %d",
7258 __func__, buff_len, ret);
7259 goto out;
7260 }
7261
7262 icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf,
7263 buff_len);
7264 dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level);
7265
7266 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
7267 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level);
7268
7269 if (ret)
7270 dev_err(hba->dev,
7271 "%s: Failed configuring bActiveICCLevel = %d ret = %d",
7272 __func__, icc_level, ret);
7273
7274 out:
7275 kfree(desc_buf);
7276 }
7277
7278 /**
7279 * ufshcd_scsi_add_wlus - Adds required W-LUs
7280 * @hba: per-adapter instance
7281 *
7282 * UFS device specification requires the UFS devices to support 4 well known
7283 * logical units:
7284 * "REPORT_LUNS" (address: 01h)
7285 * "UFS Device" (address: 50h)
7286 * "RPMB" (address: 44h)
7287 * "BOOT" (address: 30h)
7288 * UFS device's power management needs to be controlled by "POWER CONDITION"
7289 * field of SSU (START STOP UNIT) command. But this "power condition" field
7290 * will take effect only when its sent to "UFS device" well known logical unit
7291 * hence we require the scsi_device instance to represent this logical unit in
7292 * order for the UFS host driver to send the SSU command for power management.
7293 *
7294 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
7295 * Block) LU so user space process can control this LU. User space may also
7296 * want to have access to BOOT LU.
7297 *
7298 * This function adds scsi device instances for each of all well known LUs
7299 * (except "REPORT LUNS" LU).
7300 *
7301 * Returns zero on success (all required W-LUs are added successfully),
7302 * non-zero error value on failure (if failed to add any of the required W-LU).
7303 */
ufshcd_scsi_add_wlus(struct ufs_hba * hba)7304 static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
7305 {
7306 int ret = 0;
7307 struct scsi_device *sdev_boot;
7308
7309 hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0,
7310 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
7311 if (IS_ERR(hba->sdev_ufs_device)) {
7312 ret = PTR_ERR(hba->sdev_ufs_device);
7313 hba->sdev_ufs_device = NULL;
7314 goto out;
7315 }
7316 scsi_device_put(hba->sdev_ufs_device);
7317
7318 hba->sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
7319 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
7320 if (IS_ERR(hba->sdev_rpmb)) {
7321 ret = PTR_ERR(hba->sdev_rpmb);
7322 goto remove_sdev_ufs_device;
7323 }
7324 scsi_device_put(hba->sdev_rpmb);
7325
7326 sdev_boot = __scsi_add_device(hba->host, 0, 0,
7327 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
7328 if (IS_ERR(sdev_boot))
7329 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
7330 else
7331 scsi_device_put(sdev_boot);
7332 goto out;
7333
7334 remove_sdev_ufs_device:
7335 scsi_remove_device(hba->sdev_ufs_device);
7336 out:
7337 return ret;
7338 }
7339
ufshcd_wb_probe(struct ufs_hba * hba,u8 * desc_buf)7340 static void ufshcd_wb_probe(struct ufs_hba *hba, u8 *desc_buf)
7341 {
7342 struct ufs_dev_info *dev_info = &hba->dev_info;
7343 u8 lun;
7344 u32 d_lu_wb_buf_alloc;
7345
7346 if (!ufshcd_is_wb_allowed(hba))
7347 return;
7348 /*
7349 * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or
7350 * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES
7351 * enabled
7352 */
7353 if (!(dev_info->wspecversion >= 0x310 ||
7354 dev_info->wspecversion == 0x220 ||
7355 (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES)))
7356 goto wb_disabled;
7357
7358 if (hba->desc_size[QUERY_DESC_IDN_DEVICE] <
7359 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP + 4)
7360 goto wb_disabled;
7361
7362 dev_info->d_ext_ufs_feature_sup =
7363 get_unaligned_be32(desc_buf +
7364 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
7365
7366 if (!(dev_info->d_ext_ufs_feature_sup & UFS_DEV_WRITE_BOOSTER_SUP))
7367 goto wb_disabled;
7368
7369 /*
7370 * WB may be supported but not configured while provisioning.
7371 * The spec says, in dedicated wb buffer mode,
7372 * a max of 1 lun would have wb buffer configured.
7373 * Now only shared buffer mode is supported.
7374 */
7375 dev_info->b_wb_buffer_type =
7376 desc_buf[DEVICE_DESC_PARAM_WB_TYPE];
7377
7378 dev_info->b_presrv_uspc_en =
7379 desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN];
7380
7381 if (dev_info->b_wb_buffer_type == WB_BUF_MODE_SHARED) {
7382 dev_info->d_wb_alloc_units =
7383 get_unaligned_be32(desc_buf +
7384 DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS);
7385 if (!dev_info->d_wb_alloc_units)
7386 goto wb_disabled;
7387 } else {
7388 for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) {
7389 d_lu_wb_buf_alloc = 0;
7390 ufshcd_read_unit_desc_param(hba,
7391 lun,
7392 UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS,
7393 (u8 *)&d_lu_wb_buf_alloc,
7394 sizeof(d_lu_wb_buf_alloc));
7395 if (d_lu_wb_buf_alloc) {
7396 dev_info->wb_dedicated_lu = lun;
7397 break;
7398 }
7399 }
7400
7401 if (!d_lu_wb_buf_alloc)
7402 goto wb_disabled;
7403 }
7404 return;
7405
7406 wb_disabled:
7407 hba->caps &= ~UFSHCD_CAP_WB_EN;
7408 }
7409
ufshcd_fixup_dev_quirks(struct ufs_hba * hba,struct ufs_dev_fix * fixups)7410 void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, struct ufs_dev_fix *fixups)
7411 {
7412 struct ufs_dev_fix *f;
7413 struct ufs_dev_info *dev_info = &hba->dev_info;
7414
7415 if (!fixups)
7416 return;
7417
7418 for (f = fixups; f->quirk; f++) {
7419 if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
7420 f->wmanufacturerid == UFS_ANY_VENDOR) &&
7421 ((dev_info->model &&
7422 STR_PRFX_EQUAL(f->model, dev_info->model)) ||
7423 !strcmp(f->model, UFS_ANY_MODEL)))
7424 hba->dev_quirks |= f->quirk;
7425 }
7426 }
7427 EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks);
7428
ufs_fixup_device_setup(struct ufs_hba * hba)7429 static void ufs_fixup_device_setup(struct ufs_hba *hba)
7430 {
7431 /* fix by general quirk table */
7432 ufshcd_fixup_dev_quirks(hba, ufs_fixups);
7433
7434 /* allow vendors to fix quirks */
7435 ufshcd_vops_fixup_dev_quirks(hba);
7436 }
7437
ufs_get_device_desc(struct ufs_hba * hba)7438 static int ufs_get_device_desc(struct ufs_hba *hba)
7439 {
7440 int err;
7441 u8 model_index;
7442 u8 b_ufs_feature_sup;
7443 u8 *desc_buf;
7444 struct ufs_dev_info *dev_info = &hba->dev_info;
7445
7446 desc_buf = kmalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
7447 if (!desc_buf) {
7448 err = -ENOMEM;
7449 goto out;
7450 }
7451
7452 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf,
7453 hba->desc_size[QUERY_DESC_IDN_DEVICE]);
7454 if (err) {
7455 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
7456 __func__, err);
7457 goto out;
7458 }
7459
7460 /*
7461 * getting vendor (manufacturerID) and Bank Index in big endian
7462 * format
7463 */
7464 dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
7465 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
7466
7467 /* getting Specification Version in big endian format */
7468 dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 |
7469 desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1];
7470 b_ufs_feature_sup = desc_buf[DEVICE_DESC_PARAM_UFS_FEAT];
7471
7472 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
7473
7474 if (dev_info->wspecversion >= UFS_DEV_HPB_SUPPORT_VERSION &&
7475 (b_ufs_feature_sup & UFS_DEV_HPB_SUPPORT)) {
7476 bool hpb_en = false;
7477
7478 ufshpb_get_dev_info(hba, desc_buf);
7479
7480 if (!ufshpb_is_legacy(hba))
7481 err = ufshcd_query_flag_retry(hba,
7482 UPIU_QUERY_OPCODE_READ_FLAG,
7483 QUERY_FLAG_IDN_HPB_EN, 0,
7484 &hpb_en);
7485
7486 if (ufshpb_is_legacy(hba) || (!err && hpb_en))
7487 dev_info->hpb_enabled = true;
7488 }
7489
7490 err = ufshcd_read_string_desc(hba, model_index,
7491 &dev_info->model, SD_ASCII_STD);
7492 if (err < 0) {
7493 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
7494 __func__, err);
7495 goto out;
7496 }
7497
7498 ufs_fixup_device_setup(hba);
7499
7500 ufshcd_wb_probe(hba, desc_buf);
7501
7502 /*
7503 * ufshcd_read_string_desc returns size of the string
7504 * reset the error value
7505 */
7506 err = 0;
7507
7508 out:
7509 kfree(desc_buf);
7510 return err;
7511 }
7512
ufs_put_device_desc(struct ufs_hba * hba)7513 static void ufs_put_device_desc(struct ufs_hba *hba)
7514 {
7515 struct ufs_dev_info *dev_info = &hba->dev_info;
7516
7517 kfree(dev_info->model);
7518 dev_info->model = NULL;
7519 }
7520
7521 /**
7522 * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
7523 * @hba: per-adapter instance
7524 *
7525 * PA_TActivate parameter can be tuned manually if UniPro version is less than
7526 * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
7527 * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
7528 * the hibern8 exit latency.
7529 *
7530 * Returns zero on success, non-zero error value on failure.
7531 */
ufshcd_tune_pa_tactivate(struct ufs_hba * hba)7532 static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
7533 {
7534 int ret = 0;
7535 u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
7536
7537 ret = ufshcd_dme_peer_get(hba,
7538 UIC_ARG_MIB_SEL(
7539 RX_MIN_ACTIVATETIME_CAPABILITY,
7540 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7541 &peer_rx_min_activatetime);
7542 if (ret)
7543 goto out;
7544
7545 /* make sure proper unit conversion is applied */
7546 tuned_pa_tactivate =
7547 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
7548 / PA_TACTIVATE_TIME_UNIT_US);
7549 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7550 tuned_pa_tactivate);
7551
7552 out:
7553 return ret;
7554 }
7555
7556 /**
7557 * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
7558 * @hba: per-adapter instance
7559 *
7560 * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
7561 * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
7562 * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
7563 * This optimal value can help reduce the hibern8 exit latency.
7564 *
7565 * Returns zero on success, non-zero error value on failure.
7566 */
ufshcd_tune_pa_hibern8time(struct ufs_hba * hba)7567 static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
7568 {
7569 int ret = 0;
7570 u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
7571 u32 max_hibern8_time, tuned_pa_hibern8time;
7572
7573 ret = ufshcd_dme_get(hba,
7574 UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
7575 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
7576 &local_tx_hibern8_time_cap);
7577 if (ret)
7578 goto out;
7579
7580 ret = ufshcd_dme_peer_get(hba,
7581 UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
7582 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7583 &peer_rx_hibern8_time_cap);
7584 if (ret)
7585 goto out;
7586
7587 max_hibern8_time = max(local_tx_hibern8_time_cap,
7588 peer_rx_hibern8_time_cap);
7589 /* make sure proper unit conversion is applied */
7590 tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
7591 / PA_HIBERN8_TIME_UNIT_US);
7592 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
7593 tuned_pa_hibern8time);
7594 out:
7595 return ret;
7596 }
7597
7598 /**
7599 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
7600 * less than device PA_TACTIVATE time.
7601 * @hba: per-adapter instance
7602 *
7603 * Some UFS devices require host PA_TACTIVATE to be lower than device
7604 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
7605 * for such devices.
7606 *
7607 * Returns zero on success, non-zero error value on failure.
7608 */
ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba * hba)7609 static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
7610 {
7611 int ret = 0;
7612 u32 granularity, peer_granularity;
7613 u32 pa_tactivate, peer_pa_tactivate;
7614 u32 pa_tactivate_us, peer_pa_tactivate_us;
7615 u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
7616
7617 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7618 &granularity);
7619 if (ret)
7620 goto out;
7621
7622 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7623 &peer_granularity);
7624 if (ret)
7625 goto out;
7626
7627 if ((granularity < PA_GRANULARITY_MIN_VAL) ||
7628 (granularity > PA_GRANULARITY_MAX_VAL)) {
7629 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
7630 __func__, granularity);
7631 return -EINVAL;
7632 }
7633
7634 if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
7635 (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
7636 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
7637 __func__, peer_granularity);
7638 return -EINVAL;
7639 }
7640
7641 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
7642 if (ret)
7643 goto out;
7644
7645 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
7646 &peer_pa_tactivate);
7647 if (ret)
7648 goto out;
7649
7650 pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
7651 peer_pa_tactivate_us = peer_pa_tactivate *
7652 gran_to_us_table[peer_granularity - 1];
7653
7654 if (pa_tactivate_us >= peer_pa_tactivate_us) {
7655 u32 new_peer_pa_tactivate;
7656
7657 new_peer_pa_tactivate = pa_tactivate_us /
7658 gran_to_us_table[peer_granularity - 1];
7659 new_peer_pa_tactivate++;
7660 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7661 new_peer_pa_tactivate);
7662 }
7663
7664 out:
7665 return ret;
7666 }
7667
ufshcd_tune_unipro_params(struct ufs_hba * hba)7668 static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
7669 {
7670 if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
7671 ufshcd_tune_pa_tactivate(hba);
7672 ufshcd_tune_pa_hibern8time(hba);
7673 }
7674
7675 ufshcd_vops_apply_dev_quirks(hba);
7676
7677 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
7678 /* set 1ms timeout for PA_TACTIVATE */
7679 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
7680
7681 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
7682 ufshcd_quirk_tune_host_pa_tactivate(hba);
7683 }
7684
ufshcd_clear_dbg_ufs_stats(struct ufs_hba * hba)7685 static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
7686 {
7687 hba->ufs_stats.hibern8_exit_cnt = 0;
7688 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
7689 hba->req_abort_count = 0;
7690 }
7691
ufshcd_device_geo_params_init(struct ufs_hba * hba)7692 static int ufshcd_device_geo_params_init(struct ufs_hba *hba)
7693 {
7694 int err;
7695 size_t buff_len;
7696 u8 *desc_buf;
7697
7698 buff_len = hba->desc_size[QUERY_DESC_IDN_GEOMETRY];
7699 desc_buf = kmalloc(buff_len, GFP_KERNEL);
7700 if (!desc_buf) {
7701 err = -ENOMEM;
7702 goto out;
7703 }
7704
7705 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0,
7706 desc_buf, buff_len);
7707 if (err) {
7708 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n",
7709 __func__, err);
7710 goto out;
7711 }
7712
7713 if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1)
7714 hba->dev_info.max_lu_supported = 32;
7715 else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0)
7716 hba->dev_info.max_lu_supported = 8;
7717
7718 if (hba->desc_size[QUERY_DESC_IDN_GEOMETRY] >=
7719 GEOMETRY_DESC_PARAM_HPB_MAX_ACTIVE_REGS)
7720 ufshpb_get_geo_info(hba, desc_buf);
7721
7722 out:
7723 kfree(desc_buf);
7724 return err;
7725 }
7726
7727 static struct ufs_ref_clk ufs_ref_clk_freqs[] = {
7728 {19200000, REF_CLK_FREQ_19_2_MHZ},
7729 {26000000, REF_CLK_FREQ_26_MHZ},
7730 {38400000, REF_CLK_FREQ_38_4_MHZ},
7731 {52000000, REF_CLK_FREQ_52_MHZ},
7732 {0, REF_CLK_FREQ_INVAL},
7733 };
7734
7735 static enum ufs_ref_clk_freq
ufs_get_bref_clk_from_hz(unsigned long freq)7736 ufs_get_bref_clk_from_hz(unsigned long freq)
7737 {
7738 int i;
7739
7740 for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
7741 if (ufs_ref_clk_freqs[i].freq_hz == freq)
7742 return ufs_ref_clk_freqs[i].val;
7743
7744 return REF_CLK_FREQ_INVAL;
7745 }
7746
ufshcd_parse_dev_ref_clk_freq(struct ufs_hba * hba,struct clk * refclk)7747 void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
7748 {
7749 unsigned long freq;
7750
7751 freq = clk_get_rate(refclk);
7752
7753 hba->dev_ref_clk_freq =
7754 ufs_get_bref_clk_from_hz(freq);
7755
7756 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
7757 dev_err(hba->dev,
7758 "invalid ref_clk setting = %ld\n", freq);
7759 }
7760
ufshcd_set_dev_ref_clk(struct ufs_hba * hba)7761 static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
7762 {
7763 int err;
7764 u32 ref_clk;
7765 u32 freq = hba->dev_ref_clk_freq;
7766
7767 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
7768 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
7769
7770 if (err) {
7771 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
7772 err);
7773 goto out;
7774 }
7775
7776 if (ref_clk == freq)
7777 goto out; /* nothing to update */
7778
7779 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
7780 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
7781
7782 if (err) {
7783 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
7784 ufs_ref_clk_freqs[freq].freq_hz);
7785 goto out;
7786 }
7787
7788 dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
7789 ufs_ref_clk_freqs[freq].freq_hz);
7790
7791 out:
7792 return err;
7793 }
7794
ufshcd_device_params_init(struct ufs_hba * hba)7795 static int ufshcd_device_params_init(struct ufs_hba *hba)
7796 {
7797 bool flag;
7798 int ret, i;
7799
7800 /* Init device descriptor sizes */
7801 for (i = 0; i < QUERY_DESC_IDN_MAX; i++)
7802 hba->desc_size[i] = QUERY_DESC_MAX_SIZE;
7803
7804 /* Init UFS geometry descriptor related parameters */
7805 ret = ufshcd_device_geo_params_init(hba);
7806 if (ret)
7807 goto out;
7808
7809 /* Check and apply UFS device quirks */
7810 ret = ufs_get_device_desc(hba);
7811 if (ret) {
7812 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
7813 __func__, ret);
7814 goto out;
7815 }
7816
7817 ufshcd_get_ref_clk_gating_wait(hba);
7818
7819 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
7820 QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag))
7821 hba->dev_info.f_power_on_wp_en = flag;
7822
7823 /* Probe maximum power mode co-supported by both UFS host and device */
7824 if (ufshcd_get_max_pwr_mode(hba))
7825 dev_err(hba->dev,
7826 "%s: Failed getting max supported power mode\n",
7827 __func__);
7828 out:
7829 return ret;
7830 }
7831
7832 /**
7833 * ufshcd_add_lus - probe and add UFS logical units
7834 * @hba: per-adapter instance
7835 */
ufshcd_add_lus(struct ufs_hba * hba)7836 static int ufshcd_add_lus(struct ufs_hba *hba)
7837 {
7838 int ret;
7839
7840 /* Add required well known logical units to scsi mid layer */
7841 ret = ufshcd_scsi_add_wlus(hba);
7842 if (ret)
7843 goto out;
7844
7845 /* Initialize devfreq after UFS device is detected */
7846 if (ufshcd_is_clkscaling_supported(hba)) {
7847 memcpy(&hba->clk_scaling.saved_pwr_info.info,
7848 &hba->pwr_info,
7849 sizeof(struct ufs_pa_layer_attr));
7850 hba->clk_scaling.saved_pwr_info.is_valid = true;
7851 hba->clk_scaling.is_allowed = true;
7852
7853 ret = ufshcd_devfreq_init(hba);
7854 if (ret)
7855 goto out;
7856
7857 hba->clk_scaling.is_enabled = true;
7858 ufshcd_init_clk_scaling_sysfs(hba);
7859 }
7860
7861 ufs_bsg_probe(hba);
7862 ufshpb_init(hba);
7863 scsi_scan_host(hba->host);
7864 pm_runtime_put_sync(hba->dev);
7865
7866 out:
7867 return ret;
7868 }
7869
7870 /**
7871 * ufshcd_probe_hba - probe hba to detect device and initialize
7872 * @hba: per-adapter instance
7873 * @async: asynchronous execution or not
7874 *
7875 * Execute link-startup and verify device initialization
7876 */
ufshcd_probe_hba(struct ufs_hba * hba,bool async)7877 static int ufshcd_probe_hba(struct ufs_hba *hba, bool async)
7878 {
7879 int ret;
7880 unsigned long flags;
7881 ktime_t start = ktime_get();
7882
7883 hba->ufshcd_state = UFSHCD_STATE_RESET;
7884
7885 ret = ufshcd_link_startup(hba);
7886 if (ret)
7887 goto out;
7888
7889 if (hba->quirks & UFSHCD_QUIRK_SKIP_INTERFACE_CONFIGURATION)
7890 goto out;
7891
7892 /* Debug counters initialization */
7893 ufshcd_clear_dbg_ufs_stats(hba);
7894
7895 /* UniPro link is active now */
7896 ufshcd_set_link_active(hba);
7897
7898 /* Verify device initialization by sending NOP OUT UPIU */
7899 ret = ufshcd_verify_dev_init(hba);
7900 if (ret)
7901 goto out;
7902
7903 /* Initiate UFS initialization, and waiting until completion */
7904 ret = ufshcd_complete_dev_init(hba);
7905 if (ret)
7906 goto out;
7907
7908 /*
7909 * Initialize UFS device parameters used by driver, these
7910 * parameters are associated with UFS descriptors.
7911 */
7912 if (async) {
7913 ret = ufshcd_device_params_init(hba);
7914 if (ret)
7915 goto out;
7916 }
7917
7918 ufshcd_tune_unipro_params(hba);
7919
7920 /* UFS device is also active now */
7921 ufshcd_set_ufs_dev_active(hba);
7922 ufshcd_force_reset_auto_bkops(hba);
7923
7924 /* Gear up to HS gear if supported */
7925 if (hba->max_pwr_info.is_valid) {
7926 /*
7927 * Set the right value to bRefClkFreq before attempting to
7928 * switch to HS gears.
7929 */
7930 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
7931 ufshcd_set_dev_ref_clk(hba);
7932 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
7933 if (ret) {
7934 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
7935 __func__, ret);
7936 goto out;
7937 }
7938 ufshcd_print_pwr_info(hba);
7939 }
7940
7941 /*
7942 * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec)
7943 * and for removable UFS card as well, hence always set the parameter.
7944 * Note: Error handler may issue the device reset hence resetting
7945 * bActiveICCLevel as well so it is always safe to set this here.
7946 */
7947 ufshcd_set_active_icc_lvl(hba);
7948
7949 ufshcd_wb_config(hba);
7950 /* Enable Auto-Hibernate if configured */
7951 ufshcd_auto_hibern8_enable(hba);
7952
7953 ufshpb_reset(hba);
7954
7955 trace_android_rvh_ufs_complete_init(hba);
7956 out:
7957 spin_lock_irqsave(hba->host->host_lock, flags);
7958 if (ret)
7959 hba->ufshcd_state = UFSHCD_STATE_ERROR;
7960 else if (hba->ufshcd_state == UFSHCD_STATE_RESET)
7961 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
7962 spin_unlock_irqrestore(hba->host->host_lock, flags);
7963
7964 trace_ufshcd_init(dev_name(hba->dev), ret,
7965 ktime_to_us(ktime_sub(ktime_get(), start)),
7966 hba->curr_dev_pwr_mode, hba->uic_link_state);
7967 return ret;
7968 }
7969
7970 /**
7971 * ufshcd_async_scan - asynchronous execution for probing hba
7972 * @data: data pointer to pass to this function
7973 * @cookie: cookie data
7974 */
ufshcd_async_scan(void * data,async_cookie_t cookie)7975 static void ufshcd_async_scan(void *data, async_cookie_t cookie)
7976 {
7977 struct ufs_hba *hba = (struct ufs_hba *)data;
7978 int ret;
7979
7980 down(&hba->host_sem);
7981 /* Initialize hba, detect and initialize UFS device */
7982 ret = ufshcd_probe_hba(hba, true);
7983 up(&hba->host_sem);
7984 if (ret)
7985 goto out;
7986
7987 /* Probe and add UFS logical units */
7988 ret = ufshcd_add_lus(hba);
7989 out:
7990 /*
7991 * If we failed to initialize the device or the device is not
7992 * present, turn off the power/clocks etc.
7993 */
7994 if (ret) {
7995 pm_runtime_put_sync(hba->dev);
7996 ufshcd_hba_exit(hba);
7997 }
7998 }
7999
8000 static const struct attribute_group *ufshcd_driver_groups[] = {
8001 &ufs_sysfs_unit_descriptor_group,
8002 &ufs_sysfs_lun_attributes_group,
8003 #ifdef CONFIG_SCSI_UFS_HPB
8004 &ufs_sysfs_hpb_stat_group,
8005 &ufs_sysfs_hpb_param_group,
8006 #endif
8007 NULL,
8008 };
8009
8010 static struct ufs_hba_variant_params ufs_hba_vps = {
8011 .hba_enable_delay_us = 1000,
8012 .wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(40),
8013 .devfreq_profile.polling_ms = 100,
8014 .devfreq_profile.target = ufshcd_devfreq_target,
8015 .devfreq_profile.get_dev_status = ufshcd_devfreq_get_dev_status,
8016 .ondemand_data.upthreshold = 70,
8017 .ondemand_data.downdifferential = 5,
8018 };
8019
8020 static struct scsi_host_template ufshcd_driver_template = {
8021 .module = THIS_MODULE,
8022 .name = UFSHCD,
8023 .proc_name = UFSHCD,
8024 .queuecommand = ufshcd_queuecommand,
8025 .slave_alloc = ufshcd_slave_alloc,
8026 .slave_configure = ufshcd_slave_configure,
8027 .slave_destroy = ufshcd_slave_destroy,
8028 .change_queue_depth = ufshcd_change_queue_depth,
8029 .eh_abort_handler = ufshcd_abort,
8030 .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
8031 .eh_host_reset_handler = ufshcd_eh_host_reset_handler,
8032 .this_id = -1,
8033 .sg_tablesize = SG_ALL,
8034 .cmd_per_lun = UFSHCD_CMD_PER_LUN,
8035 .can_queue = UFSHCD_CAN_QUEUE,
8036 .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX,
8037 .max_host_blocked = 1,
8038 .track_queue_depth = 1,
8039 .sdev_groups = ufshcd_driver_groups,
8040 .dma_boundary = PAGE_SIZE - 1,
8041 .rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS,
8042 };
8043
ufshcd_config_vreg_load(struct device * dev,struct ufs_vreg * vreg,int ua)8044 static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
8045 int ua)
8046 {
8047 int ret;
8048
8049 if (!vreg)
8050 return 0;
8051
8052 /*
8053 * "set_load" operation shall be required on those regulators
8054 * which specifically configured current limitation. Otherwise
8055 * zero max_uA may cause unexpected behavior when regulator is
8056 * enabled or set as high power mode.
8057 */
8058 if (!vreg->max_uA)
8059 return 0;
8060
8061 ret = regulator_set_load(vreg->reg, ua);
8062 if (ret < 0) {
8063 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
8064 __func__, vreg->name, ua, ret);
8065 }
8066
8067 return ret;
8068 }
8069
ufshcd_config_vreg_lpm(struct ufs_hba * hba,struct ufs_vreg * vreg)8070 static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
8071 struct ufs_vreg *vreg)
8072 {
8073 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
8074 }
8075
ufshcd_config_vreg_hpm(struct ufs_hba * hba,struct ufs_vreg * vreg)8076 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
8077 struct ufs_vreg *vreg)
8078 {
8079 if (!vreg)
8080 return 0;
8081
8082 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
8083 }
8084
ufshcd_config_vreg(struct device * dev,struct ufs_vreg * vreg,bool on)8085 static int ufshcd_config_vreg(struct device *dev,
8086 struct ufs_vreg *vreg, bool on)
8087 {
8088 int ret = 0;
8089 struct regulator *reg;
8090 const char *name;
8091 int min_uV, uA_load;
8092
8093 BUG_ON(!vreg);
8094
8095 reg = vreg->reg;
8096 name = vreg->name;
8097
8098 if (regulator_count_voltages(reg) > 0) {
8099 uA_load = on ? vreg->max_uA : 0;
8100 ret = ufshcd_config_vreg_load(dev, vreg, uA_load);
8101 if (ret)
8102 goto out;
8103
8104 if (vreg->min_uV && vreg->max_uV) {
8105 min_uV = on ? vreg->min_uV : 0;
8106 ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
8107 if (ret)
8108 dev_err(dev,
8109 "%s: %s set voltage failed, err=%d\n",
8110 __func__, name, ret);
8111 }
8112 }
8113 out:
8114 return ret;
8115 }
8116
ufshcd_enable_vreg(struct device * dev,struct ufs_vreg * vreg)8117 static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
8118 {
8119 int ret = 0;
8120
8121 if (!vreg || vreg->enabled)
8122 goto out;
8123
8124 ret = ufshcd_config_vreg(dev, vreg, true);
8125 if (!ret)
8126 ret = regulator_enable(vreg->reg);
8127
8128 if (!ret)
8129 vreg->enabled = true;
8130 else
8131 dev_err(dev, "%s: %s enable failed, err=%d\n",
8132 __func__, vreg->name, ret);
8133 out:
8134 return ret;
8135 }
8136
ufshcd_disable_vreg(struct device * dev,struct ufs_vreg * vreg)8137 static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
8138 {
8139 int ret = 0;
8140
8141 if (!vreg || !vreg->enabled || vreg->always_on)
8142 goto out;
8143
8144 ret = regulator_disable(vreg->reg);
8145
8146 if (!ret) {
8147 /* ignore errors on applying disable config */
8148 ufshcd_config_vreg(dev, vreg, false);
8149 vreg->enabled = false;
8150 } else {
8151 dev_err(dev, "%s: %s disable failed, err=%d\n",
8152 __func__, vreg->name, ret);
8153 }
8154 out:
8155 return ret;
8156 }
8157
ufshcd_setup_vreg(struct ufs_hba * hba,bool on)8158 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
8159 {
8160 int ret = 0;
8161 struct device *dev = hba->dev;
8162 struct ufs_vreg_info *info = &hba->vreg_info;
8163
8164 ret = ufshcd_toggle_vreg(dev, info->vcc, on);
8165 if (ret)
8166 goto out;
8167
8168 ret = ufshcd_toggle_vreg(dev, info->vccq, on);
8169 if (ret)
8170 goto out;
8171
8172 ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
8173
8174 out:
8175 if (ret) {
8176 ufshcd_toggle_vreg(dev, info->vccq2, false);
8177 ufshcd_toggle_vreg(dev, info->vccq, false);
8178 ufshcd_toggle_vreg(dev, info->vcc, false);
8179 }
8180 return ret;
8181 }
8182
ufshcd_setup_hba_vreg(struct ufs_hba * hba,bool on)8183 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
8184 {
8185 struct ufs_vreg_info *info = &hba->vreg_info;
8186
8187 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
8188 }
8189
ufshcd_get_vreg(struct device * dev,struct ufs_vreg * vreg)8190 static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
8191 {
8192 int ret = 0;
8193
8194 if (!vreg)
8195 goto out;
8196
8197 vreg->reg = devm_regulator_get(dev, vreg->name);
8198 if (IS_ERR(vreg->reg)) {
8199 ret = PTR_ERR(vreg->reg);
8200 dev_err(dev, "%s: %s get failed, err=%d\n",
8201 __func__, vreg->name, ret);
8202 }
8203 out:
8204 return ret;
8205 }
8206
ufshcd_init_vreg(struct ufs_hba * hba)8207 static int ufshcd_init_vreg(struct ufs_hba *hba)
8208 {
8209 int ret = 0;
8210 struct device *dev = hba->dev;
8211 struct ufs_vreg_info *info = &hba->vreg_info;
8212
8213 ret = ufshcd_get_vreg(dev, info->vcc);
8214 if (ret)
8215 goto out;
8216
8217 ret = ufshcd_get_vreg(dev, info->vccq);
8218 if (!ret)
8219 ret = ufshcd_get_vreg(dev, info->vccq2);
8220 out:
8221 return ret;
8222 }
8223
ufshcd_init_hba_vreg(struct ufs_hba * hba)8224 static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
8225 {
8226 struct ufs_vreg_info *info = &hba->vreg_info;
8227
8228 if (info)
8229 return ufshcd_get_vreg(hba->dev, info->vdd_hba);
8230
8231 return 0;
8232 }
8233
ufshcd_setup_clocks(struct ufs_hba * hba,bool on)8234 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
8235 {
8236 int ret = 0;
8237 struct ufs_clk_info *clki;
8238 struct list_head *head = &hba->clk_list_head;
8239 unsigned long flags;
8240 ktime_t start = ktime_get();
8241 bool clk_state_changed = false;
8242
8243 if (list_empty(head))
8244 goto out;
8245
8246 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
8247 if (ret)
8248 return ret;
8249
8250 list_for_each_entry(clki, head, list) {
8251 if (!IS_ERR_OR_NULL(clki->clk)) {
8252 /*
8253 * Don't disable clocks which are needed
8254 * to keep the link active.
8255 */
8256 if (ufshcd_is_link_active(hba) &&
8257 clki->keep_link_active)
8258 continue;
8259
8260 clk_state_changed = on ^ clki->enabled;
8261 if (on && !clki->enabled) {
8262 ret = clk_prepare_enable(clki->clk);
8263 if (ret) {
8264 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
8265 __func__, clki->name, ret);
8266 goto out;
8267 }
8268 } else if (!on && clki->enabled) {
8269 clk_disable_unprepare(clki->clk);
8270 }
8271 clki->enabled = on;
8272 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
8273 clki->name, on ? "en" : "dis");
8274 }
8275 }
8276
8277 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
8278 if (ret)
8279 return ret;
8280
8281 out:
8282 if (ret) {
8283 list_for_each_entry(clki, head, list) {
8284 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
8285 clk_disable_unprepare(clki->clk);
8286 }
8287 } else if (!ret && on) {
8288 spin_lock_irqsave(hba->host->host_lock, flags);
8289 hba->clk_gating.state = CLKS_ON;
8290 trace_ufshcd_clk_gating(dev_name(hba->dev),
8291 hba->clk_gating.state);
8292 spin_unlock_irqrestore(hba->host->host_lock, flags);
8293 }
8294
8295 if (clk_state_changed)
8296 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
8297 (on ? "on" : "off"),
8298 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
8299 return ret;
8300 }
8301
ufshcd_init_clocks(struct ufs_hba * hba)8302 static int ufshcd_init_clocks(struct ufs_hba *hba)
8303 {
8304 int ret = 0;
8305 struct ufs_clk_info *clki;
8306 struct device *dev = hba->dev;
8307 struct list_head *head = &hba->clk_list_head;
8308
8309 if (list_empty(head))
8310 goto out;
8311
8312 list_for_each_entry(clki, head, list) {
8313 if (!clki->name)
8314 continue;
8315
8316 clki->clk = devm_clk_get(dev, clki->name);
8317 if (IS_ERR(clki->clk)) {
8318 ret = PTR_ERR(clki->clk);
8319 dev_err(dev, "%s: %s clk get failed, %d\n",
8320 __func__, clki->name, ret);
8321 goto out;
8322 }
8323
8324 /*
8325 * Parse device ref clk freq as per device tree "ref_clk".
8326 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
8327 * in ufshcd_alloc_host().
8328 */
8329 if (!strcmp(clki->name, "ref_clk"))
8330 ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
8331
8332 if (clki->max_freq) {
8333 ret = clk_set_rate(clki->clk, clki->max_freq);
8334 if (ret) {
8335 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
8336 __func__, clki->name,
8337 clki->max_freq, ret);
8338 goto out;
8339 }
8340 clki->curr_freq = clki->max_freq;
8341 }
8342 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
8343 clki->name, clk_get_rate(clki->clk));
8344 }
8345 out:
8346 return ret;
8347 }
8348
ufshcd_variant_hba_init(struct ufs_hba * hba)8349 static int ufshcd_variant_hba_init(struct ufs_hba *hba)
8350 {
8351 int err = 0;
8352
8353 if (!hba->vops)
8354 goto out;
8355
8356 err = ufshcd_vops_init(hba);
8357 if (err)
8358 goto out;
8359
8360 err = ufshcd_vops_setup_regulators(hba, true);
8361 if (err)
8362 ufshcd_vops_exit(hba);
8363 out:
8364 if (err)
8365 dev_err(hba->dev, "%s: variant %s init failed err %d\n",
8366 __func__, ufshcd_get_var_name(hba), err);
8367 return err;
8368 }
8369
ufshcd_variant_hba_exit(struct ufs_hba * hba)8370 static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
8371 {
8372 if (!hba->vops)
8373 return;
8374
8375 ufshcd_vops_setup_regulators(hba, false);
8376
8377 ufshcd_vops_exit(hba);
8378 }
8379
ufshcd_hba_init(struct ufs_hba * hba)8380 static int ufshcd_hba_init(struct ufs_hba *hba)
8381 {
8382 int err;
8383
8384 /*
8385 * Handle host controller power separately from the UFS device power
8386 * rails as it will help controlling the UFS host controller power
8387 * collapse easily which is different than UFS device power collapse.
8388 * Also, enable the host controller power before we go ahead with rest
8389 * of the initialization here.
8390 */
8391 err = ufshcd_init_hba_vreg(hba);
8392 if (err)
8393 goto out;
8394
8395 err = ufshcd_setup_hba_vreg(hba, true);
8396 if (err)
8397 goto out;
8398
8399 err = ufshcd_init_clocks(hba);
8400 if (err)
8401 goto out_disable_hba_vreg;
8402
8403 err = ufshcd_setup_clocks(hba, true);
8404 if (err)
8405 goto out_disable_hba_vreg;
8406
8407 err = ufshcd_init_vreg(hba);
8408 if (err)
8409 goto out_disable_clks;
8410
8411 err = ufshcd_setup_vreg(hba, true);
8412 if (err)
8413 goto out_disable_clks;
8414
8415 err = ufshcd_variant_hba_init(hba);
8416 if (err)
8417 goto out_disable_vreg;
8418
8419 ufs_debugfs_hba_init(hba);
8420
8421 hba->is_powered = true;
8422 goto out;
8423
8424 out_disable_vreg:
8425 ufshcd_setup_vreg(hba, false);
8426 out_disable_clks:
8427 ufshcd_setup_clocks(hba, false);
8428 out_disable_hba_vreg:
8429 ufshcd_setup_hba_vreg(hba, false);
8430 out:
8431 return err;
8432 }
8433
ufshcd_hba_exit(struct ufs_hba * hba)8434 static void ufshcd_hba_exit(struct ufs_hba *hba)
8435 {
8436 if (hba->is_powered) {
8437 ufshcd_exit_clk_scaling(hba);
8438 ufshcd_exit_clk_gating(hba);
8439 if (hba->eh_wq)
8440 destroy_workqueue(hba->eh_wq);
8441 ufs_debugfs_hba_exit(hba);
8442 ufshcd_variant_hba_exit(hba);
8443 ufshcd_setup_vreg(hba, false);
8444 ufshcd_setup_clocks(hba, false);
8445 ufshcd_setup_hba_vreg(hba, false);
8446 hba->is_powered = false;
8447 ufs_put_device_desc(hba);
8448 }
8449 }
8450
8451 /**
8452 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
8453 * power mode
8454 * @hba: per adapter instance
8455 * @pwr_mode: device power mode to set
8456 *
8457 * Returns 0 if requested power mode is set successfully
8458 * Returns non-zero if failed to set the requested power mode
8459 */
ufshcd_set_dev_pwr_mode(struct ufs_hba * hba,enum ufs_dev_pwr_mode pwr_mode)8460 static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
8461 enum ufs_dev_pwr_mode pwr_mode)
8462 {
8463 unsigned char cmd[6] = { START_STOP };
8464 struct scsi_sense_hdr sshdr;
8465 struct scsi_device *sdp;
8466 unsigned long flags;
8467 int ret, retries;
8468
8469 spin_lock_irqsave(hba->host->host_lock, flags);
8470 sdp = hba->sdev_ufs_device;
8471 if (sdp) {
8472 ret = scsi_device_get(sdp);
8473 if (!ret && !scsi_device_online(sdp)) {
8474 ret = -ENODEV;
8475 scsi_device_put(sdp);
8476 }
8477 } else {
8478 ret = -ENODEV;
8479 }
8480 spin_unlock_irqrestore(hba->host->host_lock, flags);
8481
8482 if (ret)
8483 return ret;
8484
8485 /*
8486 * If scsi commands fail, the scsi mid-layer schedules scsi error-
8487 * handling, which would wait for host to be resumed. Since we know
8488 * we are functional while we are here, skip host resume in error
8489 * handling context.
8490 */
8491 hba->host->eh_noresume = 1;
8492
8493 cmd[4] = pwr_mode << 4;
8494
8495 /*
8496 * Current function would be generally called from the power management
8497 * callbacks hence set the RQF_PM flag so that it doesn't resume the
8498 * already suspended childs.
8499 */
8500 for (retries = 3; retries > 0; --retries) {
8501 ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
8502 START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL);
8503 if (!scsi_status_is_check_condition(ret) ||
8504 !scsi_sense_valid(&sshdr) ||
8505 sshdr.sense_key != UNIT_ATTENTION)
8506 break;
8507 }
8508 if (ret) {
8509 sdev_printk(KERN_WARNING, sdp,
8510 "START_STOP failed for power mode: %d, result %x\n",
8511 pwr_mode, ret);
8512 if (driver_byte(ret) == DRIVER_SENSE)
8513 scsi_print_sense_hdr(sdp, NULL, &sshdr);
8514 }
8515
8516 if (!ret)
8517 hba->curr_dev_pwr_mode = pwr_mode;
8518
8519 scsi_device_put(sdp);
8520 hba->host->eh_noresume = 0;
8521 return ret;
8522 }
8523
ufshcd_link_state_transition(struct ufs_hba * hba,enum uic_link_state req_link_state,int check_for_bkops)8524 static int ufshcd_link_state_transition(struct ufs_hba *hba,
8525 enum uic_link_state req_link_state,
8526 int check_for_bkops)
8527 {
8528 int ret = 0;
8529
8530 if (req_link_state == hba->uic_link_state)
8531 return 0;
8532
8533 if (req_link_state == UIC_LINK_HIBERN8_STATE) {
8534 ret = ufshcd_uic_hibern8_enter(hba);
8535 if (!ret) {
8536 ufshcd_set_link_hibern8(hba);
8537 } else {
8538 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8539 __func__, ret);
8540 goto out;
8541 }
8542 }
8543 /*
8544 * If autobkops is enabled, link can't be turned off because
8545 * turning off the link would also turn off the device.
8546 */
8547 else if ((req_link_state == UIC_LINK_OFF_STATE) &&
8548 (!check_for_bkops || !hba->auto_bkops_enabled)) {
8549 /*
8550 * Let's make sure that link is in low power mode, we are doing
8551 * this currently by putting the link in Hibern8. Otherway to
8552 * put the link in low power mode is to send the DME end point
8553 * to device and then send the DME reset command to local
8554 * unipro. But putting the link in hibern8 is much faster.
8555 */
8556 ret = ufshcd_uic_hibern8_enter(hba);
8557 if (ret) {
8558 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8559 __func__, ret);
8560 goto out;
8561 }
8562 /*
8563 * Change controller state to "reset state" which
8564 * should also put the link in off/reset state
8565 */
8566 ufshcd_hba_stop(hba);
8567 /*
8568 * TODO: Check if we need any delay to make sure that
8569 * controller is reset
8570 */
8571 ufshcd_set_link_off(hba);
8572 }
8573
8574 out:
8575 return ret;
8576 }
8577
ufshcd_vreg_set_lpm(struct ufs_hba * hba)8578 static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
8579 {
8580 bool vcc_off = false;
8581
8582 /*
8583 * It seems some UFS devices may keep drawing more than sleep current
8584 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
8585 * To avoid this situation, add 2ms delay before putting these UFS
8586 * rails in LPM mode.
8587 */
8588 if (!ufshcd_is_link_active(hba) &&
8589 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
8590 usleep_range(2000, 2100);
8591
8592 /*
8593 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
8594 * power.
8595 *
8596 * If UFS device and link is in OFF state, all power supplies (VCC,
8597 * VCCQ, VCCQ2) can be turned off if power on write protect is not
8598 * required. If UFS link is inactive (Hibern8 or OFF state) and device
8599 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
8600 *
8601 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
8602 * in low power state which would save some power.
8603 *
8604 * If Write Booster is enabled and the device needs to flush the WB
8605 * buffer OR if bkops status is urgent for WB, keep Vcc on.
8606 */
8607 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8608 !hba->dev_info.is_lu_power_on_wp) {
8609 ufshcd_setup_vreg(hba, false);
8610 vcc_off = true;
8611 } else if (!ufshcd_is_ufs_dev_active(hba)) {
8612 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
8613 vcc_off = true;
8614 if (ufshcd_is_link_hibern8(hba) || ufshcd_is_link_off(hba)) {
8615 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8616 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
8617 }
8618 }
8619
8620 /*
8621 * Some UFS devices require delay after VCC power rail is turned-off.
8622 */
8623 if (vcc_off && hba->vreg_info.vcc &&
8624 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)
8625 usleep_range(5000, 5100);
8626 }
8627
ufshcd_vreg_set_hpm(struct ufs_hba * hba)8628 static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
8629 {
8630 int ret = 0;
8631
8632 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8633 !hba->dev_info.is_lu_power_on_wp) {
8634 ret = ufshcd_setup_vreg(hba, true);
8635 } else if (!ufshcd_is_ufs_dev_active(hba)) {
8636 if (!ufshcd_is_link_active(hba)) {
8637 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
8638 if (ret)
8639 goto vcc_disable;
8640 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
8641 if (ret)
8642 goto vccq_lpm;
8643 }
8644 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
8645 }
8646 goto out;
8647
8648 vccq_lpm:
8649 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8650 vcc_disable:
8651 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
8652 out:
8653 return ret;
8654 }
8655
ufshcd_hba_vreg_set_lpm(struct ufs_hba * hba)8656 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
8657 {
8658 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
8659 ufshcd_setup_hba_vreg(hba, false);
8660 }
8661
ufshcd_hba_vreg_set_hpm(struct ufs_hba * hba)8662 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
8663 {
8664 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
8665 ufshcd_setup_hba_vreg(hba, true);
8666 }
8667
8668 /**
8669 * ufshcd_suspend - helper function for suspend operations
8670 * @hba: per adapter instance
8671 * @pm_op: desired low power operation type
8672 *
8673 * This function will try to put the UFS device and link into low power
8674 * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl"
8675 * (System PM level).
8676 *
8677 * If this function is called during shutdown, it will make sure that
8678 * both UFS device and UFS link is powered off.
8679 *
8680 * NOTE: UFS device & link must be active before we enter in this function.
8681 *
8682 * Returns 0 for success and non-zero for failure
8683 */
ufshcd_suspend(struct ufs_hba * hba,enum ufs_pm_op pm_op)8684 static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
8685 {
8686 int ret = 0;
8687 enum ufs_pm_level pm_lvl;
8688 enum ufs_dev_pwr_mode req_dev_pwr_mode;
8689 enum uic_link_state req_link_state;
8690
8691 hba->pm_op_in_progress = 1;
8692 if (!ufshcd_is_shutdown_pm(pm_op)) {
8693 pm_lvl = ufshcd_is_runtime_pm(pm_op) ?
8694 hba->rpm_lvl : hba->spm_lvl;
8695 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
8696 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
8697 } else {
8698 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
8699 req_link_state = UIC_LINK_OFF_STATE;
8700 }
8701
8702 ufshpb_suspend(hba);
8703
8704 /*
8705 * If we can't transition into any of the low power modes
8706 * just gate the clocks.
8707 */
8708 ufshcd_hold(hba, false);
8709 hba->clk_gating.is_suspended = true;
8710
8711 if (ufshcd_is_clkscaling_supported(hba))
8712 ufshcd_clk_scaling_suspend(hba, true);
8713
8714 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
8715 req_link_state == UIC_LINK_ACTIVE_STATE) {
8716 goto disable_clks;
8717 }
8718
8719 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
8720 (req_link_state == hba->uic_link_state))
8721 goto enable_gating;
8722
8723 /* UFS device & link must be active before we enter in this function */
8724 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
8725 ret = -EINVAL;
8726 goto enable_gating;
8727 }
8728
8729 if (ufshcd_is_runtime_pm(pm_op)) {
8730 if (ufshcd_can_autobkops_during_suspend(hba)) {
8731 /*
8732 * The device is idle with no requests in the queue,
8733 * allow background operations if bkops status shows
8734 * that performance might be impacted.
8735 */
8736 ret = ufshcd_urgent_bkops(hba);
8737 if (ret)
8738 goto enable_gating;
8739 } else {
8740 /* make sure that auto bkops is disabled */
8741 ufshcd_disable_auto_bkops(hba);
8742 }
8743 /*
8744 * If device needs to do BKOP or WB buffer flush during
8745 * Hibern8, keep device power mode as "active power mode"
8746 * and VCC supply.
8747 */
8748 hba->dev_info.b_rpm_dev_flush_capable =
8749 hba->auto_bkops_enabled ||
8750 (((req_link_state == UIC_LINK_HIBERN8_STATE) ||
8751 ((req_link_state == UIC_LINK_ACTIVE_STATE) &&
8752 ufshcd_is_auto_hibern8_enabled(hba))) &&
8753 ufshcd_wb_need_flush(hba));
8754 }
8755
8756 flush_work(&hba->eeh_work);
8757
8758 if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) {
8759 if ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) ||
8760 !ufshcd_is_runtime_pm(pm_op)) {
8761 /* ensure that bkops is disabled */
8762 ufshcd_disable_auto_bkops(hba);
8763 }
8764
8765 if (!hba->dev_info.b_rpm_dev_flush_capable) {
8766 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
8767 if (ret)
8768 goto enable_gating;
8769 }
8770 }
8771
8772 ret = ufshcd_link_state_transition(hba, req_link_state, 1);
8773 if (ret)
8774 goto set_dev_active;
8775
8776 disable_clks:
8777 /*
8778 * Call vendor specific suspend callback. As these callbacks may access
8779 * vendor specific host controller register space call them before the
8780 * host clocks are ON.
8781 */
8782 ret = ufshcd_vops_suspend(hba, pm_op);
8783 if (ret)
8784 goto set_link_active;
8785 /*
8786 * Disable the host irq as host controller as there won't be any
8787 * host controller transaction expected till resume.
8788 */
8789 ufshcd_disable_irq(hba);
8790
8791 ufshcd_setup_clocks(hba, false);
8792
8793 if (ufshcd_is_clkgating_allowed(hba)) {
8794 hba->clk_gating.state = CLKS_OFF;
8795 trace_ufshcd_clk_gating(dev_name(hba->dev),
8796 hba->clk_gating.state);
8797 }
8798
8799 ufshcd_vreg_set_lpm(hba);
8800
8801 /* Put the host controller in low power mode if possible */
8802 ufshcd_hba_vreg_set_lpm(hba);
8803 goto out;
8804
8805 set_link_active:
8806 ufshcd_vreg_set_hpm(hba);
8807 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
8808 ufshcd_set_link_active(hba);
8809 else if (ufshcd_is_link_off(hba))
8810 ufshcd_host_reset_and_restore(hba);
8811 set_dev_active:
8812 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
8813 ufshcd_disable_auto_bkops(hba);
8814 enable_gating:
8815 if (ufshcd_is_clkscaling_supported(hba))
8816 ufshcd_clk_scaling_suspend(hba, false);
8817
8818 hba->clk_gating.is_suspended = false;
8819 hba->dev_info.b_rpm_dev_flush_capable = false;
8820 ufshcd_release(hba);
8821 ufshpb_resume(hba);
8822 out:
8823 if (hba->dev_info.b_rpm_dev_flush_capable) {
8824 schedule_delayed_work(&hba->rpm_dev_flush_recheck_work,
8825 msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS));
8826 }
8827
8828 hba->pm_op_in_progress = 0;
8829
8830 if (ret)
8831 ufshcd_update_evt_hist(hba, UFS_EVT_SUSPEND_ERR, (u32)ret);
8832 return ret;
8833 }
8834
8835 /**
8836 * ufshcd_resume - helper function for resume operations
8837 * @hba: per adapter instance
8838 * @pm_op: runtime PM or system PM
8839 *
8840 * This function basically brings the UFS device, UniPro link and controller
8841 * to active state.
8842 *
8843 * Returns 0 for success and non-zero for failure
8844 */
ufshcd_resume(struct ufs_hba * hba,enum ufs_pm_op pm_op)8845 static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
8846 {
8847 int ret;
8848 enum uic_link_state old_link_state;
8849
8850 hba->pm_op_in_progress = 1;
8851 old_link_state = hba->uic_link_state;
8852
8853 ufshcd_hba_vreg_set_hpm(hba);
8854 ret = ufshcd_vreg_set_hpm(hba);
8855 if (ret)
8856 goto out;
8857
8858 /* Make sure clocks are enabled before accessing controller */
8859 ret = ufshcd_setup_clocks(hba, true);
8860 if (ret)
8861 goto disable_vreg;
8862
8863 /* enable the host irq as host controller would be active soon */
8864 ufshcd_enable_irq(hba);
8865
8866 /*
8867 * Call vendor specific resume callback. As these callbacks may access
8868 * vendor specific host controller register space call them when the
8869 * host clocks are ON.
8870 */
8871 ret = ufshcd_vops_resume(hba, pm_op);
8872 if (ret)
8873 goto disable_irq_and_vops_clks;
8874
8875 if (ufshcd_is_link_hibern8(hba)) {
8876 ret = ufshcd_uic_hibern8_exit(hba);
8877 if (!ret) {
8878 ufshcd_set_link_active(hba);
8879 } else {
8880 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
8881 __func__, ret);
8882 goto vendor_suspend;
8883 }
8884 } else if (ufshcd_is_link_off(hba)) {
8885 /*
8886 * A full initialization of the host and the device is
8887 * required since the link was put to off during suspend.
8888 */
8889 ret = ufshcd_reset_and_restore(hba);
8890 /*
8891 * ufshcd_reset_and_restore() should have already
8892 * set the link state as active
8893 */
8894 if (ret || !ufshcd_is_link_active(hba))
8895 goto vendor_suspend;
8896 }
8897
8898 if (!ufshcd_is_ufs_dev_active(hba)) {
8899 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
8900 if (ret)
8901 goto set_old_link_state;
8902 }
8903
8904 if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
8905 ufshcd_enable_auto_bkops(hba);
8906 else
8907 /*
8908 * If BKOPs operations are urgently needed at this moment then
8909 * keep auto-bkops enabled or else disable it.
8910 */
8911 ufshcd_urgent_bkops(hba);
8912
8913 hba->clk_gating.is_suspended = false;
8914
8915 if (ufshcd_is_clkscaling_supported(hba))
8916 ufshcd_clk_scaling_suspend(hba, false);
8917
8918 /* Enable Auto-Hibernate if configured */
8919 ufshcd_auto_hibern8_enable(hba);
8920
8921 ufshpb_resume(hba);
8922
8923 if (hba->dev_info.b_rpm_dev_flush_capable) {
8924 hba->dev_info.b_rpm_dev_flush_capable = false;
8925 cancel_delayed_work(&hba->rpm_dev_flush_recheck_work);
8926 }
8927
8928 /* Schedule clock gating in case of no access to UFS device yet */
8929 ufshcd_release(hba);
8930
8931 goto out;
8932
8933 set_old_link_state:
8934 ufshcd_link_state_transition(hba, old_link_state, 0);
8935 vendor_suspend:
8936 ufshcd_vops_suspend(hba, pm_op);
8937 disable_irq_and_vops_clks:
8938 ufshcd_disable_irq(hba);
8939 ufshcd_setup_clocks(hba, false);
8940 if (ufshcd_is_clkgating_allowed(hba)) {
8941 hba->clk_gating.state = CLKS_OFF;
8942 trace_ufshcd_clk_gating(dev_name(hba->dev),
8943 hba->clk_gating.state);
8944 }
8945 disable_vreg:
8946 ufshcd_vreg_set_lpm(hba);
8947 out:
8948 hba->pm_op_in_progress = 0;
8949 if (ret)
8950 ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)ret);
8951 return ret;
8952 }
8953
8954 /**
8955 * ufshcd_system_suspend - system suspend routine
8956 * @hba: per adapter instance
8957 *
8958 * Check the description of ufshcd_suspend() function for more details.
8959 *
8960 * Returns 0 for success and non-zero for failure
8961 */
ufshcd_system_suspend(struct ufs_hba * hba)8962 int ufshcd_system_suspend(struct ufs_hba *hba)
8963 {
8964 int ret = 0;
8965 ktime_t start = ktime_get();
8966
8967 down(&hba->host_sem);
8968
8969 if (!hba->is_powered)
8970 return 0;
8971
8972 cancel_delayed_work_sync(&hba->rpm_dev_flush_recheck_work);
8973
8974 if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) ==
8975 hba->curr_dev_pwr_mode) &&
8976 (ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl) ==
8977 hba->uic_link_state) &&
8978 pm_runtime_suspended(hba->dev) &&
8979 !hba->dev_info.b_rpm_dev_flush_capable)
8980 goto out;
8981
8982 if (pm_runtime_suspended(hba->dev)) {
8983 /*
8984 * UFS device and/or UFS link low power states during runtime
8985 * suspend seems to be different than what is expected during
8986 * system suspend. Hence runtime resume the devic & link and
8987 * let the system suspend low power states to take effect.
8988 * TODO: If resume takes longer time, we might have optimize
8989 * it in future by not resuming everything if possible.
8990 */
8991 ret = ufshcd_runtime_resume(hba);
8992 if (ret)
8993 goto out;
8994 }
8995
8996 ret = ufshcd_suspend(hba, UFS_SYSTEM_PM);
8997 out:
8998 trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
8999 ktime_to_us(ktime_sub(ktime_get(), start)),
9000 hba->curr_dev_pwr_mode, hba->uic_link_state);
9001 if (!ret)
9002 hba->is_sys_suspended = true;
9003 else
9004 up(&hba->host_sem);
9005 return ret;
9006 }
9007 EXPORT_SYMBOL(ufshcd_system_suspend);
9008
9009 /**
9010 * ufshcd_system_resume - system resume routine
9011 * @hba: per adapter instance
9012 *
9013 * Returns 0 for success and non-zero for failure
9014 */
9015
ufshcd_system_resume(struct ufs_hba * hba)9016 int ufshcd_system_resume(struct ufs_hba *hba)
9017 {
9018 int ret = 0;
9019 ktime_t start = ktime_get();
9020
9021 if (!hba->is_powered || pm_runtime_suspended(hba->dev))
9022 /*
9023 * Let the runtime resume take care of resuming
9024 * if runtime suspended.
9025 */
9026 goto out;
9027 else
9028 ret = ufshcd_resume(hba, UFS_SYSTEM_PM);
9029 out:
9030 trace_ufshcd_system_resume(dev_name(hba->dev), ret,
9031 ktime_to_us(ktime_sub(ktime_get(), start)),
9032 hba->curr_dev_pwr_mode, hba->uic_link_state);
9033 if (!ret)
9034 hba->is_sys_suspended = false;
9035 up(&hba->host_sem);
9036 return ret;
9037 }
9038 EXPORT_SYMBOL(ufshcd_system_resume);
9039
9040 /**
9041 * ufshcd_runtime_suspend - runtime suspend routine
9042 * @hba: per adapter instance
9043 *
9044 * Check the description of ufshcd_suspend() function for more details.
9045 *
9046 * Returns 0 for success and non-zero for failure
9047 */
ufshcd_runtime_suspend(struct ufs_hba * hba)9048 int ufshcd_runtime_suspend(struct ufs_hba *hba)
9049 {
9050 int ret = 0;
9051 ktime_t start = ktime_get();
9052
9053 if (!hba->is_powered)
9054 goto out;
9055 else
9056 ret = ufshcd_suspend(hba, UFS_RUNTIME_PM);
9057 out:
9058 trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
9059 ktime_to_us(ktime_sub(ktime_get(), start)),
9060 hba->curr_dev_pwr_mode, hba->uic_link_state);
9061 return ret;
9062 }
9063 EXPORT_SYMBOL(ufshcd_runtime_suspend);
9064
9065 /**
9066 * ufshcd_runtime_resume - runtime resume routine
9067 * @hba: per adapter instance
9068 *
9069 * This function basically brings the UFS device, UniPro link and controller
9070 * to active state. Following operations are done in this function:
9071 *
9072 * 1. Turn on all the controller related clocks
9073 * 2. Bring the UniPro link out of Hibernate state
9074 * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device
9075 * to active state.
9076 * 4. If auto-bkops is enabled on the device, disable it.
9077 *
9078 * So following would be the possible power state after this function return
9079 * successfully:
9080 * S1: UFS device in Active state with VCC rail ON
9081 * UniPro link in Active state
9082 * All the UFS/UniPro controller clocks are ON
9083 *
9084 * Returns 0 for success and non-zero for failure
9085 */
ufshcd_runtime_resume(struct ufs_hba * hba)9086 int ufshcd_runtime_resume(struct ufs_hba *hba)
9087 {
9088 int ret = 0;
9089 ktime_t start = ktime_get();
9090
9091 if (!hba->is_powered)
9092 goto out;
9093 else
9094 ret = ufshcd_resume(hba, UFS_RUNTIME_PM);
9095 out:
9096 trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
9097 ktime_to_us(ktime_sub(ktime_get(), start)),
9098 hba->curr_dev_pwr_mode, hba->uic_link_state);
9099 return ret;
9100 }
9101 EXPORT_SYMBOL(ufshcd_runtime_resume);
9102
ufshcd_runtime_idle(struct ufs_hba * hba)9103 int ufshcd_runtime_idle(struct ufs_hba *hba)
9104 {
9105 return 0;
9106 }
9107 EXPORT_SYMBOL(ufshcd_runtime_idle);
9108
9109 /**
9110 * ufshcd_shutdown - shutdown routine
9111 * @hba: per adapter instance
9112 *
9113 * This function would power off both UFS device and UFS link.
9114 *
9115 * Returns 0 always to allow force shutdown even in case of errors.
9116 */
ufshcd_shutdown(struct ufs_hba * hba)9117 int ufshcd_shutdown(struct ufs_hba *hba)
9118 {
9119 int ret = 0;
9120
9121 down(&hba->host_sem);
9122 hba->shutting_down = true;
9123 up(&hba->host_sem);
9124
9125 if (!hba->is_powered)
9126 goto out;
9127
9128 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
9129 goto out;
9130
9131 pm_runtime_get_sync(hba->dev);
9132
9133 ret = ufshcd_suspend(hba, UFS_SHUTDOWN_PM);
9134 out:
9135 if (ret)
9136 dev_err(hba->dev, "%s failed, err %d\n", __func__, ret);
9137 hba->is_powered = false;
9138 /* allow force shutdown even in case of errors */
9139 return 0;
9140 }
9141 EXPORT_SYMBOL(ufshcd_shutdown);
9142
9143 /**
9144 * ufshcd_remove - de-allocate SCSI host and host memory space
9145 * data structure memory
9146 * @hba: per adapter instance
9147 */
ufshcd_remove(struct ufs_hba * hba)9148 void ufshcd_remove(struct ufs_hba *hba)
9149 {
9150 ufs_bsg_remove(hba);
9151 ufshpb_remove(hba);
9152 ufs_sysfs_remove_nodes(hba->dev);
9153 blk_cleanup_queue(hba->tmf_queue);
9154 blk_mq_free_tag_set(&hba->tmf_tag_set);
9155 blk_cleanup_queue(hba->cmd_queue);
9156 scsi_remove_host(hba->host);
9157 /* disable interrupts */
9158 ufshcd_disable_intr(hba, hba->intr_mask);
9159 ufshcd_hba_stop(hba);
9160 ufshcd_hba_exit(hba);
9161 }
9162 EXPORT_SYMBOL_GPL(ufshcd_remove);
9163
9164 /**
9165 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
9166 * @hba: pointer to Host Bus Adapter (HBA)
9167 */
ufshcd_dealloc_host(struct ufs_hba * hba)9168 void ufshcd_dealloc_host(struct ufs_hba *hba)
9169 {
9170 scsi_host_put(hba->host);
9171 }
9172 EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
9173
9174 /**
9175 * ufshcd_set_dma_mask - Set dma mask based on the controller
9176 * addressing capability
9177 * @hba: per adapter instance
9178 *
9179 * Returns 0 for success, non-zero for failure
9180 */
ufshcd_set_dma_mask(struct ufs_hba * hba)9181 static int ufshcd_set_dma_mask(struct ufs_hba *hba)
9182 {
9183 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
9184 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
9185 return 0;
9186 }
9187 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
9188 }
9189
9190 /**
9191 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
9192 * @dev: pointer to device handle
9193 * @hba_handle: driver private handle
9194 * Returns 0 on success, non-zero value on failure
9195 */
ufshcd_alloc_host(struct device * dev,struct ufs_hba ** hba_handle)9196 int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
9197 {
9198 struct Scsi_Host *host;
9199 struct ufs_hba *hba;
9200 int err = 0;
9201
9202 if (!dev) {
9203 dev_err(dev,
9204 "Invalid memory reference for dev is NULL\n");
9205 err = -ENODEV;
9206 goto out_error;
9207 }
9208
9209 host = scsi_host_alloc(&ufshcd_driver_template,
9210 sizeof(struct ufs_hba_add_info));
9211 if (!host) {
9212 dev_err(dev, "scsi_host_alloc failed\n");
9213 err = -ENOMEM;
9214 goto out_error;
9215 }
9216 hba = shost_priv(host);
9217 hba->host = host;
9218 hba->dev = dev;
9219 *hba_handle = hba;
9220 hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
9221 hba->sg_entry_size = sizeof(struct ufshcd_sg_entry);
9222
9223 INIT_LIST_HEAD(&hba->clk_list_head);
9224
9225 out_error:
9226 return err;
9227 }
9228 EXPORT_SYMBOL(ufshcd_alloc_host);
9229
9230 /* This function exists because blk_mq_alloc_tag_set() requires this. */
ufshcd_queue_tmf(struct blk_mq_hw_ctx * hctx,const struct blk_mq_queue_data * qd)9231 static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx,
9232 const struct blk_mq_queue_data *qd)
9233 {
9234 WARN_ON_ONCE(true);
9235 return BLK_STS_NOTSUPP;
9236 }
9237
9238 static const struct blk_mq_ops ufshcd_tmf_ops = {
9239 .queue_rq = ufshcd_queue_tmf,
9240 };
9241
9242 /**
9243 * ufshcd_init - Driver initialization routine
9244 * @hba: per-adapter instance
9245 * @mmio_base: base register address
9246 * @irq: Interrupt line of device
9247 * Returns 0 on success, non-zero value on failure
9248 */
ufshcd_init(struct ufs_hba * hba,void __iomem * mmio_base,unsigned int irq)9249 int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
9250 {
9251 struct request ***tmf_rqs = &ufs_hba_add_info(hba)->tmf_rqs;
9252 int err;
9253 struct Scsi_Host *host = hba->host;
9254 struct device *dev = hba->dev;
9255 char eh_wq_name[sizeof("ufs_eh_wq_00")];
9256
9257 /*
9258 * dev_set_drvdata() must be called before any callbacks are registered
9259 * that use dev_get_drvdata() (frequency scaling, clock scaling, hwmon,
9260 * sysfs).
9261 */
9262 dev_set_drvdata(dev, hba);
9263
9264 if (!mmio_base) {
9265 dev_err(hba->dev,
9266 "Invalid memory reference for mmio_base is NULL\n");
9267 err = -ENODEV;
9268 goto out_error;
9269 }
9270
9271 hba->mmio_base = mmio_base;
9272 hba->irq = irq;
9273 hba->vps = &ufs_hba_vps;
9274
9275 err = ufshcd_hba_init(hba);
9276 if (err)
9277 goto out_error;
9278
9279 /* Read capabilities registers */
9280 err = ufshcd_hba_capabilities(hba);
9281 if (err)
9282 goto out_disable;
9283
9284 /* Get UFS version supported by the controller */
9285 hba->ufs_version = ufshcd_get_ufs_version(hba);
9286
9287 if (hba->ufs_version < ufshci_version(1, 0))
9288 dev_err(hba->dev, "invalid UFS version 0x%x\n",
9289 hba->ufs_version);
9290
9291 /* Get Interrupt bit mask per version */
9292 hba->intr_mask = ufshcd_get_intr_mask(hba);
9293
9294 err = ufshcd_set_dma_mask(hba);
9295 if (err) {
9296 dev_err(hba->dev, "set dma mask failed\n");
9297 goto out_disable;
9298 }
9299
9300 /* Allocate memory for host memory space */
9301 err = ufshcd_memory_alloc(hba);
9302 if (err) {
9303 dev_err(hba->dev, "Memory allocation failed\n");
9304 goto out_disable;
9305 }
9306
9307 /* Configure LRB */
9308 ufshcd_host_memory_configure(hba);
9309
9310 host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED;
9311 host->cmd_per_lun = hba->nutrs - UFSHCD_NUM_RESERVED;
9312 host->max_id = UFSHCD_MAX_ID;
9313 host->max_lun = UFS_MAX_LUNS;
9314 host->max_channel = UFSHCD_MAX_CHANNEL;
9315 host->unique_id = host->host_no;
9316 host->max_cmd_len = UFS_CDB_SIZE;
9317
9318 hba->max_pwr_info.is_valid = false;
9319
9320 /* Initialize work queues */
9321 snprintf(eh_wq_name, sizeof(eh_wq_name), "ufs_eh_wq_%d",
9322 hba->host->host_no);
9323 hba->eh_wq = create_singlethread_workqueue(eh_wq_name);
9324 if (!hba->eh_wq) {
9325 dev_err(hba->dev, "%s: failed to create eh workqueue\n",
9326 __func__);
9327 err = -ENOMEM;
9328 goto out_disable;
9329 }
9330 INIT_WORK(&hba->eh_work, ufshcd_err_handler);
9331 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
9332
9333 sema_init(&hba->host_sem, 1);
9334
9335 /* Initialize UIC command mutex */
9336 mutex_init(&hba->uic_cmd_mutex);
9337
9338 /* Initialize mutex for device management commands */
9339 mutex_init(&hba->dev_cmd.lock);
9340
9341 init_rwsem(&hba->clk_scaling_lock);
9342
9343 ufshcd_init_clk_gating(hba);
9344
9345 ufshcd_init_clk_scaling(hba);
9346
9347 /*
9348 * In order to avoid any spurious interrupt immediately after
9349 * registering UFS controller interrupt handler, clear any pending UFS
9350 * interrupt status and disable all the UFS interrupts.
9351 */
9352 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
9353 REG_INTERRUPT_STATUS);
9354 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
9355 /*
9356 * Make sure that UFS interrupts are disabled and any pending interrupt
9357 * status is cleared before registering UFS interrupt handler.
9358 */
9359 mb();
9360
9361 /* IRQ registration */
9362 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
9363 if (err) {
9364 dev_err(hba->dev, "request irq failed\n");
9365 goto out_disable;
9366 } else {
9367 hba->is_irq_enabled = true;
9368 }
9369
9370 err = scsi_add_host(host, hba->dev);
9371 if (err) {
9372 dev_err(hba->dev, "scsi_add_host failed\n");
9373 goto out_disable;
9374 }
9375
9376 hba->cmd_queue = blk_mq_init_queue(&hba->host->tag_set);
9377 if (IS_ERR(hba->cmd_queue)) {
9378 err = PTR_ERR(hba->cmd_queue);
9379 goto out_remove_scsi_host;
9380 }
9381
9382 hba->tmf_tag_set = (struct blk_mq_tag_set) {
9383 .nr_hw_queues = 1,
9384 .queue_depth = hba->nutmrs,
9385 .ops = &ufshcd_tmf_ops,
9386 .flags = BLK_MQ_F_NO_SCHED,
9387 };
9388 err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
9389 if (err < 0)
9390 goto free_cmd_queue;
9391 hba->tmf_queue = blk_mq_init_queue(&hba->tmf_tag_set);
9392 if (IS_ERR(hba->tmf_queue)) {
9393 err = PTR_ERR(hba->tmf_queue);
9394 goto free_tmf_tag_set;
9395 }
9396 *tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs, sizeof(**tmf_rqs),
9397 GFP_KERNEL);
9398 if (!*tmf_rqs) {
9399 err = -ENOMEM;
9400 goto free_tmf_queue;
9401 }
9402
9403 /* Reset the attached device */
9404 ufshcd_vops_device_reset(hba);
9405
9406 ufshcd_init_crypto(hba);
9407
9408 /* Host controller enable */
9409 err = ufshcd_hba_enable(hba);
9410 if (err) {
9411 dev_err(hba->dev, "Host controller enable failed\n");
9412 ufshcd_print_evt_hist(hba);
9413 ufshcd_print_host_state(hba);
9414 goto free_tmf_queue;
9415 }
9416
9417 /*
9418 * Set the default power management level for runtime and system PM.
9419 * Default power saving mode is to keep UFS link in Hibern8 state
9420 * and UFS device in sleep state.
9421 */
9422 hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9423 UFS_SLEEP_PWR_MODE,
9424 UIC_LINK_HIBERN8_STATE);
9425 hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9426 UFS_SLEEP_PWR_MODE,
9427 UIC_LINK_HIBERN8_STATE);
9428
9429 INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work,
9430 ufshcd_rpm_dev_flush_recheck_work);
9431
9432 /* Set the default auto-hiberate idle timer value to 150 ms */
9433 if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
9434 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
9435 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
9436 }
9437
9438 /* Hold auto suspend until async scan completes */
9439 pm_runtime_get_sync(dev);
9440 atomic_set(&hba->scsi_block_reqs_cnt, 0);
9441 /*
9442 * We are assuming that device wasn't put in sleep/power-down
9443 * state exclusively during the boot stage before kernel.
9444 * This assumption helps avoid doing link startup twice during
9445 * ufshcd_probe_hba().
9446 */
9447 ufshcd_set_ufs_dev_active(hba);
9448
9449 async_schedule(ufshcd_async_scan, hba);
9450 ufs_sysfs_add_nodes(hba);
9451
9452 device_enable_async_suspend(dev);
9453 return 0;
9454
9455 free_tmf_queue:
9456 blk_cleanup_queue(hba->tmf_queue);
9457 free_tmf_tag_set:
9458 blk_mq_free_tag_set(&hba->tmf_tag_set);
9459 free_cmd_queue:
9460 blk_cleanup_queue(hba->cmd_queue);
9461 out_remove_scsi_host:
9462 scsi_remove_host(hba->host);
9463 out_disable:
9464 hba->is_irq_enabled = false;
9465 ufshcd_hba_exit(hba);
9466 out_error:
9467 return err;
9468 }
9469 EXPORT_SYMBOL_GPL(ufshcd_init);
9470
ufshcd_core_init(void)9471 static int __init ufshcd_core_init(void)
9472 {
9473 ufs_debugfs_init();
9474 return 0;
9475 }
9476
ufshcd_core_exit(void)9477 static void __exit ufshcd_core_exit(void)
9478 {
9479 ufs_debugfs_exit();
9480 }
9481
9482 module_init(ufshcd_core_init);
9483 module_exit(ufshcd_core_exit);
9484
9485 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
9486 MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
9487 MODULE_DESCRIPTION("Generic UFS host controller driver Core");
9488 MODULE_LICENSE("GPL");
9489 MODULE_VERSION(UFSHCD_DRIVER_VERSION);
9490