/* * UFS Host Controller driver for Exynos specific extensions * * Copyright (C) 2013-2014 Samsung Electronics Co., Ltd. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. */ #include #include #include #include #include #include #include "ufs-cal-if.h" #include "ufs-vs-mmio.h" #include "ufs-vs-regs.h" #include "ufshcd.h" #include "ufshcd-crypto.h" #include "ufshci.h" #include "unipro.h" #include "ufshcd-pltfrm.h" #include "ufs_quirks.h" #include "ufs-exynos.h" #include #include #include #include #include #include #include #if IS_ENABLED(CONFIG_SEC_UFS_FEATURE) #include "ufs-sec-feature.h" #endif #if IS_ENABLED(CONFIG_EXYNOS_PMU_IF) #include #endif #if IS_ENABLED(CONFIG_EXYNOS_CPUPM) #include #endif #include #include #include "../scsi_priv.h" /* Performance */ #include "ufs-exynos-perf.h" #define IS_C_STATE_ON(h) ((h)->c_state == C_ON) #define PRINT_STATES(h) \ dev_err((h)->dev, "%s: prev h_state %d, cur c_state %d\n", \ __func__, (h)->h_state, (h)->c_state); static struct exynos_ufs *ufs_host_backup[1]; static int ufs_host_index = 0; static const char *res_token[2] = { "passes", "fails", }; enum { UFS_S_TOKEN_FAIL, UFS_S_TOKEN_NUM, }; /* UFSHCD error handling flags */ enum { UFSHCD_EH_IN_PROGRESS = (1 << 0), }; static const char *ufs_s_str_token[UFS_S_TOKEN_NUM] = { "fail to", }; static const char *ufs_pmu_token = "ufs-phy-iso"; static const char *ufs_ext_blks[EXT_BLK_MAX][2] = { {"samsung,sysreg-phandle", "ufs-iocc"}, /* sysreg */ }; static int ufs_ext_ignore[EXT_BLK_MAX] = {0}; /* * This type makes 1st DW and another DW be logged. * The second one is the head of CDB for COMMAND UPIU and * the head of data for DATA UPIU. */ static const int __cport_log_type = 0x22; /* Functions to map registers or to something by other modules */ static void ufs_udelay(u32 n) { udelay(n); } static inline void ufs_map_vs_regions(struct exynos_ufs *ufs) { ufs->handle.hci = ufs->reg_hci; ufs->handle.ufsp = ufs->reg_ufsp; ufs->handle.unipro = ufs->reg_unipro; ufs->handle.pma = ufs->reg_phy; ufs->handle.cport = ufs->reg_cport; ufs->handle.udelay = ufs_udelay; } /* Helper for UFS CAL interface */ int ufs_call_cal(struct exynos_ufs *ufs, int init, void *func) { struct ufs_cal_param *p = &ufs->cal_param; struct ufs_vs_handle *handle = &ufs->handle; int ret; u32 reg; cal_if_func_init fn_init; cal_if_func fn; /* Enable MPHY APB */ reg = hci_readl(handle, HCI_CLKSTOP_CTRL); reg |= (MPHY_APBCLK_STOP | UNIPRO_MCLK_STOP); hci_writel(handle, reg, HCI_CLKSTOP_CTRL); reg = hci_readl(&ufs->handle, HCI_FORCE_HCS); reg &= ~UNIPRO_MCLK_STOP_EN; hci_writel(&ufs->handle, reg & ~MPHY_APBCLK_STOP_EN, HCI_FORCE_HCS); if (init) { fn_init = (cal_if_func_init)func; ret = fn_init(p, ufs_host_index); } else { fn = (cal_if_func)func; ret = fn(p); } if (ret != UFS_CAL_NO_ERROR) { dev_err(ufs->dev, "%s: %d\n", __func__, ret); ret = -EPERM; } /* Disable MPHY APB */ hci_writel(&ufs->handle, reg | MPHY_APBCLK_STOP_EN, HCI_FORCE_HCS); return ret; } static void exynos_ufs_update_active_lanes(struct ufs_hba *hba) { struct exynos_ufs *ufs = to_exynos_ufs(hba); struct ufs_cal_param *p = &ufs->cal_param; struct ufs_vs_handle *handle = &ufs->handle; u32 active_tx_lane = 0; u32 active_rx_lane = 0; active_tx_lane = unipro_readl(handle, UNIP_PA_ACTIVETXDATALENS); active_rx_lane = unipro_readl(handle, UNIP_PA_ACTIVERXDATALENS); /* * Exynos driver doesn't consider asymmetric lanes, e.g. rx=2, tx=1 * so, for the cases, panic occurs to detect when you face new hardware */ if (!active_rx_lane || !active_tx_lane || active_rx_lane != active_tx_lane) { dev_err(hba->dev, "%s: invalid active lanes. rx=%d, tx=%d\n", __func__, active_rx_lane, active_tx_lane); WARN_ON(1); } p->active_rx_lane = (u8)active_rx_lane; p->active_tx_lane = (u8)active_tx_lane; dev_info(ufs->dev, "PA_ActiveTxDataLanes(%d), PA_ActiveRxDataLanes(%d)\n", active_tx_lane, active_rx_lane); } static void exynos_ufs_update_max_gear(struct ufs_hba *hba) { struct exynos_ufs *ufs = to_exynos_ufs(hba); struct uic_pwr_mode *pmd = &ufs->req_pmd_parm; struct ufs_cal_param *p = &ufs->cal_param; u32 max_rx_hs_gear = 0; max_rx_hs_gear = unipro_readl(&ufs->handle, UNIP_PA_MAXRXHSGEAR); p->max_gear = min_t(u8, max_rx_hs_gear, pmd->gear); dev_info(ufs->dev, "max_gear(%d), PA_MaxRxHSGear(%d)\n", p->max_gear, max_rx_hs_gear); /* set for sysfs */ ufs->params[UFS_S_PARAM_EOM_SZ] = EOM_PH_SEL_MAX * EOM_DEF_VREF_MAX * ufs_s_eom_repeat[ufs->cal_param.max_gear]; } static inline void exynos_ufs_ctrl_phy_pwr(struct exynos_ufs *ufs, bool en) { struct ext_cxt *cxt = &ufs->cxt_phy_iso; exynos_pmu_update(cxt->offset, cxt->mask, (en ? 1 : 0) ? cxt->val : 0); } static inline void __thaw_cport_logger(struct ufs_vs_handle *handle) { hci_writel(handle, __cport_log_type, 0x114); hci_writel(handle, 1, 0x110); } static inline void __freeze_cport_logger(struct ufs_vs_handle *handle) { hci_writel(handle, 0, 0x110); } static int exynos_ufs_check_ah8_fsm_state(struct ufs_hba *hba, u32 state) { struct exynos_ufs *ufs = to_exynos_ufs(hba); int cnt = 50; u32 reg; do { udelay(100); reg = hci_readl(&ufs->handle, HCI_AH8_STATE); } while (cnt-- && !(reg & state) && !(reg & HCI_AH8_STATE_ERROR)); dev_info(hba->dev, "%s: cnt = %d, state = %08X, reg = %08X\n", __func__, cnt, state, reg); if (!(reg & state)) { dev_err(hba->dev, "AH8 FSM state is not at required state: req_state = %08X, reg_val = %08X\n", state, reg); return -EINVAL; } return 0; } /* * Exynos debugging main function */ static void exynos_ufs_dump_debug_info(struct ufs_hba *hba) { struct exynos_ufs *ufs = to_exynos_ufs(hba); struct ufs_vs_handle *handle = &ufs->handle; /* start cs, not permit overlapped dump */ if (test_and_set_bit(EXYNOS_UFS_BIT_DBG_DUMP, &ufs->flag)) goto out; pr_info("%s: ah8_enter count: %d, ah8_exit count: %d\n", __func__, ufs->hibern8_enter_cnt, ufs->hibern8_exit_cnt); /* freeze cport logger */ __freeze_cport_logger(handle); exynos_ufs_dump_info(hba, handle, ufs->dev); if (!(hba->saved_uic_err & (1 << 6))) exynos_ufs_fmp_dump_info(hba); /* thaw cport logger */ __thaw_cport_logger(handle); /* finish cs */ clear_bit(EXYNOS_UFS_BIT_DBG_DUMP, &ufs->flag); out: #if !IS_ENABLED(CONFIG_SAMSUNG_PRODUCT_SHIP) #ifndef CONFIG_SCSI_UFS_EXYNOS_BLOCK_WDT_RST if (hba->saved_err & SYSTEM_BUS_FATAL_ERROR) dbg_snapshot_expire_watchdog(); #endif #endif return; } /* static void exynos_ufs_dbg_command_log(struct ufs_hba *hba, struct scsi_cmnd *cmd, const char *str, int tag) { struct exynos_ufs *ufs = to_exynos_ufs(hba); struct ufs_vs_handle *handle = &ufs->handle; if (!strcmp(str, "send")) { exynos_ufs_cmd_log_start(handle, hba, cmd); } else if (!strcmp(str, "complete")) exynos_ufs_cmd_log_end(handle, hba, tag); } */ inline void exynos_ufs_ctrl_auto_hci_clk(struct exynos_ufs *ufs, bool en) { u32 reg = hci_readl(&ufs->handle, HCI_FORCE_HCS); if (en) hci_writel(&ufs->handle, reg | HCI_CORECLK_STOP_EN, HCI_FORCE_HCS); else hci_writel(&ufs->handle, reg & ~HCI_CORECLK_STOP_EN, HCI_FORCE_HCS); } static inline void exynos_ufs_ctrl_clk(struct exynos_ufs *ufs, bool en) { u32 reg = hci_readl(&ufs->handle, HCI_FORCE_HCS); if (en) hci_writel(&ufs->handle, reg | CLK_STOP_CTRL_EN_ALL, HCI_FORCE_HCS); else hci_writel(&ufs->handle, reg & ~CLK_STOP_CTRL_EN_ALL, HCI_FORCE_HCS); } static inline void exynos_ufs_gate_clk(struct exynos_ufs *ufs, bool en) { u32 reg = hci_readl(&ufs->handle, HCI_CLKSTOP_CTRL); if (en) hci_writel(&ufs->handle, reg | CLK_STOP_ALL, HCI_CLKSTOP_CTRL); else hci_writel(&ufs->handle, reg & ~CLK_STOP_ALL, HCI_CLKSTOP_CTRL); } static void exynos_ufs_set_unipro_mclk(struct exynos_ufs *ufs) { int ret; u32 val; val = (u32)clk_get_rate(ufs->clk_unipro); if (val != ufs->mclk_rate) { ret = clk_set_rate(ufs->clk_unipro, ufs->mclk_rate); dev_info(ufs->dev, "previous mclk: %u, ret: %d\n", val, ret); } dev_info(ufs->dev, "mclk: %u\n", ufs->mclk_rate); } static void exynos_ufs_fit_aggr_timeout(struct exynos_ufs *ufs) { u32 cnt_val; unsigned long nVal; /* IA_TICK_SEL : 1(1us_TO_CNT_VAL) */ nVal = hci_readl(&ufs->handle, HCI_UFSHCI_V2P1_CTRL); nVal |= IA_TICK_SEL; hci_writel(&ufs->handle, nVal, HCI_UFSHCI_V2P1_CTRL); cnt_val = ufs->mclk_rate / 1000000 ; hci_writel(&ufs->handle, cnt_val & CNT_VAL_1US_MASK, HCI_1US_TO_CNT_VAL); } static void exynos_ufs_init_pmc_req(struct ufs_hba *hba, struct ufs_pa_layer_attr *pwr_max, struct ufs_pa_layer_attr *pwr_req) { struct exynos_ufs *ufs = to_exynos_ufs(hba); struct uic_pwr_mode *req_pmd = &ufs->req_pmd_parm; struct uic_pwr_mode *act_pmd = &ufs->act_pmd_parm; /* * Exynos driver doesn't consider asymmetric lanes, e.g. rx=2, tx=1 * so, for the cases, panic occurs to detect when you face new hardware. * pwr_max comes from core driver, i.e. ufshcd. That keeps the number * of connected lanes. */ if (pwr_max->lane_rx != pwr_max->lane_tx) { dev_err(hba->dev, "%s: invalid connected lanes. rx=%d, tx=%d\n", __func__, pwr_max->lane_rx, pwr_max->lane_tx); WARN_ON(1); } /* gear change parameters to core driver */ pwr_req->gear_rx = act_pmd->gear= min_t(u8, pwr_max->gear_rx, req_pmd->gear); pwr_req->gear_tx = act_pmd->gear = min_t(u8, pwr_max->gear_tx, req_pmd->gear); pwr_req->lane_rx = act_pmd->lane = min_t(u8, pwr_max->lane_rx, req_pmd->lane); pwr_req->lane_tx = act_pmd->lane = min_t(u8, pwr_max->lane_tx, req_pmd->lane); pwr_req->pwr_rx = act_pmd->mode = req_pmd->mode; pwr_req->pwr_tx = act_pmd->mode = req_pmd->mode; pwr_req->hs_rate = act_pmd->hs_series = req_pmd->hs_series; } static void exynos_ufs_dev_hw_reset(struct ufs_hba *hba) { struct exynos_ufs *ufs = to_exynos_ufs(hba); /* bit[1] for resetn */ hci_writel(&ufs->handle, 0 << 0, HCI_GPIO_OUT); udelay(5); hci_writel(&ufs->handle, 1 << 0, HCI_GPIO_OUT); } static void exynos_ufs_config_host(struct exynos_ufs *ufs) { u32 reg; /* internal clock control */ exynos_ufs_ctrl_auto_hci_clk(ufs, false); exynos_ufs_set_unipro_mclk(ufs); /* period for interrupt aggregation */ exynos_ufs_fit_aggr_timeout(ufs); /* misc HCI configurations */ hci_writel(&ufs->handle, 0xA, HCI_DATA_REORDER); hci_writel(&ufs->handle, PRDT_PREFECT_EN | PRDT_SET_SIZE(12), HCI_TXPRDT_ENTRY_SIZE); hci_writel(&ufs->handle, PRDT_SET_SIZE(12), HCI_RXPRDT_ENTRY_SIZE); /* I_T_L_Q isn't used at the beginning */ ufs->nexus = 0xFFFFFFFF; hci_writel(&ufs->handle, ufs->nexus, HCI_UTRL_NEXUS_TYPE); hci_writel(&ufs->handle, 0xFFFFFFFF, HCI_UTMRL_NEXUS_TYPE); reg = hci_readl(&ufs->handle, HCI_AXIDMA_RWDATA_BURST_LEN) & ~BURST_LEN(0); hci_writel(&ufs->handle, WLU_EN | BURST_LEN(3), HCI_AXIDMA_RWDATA_BURST_LEN); /* * enable HWACG control by IOP * * default value 1->0 at KC. * always "0"(controlled by UFS_ACG_DISABLE) */ reg = hci_readl(&ufs->handle, HCI_IOP_ACG_DISABLE); hci_writel(&ufs->handle, reg & (~HCI_IOP_ACG_DISABLE_EN), HCI_IOP_ACG_DISABLE); } static int exynos_ufs_config_externals(struct exynos_ufs *ufs) { int ret = 0; int i; struct regmap **p = NULL; struct ext_cxt *q = NULL; /* PHY isolation bypass */ exynos_ufs_ctrl_phy_pwr(ufs, true); /* Set for UFS iocc */ for (i = EXT_SYSREG, p = &ufs->regmap_sys, q = &ufs->cxt_iocc; i < EXT_BLK_MAX; i++, p++, q++) { if (IS_ERR_OR_NULL(*p)) { if (!ufs_ext_ignore[i]) ret = -EINVAL; else continue; dev_err(ufs->dev, "Unable to control %s\n", ufs_ext_blks[i][1]); goto out; } regmap_update_bits(*p, q->offset, q->mask, q->val); } out: return ret; } static int exynos_ufs_get_clks(struct ufs_hba *hba) { struct exynos_ufs *ufs = to_exynos_ufs(hba); struct list_head *head = &hba->clk_list_head; struct ufs_clk_info *clki; int i = 0; if (!head || list_empty(head)) goto out; list_for_each_entry(clki, head, list) { /* * get clock with an order listed in device tree * * hci, unipro */ if (i == 0) { ufs->clk_hci = clki->clk; } else if (i == 1) { ufs->clk_unipro = clki->clk; ufs->mclk_rate = (u32)clk_get_rate(ufs->clk_unipro); } i++; } out: if (!ufs->clk_hci || !ufs->clk_unipro) return -EINVAL; return 0; } static void exynos_ufs_set_features(struct ufs_hba *hba) { struct exynos_ufs *ufs = to_exynos_ufs(hba); struct device_node *np = hba->dev->of_node; /* caps */ #if IS_ENABLED(CONFIG_SEC_UFS_FEATURE) hba->caps = UFSHCD_CAP_CLK_GATING; #else hba->caps = UFSHCD_CAP_WB_EN | UFSHCD_CAP_CLK_GATING; #endif if (!ufs->ah8_ahit) hba->caps |= UFSHCD_CAP_HIBERN8_WITH_CLK_GATING; /* quirks of common driver */ hba->quirks = UFSHCD_QUIRK_PRDT_BYTE_GRAN | UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR | UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR | UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR | UFSHCD_QUIRK_ALIGN_SG_WITH_PAGE_SIZE | UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL | UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING; if(of_find_property(np, "fixed-prdt-req_list-ocs", NULL)) { hba->quirks &= ~(UFSHCD_QUIRK_PRDT_BYTE_GRAN | UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR | UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR); } } /* * Exynos-specific callback functions */ static int exynos_ufs_init(struct ufs_hba *hba) { struct exynos_ufs *ufs = to_exynos_ufs(hba); int ret; /* refer to hba */ ufs->hba = hba; /* configure externals */ ret = exynos_ufs_config_externals(ufs); if (ret) return ret; /* to read standard hci registers */ ufs->handle.std = hba->mmio_base; /* get some clock sources and debug infomation structures */ ret = exynos_ufs_get_clks(hba); if (ret) return ret; /* set features, such as caps or quirks */ exynos_ufs_set_features(hba); exynos_ufs_fmp_init(hba); exynos_ufs_srpmb_config(hba); /* init io perf stat, need an identifier later */ if (!ufs_perf_init(&ufs->perf, ufs->hba)) dev_err(ufs->dev, "Not enable UFS performance mode\n"); hba->ahit = ufs->ah8_ahit; return 0; } static int exynos_ufs_wait_for_register(struct ufs_vs_handle *handle, u32 reg, u32 mask, u32 val, unsigned long interval_us, unsigned long timeout_ms) { int err = 0; unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms); /* ignore bits that we don't intend to wait on */ val = val & mask; while ((std_readl(handle, reg) & mask) != val) { usleep_range(interval_us, interval_us + 50); if (time_after(jiffies, timeout)) { if ((std_readl(handle, reg) & mask) != val) err = -ETIMEDOUT; break; } } return err; } /* This is same code with ufshcd_clear_cmd() */ static int exynos_ufs_clear_cmd(struct ufs_hba *hba, int tag) { struct exynos_ufs *ufs = to_exynos_ufs(hba); struct ufs_vs_handle *handle = &ufs->handle; int err = 0; u32 mask = 1 << tag; /* clear outstanding transaction before retry */ if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR) std_writel(handle, (1 << tag), REG_UTP_TRANSFER_REQ_LIST_CLEAR); else std_writel(handle, ~(1 << tag), REG_UTP_TRANSFER_REQ_LIST_CLEAR); /* * wait for for h/w to clear corresponding bit in door-bell. * max. wait is 1 sec. */ err = exynos_ufs_wait_for_register(handle, REG_UTP_TRANSFER_REQ_DOOR_BELL, mask, ~mask, 1000, 1000); if (!err && (hba->ufs_version >= ufshci_version(3, 0))) std_writel(handle, 1UL << tag, REG_UTP_TRANSFER_REQ_LIST_COMPL); return err; } static void __requeue_after_reset(struct ufs_hba *hba, bool reset) { struct ufshcd_lrb *lrbp; struct scsi_cmnd *cmd; unsigned long completed_reqs = hba->outstanding_reqs; int index, err = 0; pr_err("%s: outstanding reqs=0x%lx\n", __func__, hba->outstanding_reqs); for_each_set_bit(index, &completed_reqs, hba->nutrs) { lrbp = &hba->lrb[index]; lrbp->compl_time_stamp = ktime_get(); cmd = lrbp->cmd; if (!cmd) continue; if (!test_and_clear_bit(index, &hba->outstanding_reqs)) continue; if (!reset) { err = exynos_ufs_clear_cmd(hba, index); if (err) pr_err("%s: Failed to clear tag = %d\n", __func__, index); } trace_android_vh_ufs_compl_command(hba, lrbp); scsi_dma_unmap(cmd); if (cmd->cmnd[0] == START_STOP) { set_driver_byte(cmd, SAM_STAT_TASK_ABORTED); set_host_byte(cmd, DID_ABORT); pr_err("%s: tag %d, cmd %02x aborted\n", __func__, index, cmd->cmnd[0]); } else { set_host_byte(cmd, DID_REQUEUE); pr_err("%s: tag %d, cmd %02x requeued\n", __func__, index, cmd->cmnd[0]); } ufshcd_crypto_clear_prdt(hba, lrbp); lrbp->cmd = NULL; ufshcd_release(hba); cmd->scsi_done(cmd); } pr_info("%s: clk_gating.active_reqs: %d\n", __func__, hba->clk_gating.active_reqs); } static void exynos_ufs_init_host(struct ufs_hba *hba) { struct exynos_ufs *ufs = to_exynos_ufs(hba); unsigned long timeout = jiffies + msecs_to_jiffies(1); int err = 0; u32 reg; /* host reset */ hci_writel(&ufs->handle, UFS_SW_RST_MASK, HCI_SW_RST); do { if (!(hci_readl(&ufs->handle, HCI_SW_RST) & UFS_SW_RST_MASK)) goto success; } while (time_before(jiffies, timeout)); dev_err(ufs->dev, "timeout host sw-reset\n"); err = -ETIMEDOUT; exynos_ufs_dump_info(hba, &ufs->handle, ufs->dev); exynos_ufs_fmp_dump_info(hba); #if !IS_ENABLED(CONFIG_SAMSUNG_PRODUCT_SHIP) #ifndef CONFIG_SCSI_UFS_EXYNOS_BLOCK_WDT_RST dbg_snapshot_expire_watchdog(); #endif #endif goto out; success: /* report to IS.UE reg when UIC error happens during AH8 */ if (ufshcd_is_auto_hibern8_supported(hba)) { reg = hci_readl(&ufs->handle, HCI_VENDOR_SPECIFIC_IE); hci_writel(&ufs->handle, (reg | AH8_ERR_REPORT_UE), HCI_VENDOR_SPECIFIC_IE); } /* performance */ if (ufs->perf) ufs_perf_reset(ufs->perf); /* configure host */ exynos_ufs_config_host(ufs); exynos_ufs_fmp_set_crypto_cfg(hba); __requeue_after_reset(hba, true); ufs->suspend_done = false; out: if (!err) ufs_sec_check_device_stuck(); return; } static void hibern8_enter_stat(struct exynos_ufs *ufs) { u32 upmcrs; /* * hibern8 enter results are comprised of upmcrs and uic result, * but you may not get the uic result because an interrupt * for ah8 error is raised and ISR handles this before this is called. * Honestly, upmcrs may also be the same case because UFS driver * considers ah8 error as fatal and host reset is asserted subsequently. * So you don't trust this return value. */ upmcrs = __get_upmcrs(ufs); trace_ufshcd_profile_hibern8(dev_name(ufs->dev), "enter", 0, upmcrs); ufs->hibern8_enter_cnt++; ufs->hibern8_state = UFS_STATE_AH8; } static void hibern8_exit_stat(struct exynos_ufs *ufs) { /* * this is called before actual hibern8 exit, * so return value is meaningless */ trace_ufshcd_profile_hibern8(dev_name(ufs->dev), "exit", 0, 0); ufs->hibern8_exit_cnt++; ufs->hibern8_state = UFS_STATE_IDLE; } static int exynos_ufs_setup_clocks(struct ufs_hba *hba, bool on, enum ufs_notify_change_status notify) { struct exynos_ufs *ufs = to_exynos_ufs(hba); int ret = 0; if (on) { if (notify == PRE_CHANGE) { /* Clear for SICD */ #if IS_ENABLED(CONFIG_EXYNOS_CPUPM) exynos_update_ip_idle_status(ufs->idle_ip_index, 0); #endif /* PM Qos hold for stability */ #if IS_ENABLED(CONFIG_EXYNOS_PM_QOS) || IS_ENABLED(CONFIG_EXYNOS_PM_QOS_MODULE) exynos_pm_qos_update_request(&ufs->pm_qos_int, ufs->pm_qos_int_value); #endif } else { hibern8_exit_stat(ufs); ufs->c_state = C_ON; } } else { if (notify == PRE_CHANGE) { ufs->c_state = C_OFF; hibern8_enter_stat(ufs); /* reset perf context to start again */ if (ufs->perf) ufs_perf_reset(ufs->perf); } else { /* PM Qos Release for stability */ #if IS_ENABLED(CONFIG_EXYNOS_PM_QOS) || IS_ENABLED(CONFIG_EXYNOS_PM_QOS_MODULE) exynos_pm_qos_update_request(&ufs->pm_qos_int, 0); #endif /* Set for SICD */ #if IS_ENABLED(CONFIG_EXYNOS_CPUPM) exynos_update_ip_idle_status(ufs->idle_ip_index, 1); #endif } } return ret; } static int exynos_ufs_get_available_lane(struct ufs_hba *hba) { struct exynos_ufs *ufs = to_exynos_ufs(hba); struct ufs_vs_handle *handle = &ufs->handle; int ret = -EINVAL; /* Get the available lane count */ ufs->available_lane_tx = unipro_readl(handle, UNIP_PA_AVAILTXDATALENS); ufs->available_lane_rx = unipro_readl(handle, UNIP_PA_AVAILRXDATALENS); /* * Exynos driver doesn't consider asymmetric lanes, e.g. rx=2, tx=1 * so, for the cases, panic occurs to detect when you face new hardware */ if (!ufs->available_lane_rx || !ufs->available_lane_tx || (ufs->available_lane_rx != ufs->available_lane_tx)) { dev_err(hba->dev, "%s: invalid host available lanes. rx=%d, tx=%d\n", __func__, ufs->available_lane_rx, ufs->available_lane_tx); BUG_ON(1); goto out; } ret = exynos_ufs_dbg_set_lanes(handle, ufs->dev, ufs->available_lane_rx); if (ret) goto out; ufs->num_rx_lanes = ufs->available_lane_rx; ufs->num_tx_lanes = ufs->available_lane_tx; ret = 0; out: return ret; } static void exynos_ufs_override_hba_params(struct ufs_hba *hba) { hba->spm_lvl = UFS_PM_LVL_5; } static int exynos_ufs_hce_enable_notify(struct ufs_hba *hba, enum ufs_notify_change_status notify) { struct exynos_ufs *ufs = to_exynos_ufs(hba); int ret = 0; PRINT_STATES(ufs); switch (notify) { case PRE_CHANGE: /* * This function is called in ufshcd_hba_enable, * maybe boot, wake-up or link start-up failure cases. * To start safely, reset of entire logics of host * is required */ exynos_ufs_init_host(hba); /* device reset */ exynos_ufs_dev_hw_reset(hba); break; case POST_CHANGE: exynos_ufs_ctrl_clk(ufs, true); exynos_ufs_gate_clk(ufs, false); ret = exynos_ufs_get_available_lane(hba); /* freeze cport logger */ __thaw_cport_logger(&ufs->handle); ufs->h_state = H_RESET; unipro_writel(&ufs->handle, DBG_SUITE1_ENABLE, UNIP_PA_DBG_OPTION_SUITE_1); unipro_writel(&ufs->handle, DBG_SUITE2_ENABLE, UNIP_PA_DBG_OPTION_SUITE_2); break; default: break; } return ret; } static int exynos_ufs_link_startup_notify(struct ufs_hba *hba, enum ufs_notify_change_status notify) { struct exynos_ufs *ufs = to_exynos_ufs(hba); struct ufs_cal_param *p = &ufs->cal_param; int ret = 0; switch (notify) { case PRE_CHANGE: /* override some parameters from core driver */ exynos_ufs_override_hba_params(hba); if (!IS_C_STATE_ON(ufs) || ufs->h_state != H_RESET) PRINT_STATES(ufs); /* hci */ hci_writel(&ufs->handle, DFES_ERR_EN | DFES_DEF_DL_ERRS, HCI_ERROR_EN_DL_LAYER); hci_writel(&ufs->handle, DFES_ERR_EN | DFES_DEF_N_ERRS, HCI_ERROR_EN_N_LAYER); hci_writel(&ufs->handle, DFES_ERR_EN | DFES_DEF_T_ERRS, HCI_ERROR_EN_T_LAYER); /* cal */ p->mclk_rate = ufs->mclk_rate; p->available_lane = ufs->num_rx_lanes; ret = ufs_call_cal(ufs, 0, ufs_cal_pre_link); break; case POST_CHANGE: /* update max gear after link*/ exynos_ufs_update_max_gear(ufs->hba); p->connected_tx_lane = unipro_readl(&ufs->handle, UNIP_PA_CONNECTEDTXDATALENS); p->connected_rx_lane = unipro_readl(&ufs->handle, UNIP_PA_CONNECTEDRXDATALENS); /* cal */ ret = ufs_call_cal(ufs, 0, ufs_cal_post_link); /* print link start-up result */ dev_info(ufs->dev, "UFS link start-up %s\n", (!ret) ? res_token[0] : res_token[1]); ufs->h_state = H_LINK_UP; break; default: break; } return ret; } static int exynos_ufs_pwr_change_notify(struct ufs_hba *hba, enum ufs_notify_change_status notify, struct ufs_pa_layer_attr *pwr_max, struct ufs_pa_layer_attr *pwr_req) { struct exynos_ufs *ufs = to_exynos_ufs(hba); struct uic_pwr_mode *act_pmd = &ufs->act_pmd_parm; struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; int ret = 0; switch (notify) { case PRE_CHANGE: if (!IS_C_STATE_ON(ufs) || ufs->h_state != H_REQ_BUSY) PRINT_STATES(ufs); /* Set PMC parameters to be requested */ exynos_ufs_init_pmc_req(hba, pwr_max, pwr_req); /* cal */ ufs->cal_param.pmd = act_pmd; ret = ufs_call_cal(ufs, 0, ufs_cal_pre_pmc); break; case POST_CHANGE: /* update active lanes after pmc */ exynos_ufs_update_active_lanes(hba); /* cal */ ret = ufs_call_cal(ufs, 0, ufs_cal_post_pmc); dev_info(ufs->dev, "Power mode change(%d): M(%d)G(%d)L(%d)HS-series(%d)\n", ret, act_pmd->mode, act_pmd->gear, act_pmd->lane, act_pmd->hs_series); /* * print gear change result. * Exynos driver always considers gear change to * HS-B and fast mode. */ if (ufs->req_pmd_parm.mode == FAST_MODE && ufs->req_pmd_parm.hs_series == PA_HS_MODE_B) dev_info(ufs->dev, "HS mode config %s\n", (!ret) ? res_token[0] : res_token[1]); ufs->h_state = H_LINK_BOOST; /* * W/A for AH8 * have to use dme_peer cmd after send uic cmd */ ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_tx); ufs->skip_flush = false; break; default: break; } return ret; } /* * Translating a bit-wise variable to a count essentially requires * requires an iteration that sometimes lead to a big cost. * This function remove the iteration and reduce time complexbility * up to O(1). Now, even if the doorbell becomes biggers, * there will be no change of time. */ static u32 __ufs_cal_set_bits(u32 reqs) { /* Hamming Weight Algorithm */ reqs = reqs - ((reqs >> 1) & 0x55555555); reqs = (reqs & 0x33333333) + ((reqs >> 2) & 0x33333333); reqs = (reqs + (reqs >> 4)) & 0x0F0F0F0F; return (reqs * 0x01010101) >> 24; } static void exynos_ufs_set_nexus_t_xfer_req(struct ufs_hba *hba, int tag, bool cmd) { struct exynos_ufs *ufs = to_exynos_ufs(hba); enum ufs_perf_op op = UFS_PERF_OP_NONE; struct ufshcd_lrb *lrbp; struct scsi_cmnd *scmd; struct ufs_vs_handle *handle = &ufs->handle; u32 qd; /* * Wait up to 50us, seen as safe value for nexus configuration. * Accessing HCI_UTRL_NEXUS_TYPE takes hundreds of nanoseconds * given w/ simulation but we don't know when the previous access * will finish. To reduce its polling latency, we use 10ns. */ int timeout_cnt = 50000 / 10; int wait_ns = 10; if (!IS_C_STATE_ON(ufs) || (ufs->h_state != H_LINK_UP && ufs->h_state != H_LINK_BOOST && ufs->h_state != H_REQ_BUSY)) PRINT_STATES(ufs); /* perf */ lrbp = &hba->lrb[tag]; if (lrbp && lrbp->cmd) { scmd = lrbp->cmd; /* performance, only for SCSI */ if (scmd->cmnd[0] == 0x28) op = UFS_PERF_OP_R; else if (scmd->cmnd[0] == 0x2A) op = UFS_PERF_OP_W; else if (scmd->cmnd[0] == 0x35) op = UFS_PERF_OP_S; if (ufs->perf) { qd = __ufs_cal_set_bits(hba->outstanding_reqs); ufs_perf_update(ufs->perf, qd, scmd, op); } /* cmd_logging */ exynos_ufs_cmd_log_start(handle, hba, scmd); } /* check if an update is needed */ while (test_and_set_bit(EXYNOS_UFS_BIT_CHK_NEXUS, &ufs->flag) && timeout_cnt--) ndelay(wait_ns); if (cmd) { if (test_and_set_bit(tag, &ufs->nexus)) goto out; } else { if (!test_and_clear_bit(tag, &ufs->nexus)) goto out; } hci_writel(&ufs->handle, (u32)ufs->nexus, HCI_UTRL_NEXUS_TYPE); out: clear_bit(EXYNOS_UFS_BIT_CHK_NEXUS, &ufs->flag); ufs->h_state = H_REQ_BUSY; } static void exynos_ufs_check_uac(struct ufs_hba *hba, int tag, bool cmd) { struct exynos_ufs *ufs = to_exynos_ufs(hba); struct ufshcd_lrb *lrbp; struct scsi_cmnd *scmd; struct utp_upiu_rsp *ucd_rsp_ptr; int result = 0; if (!cmd) return; lrbp = &hba->lrb[tag]; ucd_rsp_ptr = lrbp->ucd_rsp_ptr; scmd = lrbp->cmd; result = be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24; if (result != UPIU_TRANSACTION_RESPONSE) return; result = be32_to_cpu(ucd_rsp_ptr->header.dword_1); if (result & SAM_STAT_CHECK_CONDITION) { result &= ~SAM_STAT_CHECK_CONDITION; result |= SAM_STAT_BUSY; ucd_rsp_ptr->header.dword_1 = cpu_to_be32(result); scmd->result |= (DID_SOFT_ERROR << 16); } ufs->resume_state = 0; } static void exynos_ufs_compl_nexus_t_xfer_req(void *data, struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { struct exynos_ufs *ufs = to_exynos_ufs(hba); struct ufs_vs_handle *handle = &ufs->handle; unsigned long completed_reqs; u32 tr_doorbell; int tag; if (!lrbp) { dev_err(hba->dev, "%s: lrbp: local reference block is null\n", __func__); return; } tag = lrbp->task_tag; /* When it's first command completion after resume, check uac. */ if (ufs->resume_state || (lrbp->lun != 0)) exynos_ufs_check_uac(hba, tag, (lrbp->cmd ? true : false)); /* cmd_logging */ if (lrbp->cmd) exynos_ufs_cmd_log_end(handle, hba, tag); tr_doorbell = std_readl(handle, REG_UTP_TRANSFER_REQ_DOOR_BELL); completed_reqs = tr_doorbell ^ hba->outstanding_reqs; if (!(hba->outstanding_reqs^completed_reqs)) { if (ufs->perf) ufs_perf_reset(ufs->perf); } } static void exynos_ufs_set_nexus_t_task_mgmt(struct ufs_hba *hba, int tag, u8 tm_func) { struct exynos_ufs *ufs = to_exynos_ufs(hba); u32 type; if (!IS_C_STATE_ON(ufs) || (ufs->h_state != H_LINK_BOOST && ufs->h_state != H_TM_BUSY && ufs->h_state != H_REQ_BUSY)) PRINT_STATES(ufs); type = hci_readl(&ufs->handle, HCI_UTMRL_NEXUS_TYPE); switch (tm_func) { case UFS_ABORT_TASK: case UFS_QUERY_TASK: type |= (1 << tag); break; case UFS_ABORT_TASK_SET: case UFS_CLEAR_TASK_SET: case UFS_LOGICAL_RESET: case UFS_QUERY_TASK_SET: type &= ~(1 << tag); break; } hci_writel(&ufs->handle, type, HCI_UTMRL_NEXUS_TYPE); ufs->h_state = H_TM_BUSY; } static void __check_int_errors(void *data, struct ufs_hba *hba, bool queue_eh_work) { struct exynos_ufs *ufs = to_exynos_ufs(hba); u32 reg; reg = hci_readl(&ufs->handle, HCI_AH8_STATE); if ((hba->errors & UIC_ERROR) && (reg & HCI_AH8_STATE_ERROR)) { dev_err(hba->dev, "%s: ufs uic error: 0x%x, ah8 state err: 0x%x\n", __func__, (hba->errors & UIC_ERROR), (reg & HCI_AH8_STATE_ERROR)); ufshcd_set_link_broken(hba); } if (hba->pm_op_in_progress && queue_eh_work && !ufs->suspend_done) { pr_err("%s: reset during suspend\n", __func__); __requeue_after_reset(hba, false); } if (ufshcd_is_auto_hibern8_supported(hba)) hci_writel(&ufs->handle, AH8_ERR_REPORT_UE, HCI_VENDOR_SPECIFIC_IS); } static void exynos_ufs_hibern8_notify(struct ufs_hba *hba, enum uic_cmd_dme cmd, enum ufs_notify_change_status notify) { struct exynos_ufs *ufs = to_exynos_ufs(hba); int ret; if (cmd == UIC_CMD_DME_HIBER_ENTER) { if (!IS_C_STATE_ON(ufs) || (ufs->h_state != H_LINK_UP && ufs->h_state != H_REQ_BUSY && ufs->h_state != H_LINK_BOOST)) PRINT_STATES(ufs); if (notify == PRE_CHANGE) { if (ufshcd_is_auto_hibern8_supported(hba) && ufs->ah8_ahit) { ufshcd_auto_hibern8_update(hba, 0); ret = exynos_ufs_check_ah8_fsm_state(hba, HCI_AH8_IDLE_STATE); if (ret) dev_err(hba->dev, "%s: exynos_ufs_check_ah8_fsm_state return value is %d\n", __func__, ret); } } else { /* cal */ ufs_call_cal(ufs, 0, ufs_cal_post_h8_enter); /* Internal clock off */ exynos_ufs_gate_clk(ufs, true); ufs->h_state_prev = ufs->h_state; ufs->h_state = H_HIBERN8; } } else { if (notify == PRE_CHANGE) { ufs->h_state = ufs->h_state_prev; /* Internal clock on */ exynos_ufs_gate_clk(ufs, false); /* cal */ ufs_call_cal(ufs, 0, ufs_cal_pre_h8_exit); } else { int h8_delay_ms_ovly = ufs->params[UFS_S_PARAM_H8_D_MS]; /* override h8 enter delay */ if (h8_delay_ms_ovly) hba->clk_gating.delay_ms = (unsigned long)h8_delay_ms_ovly; } } } static int __exynos_ufs_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) { struct exynos_ufs *ufs = to_exynos_ufs(hba); int ret; if (!IS_C_STATE_ON(ufs) || ufs->h_state != H_HIBERN8) PRINT_STATES(ufs); #if IS_ENABLED(CONFIG_SEC_UFS_FEATURE) if (pm_op == UFS_SHUTDOWN_PM) ufs_sec_print_err_info(hba); else ufs_sec_print_err(); #endif /* Make sure AH8 FSM is at Hibern State. * When doing SW H8 Enter UIC CMD, don't need to check this state. */ if (ufshcd_is_auto_hibern8_enabled(hba)) { ret = exynos_ufs_check_ah8_fsm_state(hba, HCI_AH8_HIBERNATION_STATE); if (ret) { dev_err(hba->dev, "%s: exynos_ufs_check_ah8_fsm_state return value = %d\n", __func__, ret); exynos_ufs_dump_debug_info(hba); ufshcd_set_link_off(hba); return ret; } } hci_writel(&ufs->handle, 0 << 0, HCI_GPIO_OUT); exynos_ufs_ctrl_phy_pwr(ufs, false); #if IS_ENABLED(CONFIG_EXYNOS_PM_QOS) || IS_ENABLED(CONFIG_EXYNOS_PM_QOS_MODULE) exynos_pm_qos_update_request(&ufs->pm_qos_int, 0); #endif ufs->suspend_done = true; ufs->h_state = H_SUSPEND; return 0; } static int __exynos_ufs_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) { struct exynos_ufs *ufs = to_exynos_ufs(hba); int ret = 0; if (!IS_C_STATE_ON(ufs) || ufs->h_state != H_SUSPEND) PRINT_STATES(ufs); /* system init */ ret = exynos_ufs_config_externals(ufs); if (ret) return ret; exynos_ufs_fmp_resume(hba); ufs->resume_state = 1; #if IS_ENABLED(CONFIG_SEC_UFS_FEATURE) /* * Change WB state to WB_OFF to default in resume sequence. * In system PM, the link is "link off state" or "hibern8". * In case of link off state, * just reset the WB state because UFS device needs to setup link. * In Hibern8 state, * wb_off reset and WB off are required. */ if (ufshcd_is_system_pm(pm_op)) ufs_sec_wb_force_off(hba); #endif ufshcd_set_link_off(hba); return 0; } #if 0 static void exynos_ufs_perf_mode(struct ufs_hba *hba, struct scsi_cmnd *cmd) { struct exynos_ufs *ufs = to_exynos_ufs(hba); enum ufs_perf_op op = UFS_PERF_OP_NONE; /* performance, only for SCSI */ if (cmd->cmnd[0] == 0x28) op = UFS_PERF_OP_R; else if (cmd->cmnd[0] == 0x2A) op = UFS_PERF_OP_W; ufs_perf_update_stat(ufs->perf, cmd->request->__data_len, op); } #endif static int __apply_dev_quirks(struct ufs_hba *hba) { struct exynos_ufs *ufs = to_exynos_ufs(hba); struct ufs_vs_handle *handle = &ufs->handle; u32 peer_hibern8time; struct ufs_cal_param *p = &ufs->cal_param; /* 50us is a heuristic value, so it could change later */ u32 ref_gate_margin = (hba->dev_info.wspecversion >= 0x300) ? hba->dev_info.clk_gating_wait_us : 50; /* * we're here, that means the sequence up to fDeviceinit * is done successfully. */ dev_info(ufs->dev, "UFS device initialized\n"); peer_hibern8time = unipro_readl(handle, UNIP_PA_HIBERN8TIME); unipro_writel(handle, peer_hibern8time + 1, UNIP_PA_HIBERN8TIME); p->ah8_thinern8_time = unipro_readl(handle, UNIP_PA_HIBERN8TIME) * H8T_GRANULARITY; p->ah8_brefclkgatingwaittime = ref_gate_margin; #if IS_ENABLED(CONFIG_SEC_UFS_FEATURE) /* check only at the first init */ if (!(hba->eh_flags || hba->pm_op_in_progress)) { /* sec special features */ ufs_set_sec_features(hba); #if IS_ENABLED(CONFIG_SCSI_UFS_TEST_MODE) dev_info(hba->dev, "UFS test mode enabled\n"); #endif } ufs_sec_feature_config(hba); #endif return 0; } static void __fixup_dev_quirks(struct ufs_hba *hba) { struct device_node *np = hba->dev->of_node; hba->dev_quirks &= ~(UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS); if(of_find_property(np, "support-extended-features", NULL)) hba->dev_quirks |= UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES; } static void exynos_ufs_register_vendor_hooks(void) { register_trace_android_vh_ufs_compl_command(exynos_ufs_compl_nexus_t_xfer_req, NULL); register_trace_android_vh_ufs_check_int_errors(__check_int_errors, NULL); #if IS_ENABLED(CONFIG_SEC_UFS_FEATURE) /* register vendor hooks */ ufs_sec_register_vendor_hooks(); #endif } /* to make device state active */ static int __device_reset(struct ufs_hba *hba) { struct exynos_ufs *ufs = to_exynos_ufs(hba); struct ufs_vs_handle *handle = &ufs->handle; u32 reg; hba->ahit = ufs->ah8_ahit; ufs->hibern8_enter_cnt = 0; ufs->hibern8_exit_cnt = 0; reg = hci_readl(handle, HCI_AH8_STATE); if (reg & HCI_AH8_STATE_ERROR) { reg = std_readl(handle, REG_INTERRUPT_STATUS); if (reg & UFSHCD_AH8_CHECK_IS) { std_writel(handle, reg, REG_INTERRUPT_STATUS); reg = std_readl(handle, REG_INTERRUPT_STATUS); } if (reg & UFSHCD_AH8_CHECK_IS) pr_info("%s: can't handle AH8_RESET, IS:%08x\n", __func__, reg); else { pr_info("%s: before AH8_RESET, AH8_STATE:%08x\n", __func__, hci_readl(handle, HCI_AH8_STATE)); hci_writel(handle, HCI_CLEAR_AH8_FSM, HCI_AH8_RESET); pr_info("%s: after AH8_RESET, AH8_STATE:%08x\n", __func__, hci_readl(handle, HCI_AH8_STATE)); } } /* guarantee device internal cache flush */ if (hba->eh_flags && !ufs->skip_flush) { dev_info(hba->dev, "%s: Waiting device internal cache flush\n", __func__); ssleep(2); ufs->skip_flush = true; #if IS_ENABLED(CONFIG_SEC_UFS_FEATURE) ufs_sec_check_hwrst_cnt(); #endif #if IS_ENABLED(CONFIG_SCSI_UFS_TEST_MODE) /* no dump for some cases, get dump before recovery */ exynos_ufs_dump_debug_info(hba); #endif } return 0; } #define UFS_TEST_COUNT 3 static void exynos_ufs_event_notify(struct ufs_hba *hba, enum ufs_event_type evt, void *data) { #if IS_ENABLED(CONFIG_SCSI_UFS_TEST_MODE) struct ufs_event_hist *e; e = &hba->ufs_stats.event[evt]; if ((e->cnt > UFS_TEST_COUNT) && (evt == UFS_EVT_PA_ERR || evt == UFS_EVT_DL_ERR || evt == UFS_EVT_LINK_STARTUP_FAIL)) { exynos_ufs_dump_debug_info(hba); BUG(); } #endif #if IS_ENABLED(CONFIG_SEC_UFS_FEATURE) ufs_sec_check_op_err(hba, evt, data); #endif } static struct ufs_hba_variant_ops exynos_ufs_ops = { .init = exynos_ufs_init, .setup_clocks = exynos_ufs_setup_clocks, .hce_enable_notify = exynos_ufs_hce_enable_notify, .link_startup_notify = exynos_ufs_link_startup_notify, .pwr_change_notify = exynos_ufs_pwr_change_notify, .setup_xfer_req = exynos_ufs_set_nexus_t_xfer_req, .setup_task_mgmt = exynos_ufs_set_nexus_t_task_mgmt, .hibern8_notify = exynos_ufs_hibern8_notify, .dbg_register_dump = exynos_ufs_dump_debug_info, //.dbg_command_log = exynos_ufs_dbg_command_log, .suspend = __exynos_ufs_suspend, .resume = __exynos_ufs_resume, //.perf_mode = exynos_ufs_perf_mode, .apply_dev_quirks = __apply_dev_quirks, .fixup_dev_quirks = __fixup_dev_quirks, .device_reset = __device_reset, .event_notify = exynos_ufs_event_notify, }; /* * This function is to define offset, mask and shift to access somewhere. */ static int exynos_ufs_set_context_for_access(struct device *dev, const char *name, struct ext_cxt *cxt) { struct device_node *np; int ret = -EINVAL; np = of_get_child_by_name(dev->of_node, name); if (!np) { dev_info(dev, "get node(%s) doesn't exist\n", name); goto out; } ret = of_property_read_u32(np, "offset", &cxt->offset); if (ret == 0) { ret = of_property_read_u32(np, "mask", &cxt->mask); if (ret == 0) ret = of_property_read_u32(np, "val", &cxt->val); } if (ret != 0) { dev_err(dev, "%s set cxt(%s) val\n", ufs_s_str_token[UFS_S_TOKEN_FAIL], name); goto out; } ret = 0; out: return ret; } static int exynos_ufs_populate_dt_extern(struct device *dev, struct exynos_ufs *ufs) { struct device_node *np = dev->of_node; struct regmap **reg = NULL; struct ext_cxt *cxt; bool is_dma_coherent = !!of_find_property(dev->of_node, "dma-coherent", NULL); int i; int ret = -EINPROGRESS; /* * pmu for phy isolation. for the pmu, we use api from outside, not regmap */ cxt = &ufs->cxt_phy_iso; ret = exynos_ufs_set_context_for_access(dev, ufs_pmu_token, cxt); if (ret) { dev_err(dev, "%s: %u: %s get %s\n", __func__, __LINE__, ufs_s_str_token[UFS_S_TOKEN_FAIL], ufs_pmu_token); goto out; } /* others */ /* * w/o 'dma-coherent' means the descriptors would be non-cacheable. * so, iocc should be disabled. */ if (!is_dma_coherent) { ufs_ext_ignore[EXT_SYSREG] = 1; dev_info(dev, "no 'dma-coherent', ufs iocc disabled\n"); } for (i = 0, reg = &ufs->regmap_sys, cxt = &ufs->cxt_iocc; i < EXT_BLK_MAX; i++, reg++, cxt++) { /* look up phandle for external regions */ *reg = syscon_regmap_lookup_by_phandle(np, ufs_ext_blks[i][0]); if (IS_ERR(*reg)) { dev_err(dev, "%s: %u: %s find %s\n", __func__, __LINE__, ufs_s_str_token[UFS_S_TOKEN_FAIL], ufs_ext_blks[i][0]); if (ufs_ext_ignore[i]) continue; else ret = PTR_ERR(*reg); goto out; } /* get and pars echild nodes for external regions in ufs node */ ret = exynos_ufs_set_context_for_access(dev, ufs_ext_blks[i][1], cxt); if (ret) { dev_err(dev, "%s: %u: %s get %s\n", __func__, __LINE__, ufs_s_str_token[UFS_S_TOKEN_FAIL], ufs_ext_blks[i][1]); if (ufs_ext_ignore[i]) { ret = 0; continue; } goto out; } dev_info(dev, "%s: offset 0x%x, mask 0x%x, value 0x%x\n", ufs_ext_blks[i][1], cxt->offset, cxt->mask, cxt->val); } out: return ret; } static int exynos_ufs_get_pwr_mode(struct device_node *np, struct exynos_ufs *ufs) { struct uic_pwr_mode *pmd = &ufs->req_pmd_parm; pmd->mode = FAST_MODE; if (of_property_read_u8(np, "ufs,pmd-attr-lane", &pmd->lane)) pmd->lane = 1; if (of_property_read_u8(np, "ufs,pmd-attr-gear", &pmd->gear)) pmd->gear = 1; pmd->hs_series = PA_HS_MODE_B; return 0; } static int exynos_ufs_populate_dt(struct device *dev, struct exynos_ufs *ufs) { struct device_node *np = dev->of_node; struct device_node *child_np; int ret; /* Regmap for external regions */ ret = exynos_ufs_populate_dt_extern(dev, ufs); if (ret) { dev_err(dev, "%s populate dt-pmu\n", ufs_s_str_token[UFS_S_TOKEN_FAIL]); goto out; } /* Get exynos-evt version for featuring. Now get main_rev instead of dt */ ufs->cal_param.evt_ver = (u8)exynos_soc_info.main_rev; dev_info(dev, "exynos ufs evt version : %d\n", ufs->cal_param.evt_ver); /* PM QoS */ child_np = of_get_child_by_name(np, "ufs-pm-qos"); ufs->pm_qos_int_value = 0; if (!child_np) dev_info(dev, "No ufs-pm-qos node, not guarantee pm qos\n"); else { of_property_read_u32(child_np, "freq-int", &ufs->pm_qos_int_value); } /* UIC specifics */ exynos_ufs_get_pwr_mode(np, ufs); ufs->cal_param.board = 0; of_property_read_u8(np, "brd-for-cal", &ufs->cal_param.board); // ufs_perf_populate_dt(ufs->perf, np); ufs->ah8_ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 20) | FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 2); if (ufs->ah8_ahit) { ufs->cal_param.support_ah8_cal = true; ufs->cal_param.ah8_thinern8_time = 3; ufs->cal_param.ah8_brefclkgatingwaittime = 1; } else { ufs->cal_param.support_ah8_cal = false; } out: return ret; } static int exynos_ufs_ioremap(struct exynos_ufs *ufs, struct platform_device *pdev) { /* Indicators for logs */ static const char *ufs_region_names[NUM_OF_UFS_MMIO_REGIONS + 1] = { "", /* standard hci */ "reg_hci", /* exynos-specific hci */ "reg_unipro", /* unipro */ "reg_ufsp", /* ufs protector */ "reg_phy", /* phy */ "reg_cport", /* cport */ }; struct device *dev = &pdev->dev; struct resource *res; void **p = NULL; int i = 0; int ret = 0; for (i = 1, p = &ufs->reg_hci; i < NUM_OF_UFS_MMIO_REGIONS + 1; i++, p++) { res = platform_get_resource(pdev, IORESOURCE_MEM, i); if (!res) { ret = -ENOMEM; break; } *p = devm_ioremap_resource(dev, res); if (!*p) { ret = -ENOMEM; break; } dev_info(dev, "%-10s 0x%llx\n", ufs_region_names[i], *p); } if (ret) dev_err(dev, "%s ioremap for %s, 0x%llx\n", ufs_s_str_token[UFS_S_TOKEN_FAIL], ufs_region_names[i]); dev_info(dev, "\n"); return ret; } /* sysfs to support utc, eom or whatever */ struct exynos_ufs_sysfs_attr { struct attribute attr; ssize_t (*show)(struct exynos_ufs *ufs, char *buf, enum exynos_ufs_param_id id); int (*store)(struct exynos_ufs *ufs, u32 value, enum exynos_ufs_param_id id); int id; }; static ssize_t exynos_ufs_sysfs_default_show(struct exynos_ufs *ufs, char *buf, enum exynos_ufs_param_id id) { return snprintf(buf, PAGE_SIZE, "%u\n", ufs->params[id]); } #define UFS_S_RO(_name, _id) \ static struct exynos_ufs_sysfs_attr ufs_s_##_name = { \ .attr = { .name = #_name, .mode = 0444 }, \ .id = _id, \ .show = exynos_ufs_sysfs_default_show, \ } #define UFS_S_RW(_name, _id) \ static struct exynos_ufs_sysfs_attr ufs_s_##_name = { \ .attr = { .name = #_name, .mode = 0666 }, \ .id = _id, \ .show = exynos_ufs_sysfs_default_show, \ } UFS_S_RO(eom_version, UFS_S_PARAM_EOM_VER); UFS_S_RO(eom_size, UFS_S_PARAM_EOM_SZ); static int exynos_ufs_sysfs_lane_store(struct exynos_ufs *ufs, u32 value, enum exynos_ufs_param_id id) { if (value >= ufs->num_rx_lanes) { dev_err(ufs->dev, "%s set lane to %u. Its max is %u\n", ufs_s_str_token[UFS_S_TOKEN_FAIL], value, ufs->num_rx_lanes); return -EINVAL; } ufs->params[id] = value; return 0; } static struct exynos_ufs_sysfs_attr ufs_s_lane = { .attr = { .name = "lane", .mode = 0666 }, .id = UFS_S_PARAM_LANE, .show = exynos_ufs_sysfs_default_show, .store = exynos_ufs_sysfs_lane_store, }; static int exynos_ufs_sysfs_mon_store(struct exynos_ufs *ufs, u32 value, enum exynos_ufs_param_id id) { struct ufs_vs_handle *handle = &ufs->handle; u32 reg; if (value & UFS_S_MON_LV1) { /* Trigger HCI error */ dev_info(ufs->dev, "Interface error test\n"); unipro_writel(handle, (1 << BIT_POS_DBG_DL_RX_INFO_FORCE | DL_RX_INFO_TYPE_ERROR_DETECTED << BIT_POS_DBG_DL_RX_INFO_TYPE | 1 << 15), UNIP_DBG_RX_INFO_CONTROL_DIRECT); } else if (value & UFS_S_MON_LV2) { /* Block all the interrupts */ dev_info(ufs->dev, "Device error test\n"); reg = std_readl(handle, REG_INTERRUPT_ENABLE); std_writel(handle, (reg & ~UTP_TRANSFER_REQ_COMPL), REG_INTERRUPT_ENABLE); } else { dev_err(ufs->dev, "Undefined level\n"); return -EINVAL; } ufs->params[id] = value; return 0; } static struct exynos_ufs_sysfs_attr ufs_s_monitor = { .attr = { .name = "monitor", .mode = 0666 }, .id = UFS_S_PARAM_MON, .show = exynos_ufs_sysfs_default_show, .store = exynos_ufs_sysfs_mon_store, }; static int exynos_ufs_sysfs_eom_offset_store(struct exynos_ufs *ufs, u32 offset, enum exynos_ufs_param_id id) { u32 num_per_lane = ufs->params[UFS_S_PARAM_EOM_SZ]; if (offset >= num_per_lane) { dev_err(ufs->dev, "%s set ofs to %u. The available offset is up to %u\n", ufs_s_str_token[UFS_S_TOKEN_FAIL], offset, num_per_lane - 1); return -EINVAL; } ufs->params[id] = offset; return 0; } static struct exynos_ufs_sysfs_attr ufs_s_eom_offset = { .attr = { .name = "eom_offset", .mode = 0666 }, .id = UFS_S_PARAM_EOM_OFS, .show = exynos_ufs_sysfs_default_show, .store = exynos_ufs_sysfs_eom_offset_store, }; static ssize_t exynos_ufs_sysfs_show_h8_delay(struct exynos_ufs *ufs, char *buf, enum exynos_ufs_param_id id) { return snprintf(buf, PAGE_SIZE, "%lu\n", ufs->hba->clk_gating.delay_ms); } static struct exynos_ufs_sysfs_attr ufs_s_h8_delay_ms = { .attr = { .name = "h8_delay_ms", .mode = 0666 }, .id = UFS_S_PARAM_H8_D_MS, .show = exynos_ufs_sysfs_show_h8_delay, }; static ssize_t exynos_ufs_sysfs_show_ah8_cnt(struct exynos_ufs *ufs, char *buf, enum exynos_ufs_param_id id) { return snprintf(buf, PAGE_SIZE, "AH8_Enter_cnt: %d, AH8_Exit_cnt: %d\n", ufs->hibern8_enter_cnt, ufs->hibern8_exit_cnt); } static struct exynos_ufs_sysfs_attr ufs_s_ah8_cnt = { .attr = { .name = "ah8_cnt_show", .mode = 0666 }, .show = exynos_ufs_sysfs_show_ah8_cnt, }; static int exynos_ufs_sysfs_eom_store(struct exynos_ufs *ufs, u32 value, enum exynos_ufs_param_id id) { ssize_t ret; ret = ufs_call_cal(ufs, 0, ufs_cal_eom); if (ret) dev_err(ufs->dev, "%s store eom data\n", ufs_s_str_token[UFS_S_TOKEN_FAIL]); return ret; } static struct exynos_ufs_sysfs_attr ufs_s_eom = { .attr = { .name = "eom", .mode = 0222 }, .store = exynos_ufs_sysfs_eom_store, }; /* Convert Auto-Hibernate Idle Timer register value to microseconds */ static int exynos_ufs_ahit_to_us(u32 ahit) { int timer = FIELD_GET(UFSHCI_AHIBERN8_TIMER_MASK, ahit); int scale = FIELD_GET(UFSHCI_AHIBERN8_SCALE_MASK, ahit); for (; scale > 0; --scale) timer *= UFSHCI_AHIBERN8_SCALE_FACTOR; return timer; } /* Convert microseconds to Auto-Hibernate Idle Timer register value */ static u32 exynos_ufs_us_to_ahit(unsigned int timer) { unsigned int scale; for (scale = 0; timer > UFSHCI_AHIBERN8_TIMER_MASK; ++scale) timer /= UFSHCI_AHIBERN8_SCALE_FACTOR; return FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, timer) | FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, scale); } static ssize_t exynos_ufs_auto_hibern8_show(struct exynos_ufs *ufs, char *buf, enum exynos_ufs_param_id id) { u32 ahit; struct ufs_hba *hba = ufs->hba; if (!ufshcd_is_auto_hibern8_supported(hba)) return -EOPNOTSUPP; pm_runtime_get_sync(hba->dev); ufshcd_hold(hba, false); ahit = ufshcd_readl(hba, REG_AUTO_HIBERNATE_IDLE_TIMER); ufshcd_release(hba); pm_runtime_put_sync(hba->dev); return scnprintf(buf, PAGE_SIZE, "%d\n", exynos_ufs_ahit_to_us(ahit)); } static int exynos_ufs_auto_hibern8_store(struct exynos_ufs *ufs, u32 value, enum exynos_ufs_param_id id) { struct ufs_hba *hba = ufs->hba; if (!ufshcd_is_auto_hibern8_supported(hba)) return -EOPNOTSUPP; if (value > UFSHCI_AHIBERN8_MAX) return -EINVAL; ufshcd_auto_hibern8_update(hba, exynos_ufs_us_to_ahit(value)); return 0; } static struct exynos_ufs_sysfs_attr ufs_s_auto_hibern8 = { .attr = {.name = "auto_hibern8", .mode = 0666}, .show = exynos_ufs_auto_hibern8_show, .store = exynos_ufs_auto_hibern8_store, }; static void exynos_ufs_release(struct ufs_hba *hba) { if (!ufshcd_is_clkgating_allowed(hba)) return; hba->clk_gating.active_reqs--; if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL || hba->outstanding_tasks || hba->active_uic_cmd || hba->uic_async_done || hba->clk_gating.state == CLKS_OFF) return; hba->clk_gating.state = REQ_CLKS_OFF; queue_delayed_work(hba->clk_gating.clk_gating_workq, &hba->clk_gating.gate_work, msecs_to_jiffies(hba->clk_gating.delay_ms)); } static ssize_t exynos_ufs_clkgate_enable_show(struct exynos_ufs *ufs, char *buf, enum exynos_ufs_param_id id) { struct ufs_hba *hba = ufs->hba; return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_gating.is_enabled); } static int exynos_ufs_clkgate_enable_store(struct exynos_ufs *ufs, u32 value, enum exynos_ufs_param_id id) { struct ufs_hba *hba = ufs->hba; unsigned long flags; value = !!value; spin_lock_irqsave(hba->host->host_lock, flags); if (value == hba->clk_gating.is_enabled) goto out; if (value) exynos_ufs_release(hba); else hba->clk_gating.active_reqs++; hba->clk_gating.is_enabled = value; out: spin_unlock_irqrestore(hba->host->host_lock, flags); return 0; } static struct exynos_ufs_sysfs_attr ufs_s_clkgate_enable = { .attr = {.name = "clkgate_enable", .mode = 0644}, .show = exynos_ufs_clkgate_enable_show, .store = exynos_ufs_clkgate_enable_store, }; static ssize_t ufs_exynos_gear_scale_show(struct exynos_ufs *ufs, char *buf, enum exynos_ufs_param_id id) { struct uic_pwr_mode *pmd = &ufs->req_pmd_parm; return snprintf(buf, PAGE_SIZE, "%d\n", pmd->gear); } static int ufs_exynos_gear_scale_store(struct exynos_ufs *ufs, u32 value, enum exynos_ufs_param_id id) { struct ufs_hba *hba = ufs->hba; ssize_t ret = 0; value = !!value; ret = ufs_gear_change(hba, value); return ret; } static struct exynos_ufs_sysfs_attr ufs_s_gear_scale = { .attr = { .name = "gear_scale", .mode = 0666 }, .store = ufs_exynos_gear_scale_store, .show = ufs_exynos_gear_scale_show, }; #define UFS_S_EOM_RO(_name) \ static ssize_t exynos_ufs_sysfs_eom_##_name##_show(struct exynos_ufs *ufs, \ char *buf, \ enum exynos_ufs_param_id id) \ { \ struct ufs_eom_result_s *p = \ ufs->cal_param.eom[ufs->params[UFS_S_PARAM_LANE]] + \ ufs->params[UFS_S_PARAM_EOM_OFS]; \ return snprintf(buf, PAGE_SIZE, "%u", p->v_##_name); \ }; \ static struct exynos_ufs_sysfs_attr ufs_s_eom_##_name = { \ .attr = { .name = "eom_"#_name, .mode = 0444 }, \ .id = -1, \ .show = exynos_ufs_sysfs_eom_##_name##_show, \ } UFS_S_EOM_RO(phase); UFS_S_EOM_RO(vref); UFS_S_EOM_RO(err); const static struct attribute *ufs_s_sysfs_attrs[] = { &ufs_s_eom_version.attr, &ufs_s_eom_size.attr, &ufs_s_eom_offset.attr, &ufs_s_eom.attr, &ufs_s_eom_phase.attr, &ufs_s_eom_vref.attr, &ufs_s_eom_err.attr, &ufs_s_lane.attr, &ufs_s_h8_delay_ms.attr, &ufs_s_monitor.attr, &ufs_s_auto_hibern8.attr, &ufs_s_clkgate_enable.attr, &ufs_s_ah8_cnt.attr, &ufs_s_gear_scale.attr, NULL, }; static ssize_t exynos_ufs_sysfs_show(struct kobject *kobj, struct attribute *attr, char *buf) { struct exynos_ufs *ufs = container_of(kobj, struct exynos_ufs, sysfs_kobj); struct exynos_ufs_sysfs_attr *param = container_of(attr, struct exynos_ufs_sysfs_attr, attr); if (!param->show) { dev_err(ufs->dev, "%s show thanks to no existence of show\n", ufs_s_str_token[UFS_S_TOKEN_FAIL]); return 0; } return param->show(ufs, buf, param->id); } static ssize_t exynos_ufs_sysfs_store(struct kobject *kobj, struct attribute *attr, const char *buf, size_t length) { struct exynos_ufs *ufs = container_of(kobj, struct exynos_ufs, sysfs_kobj); struct exynos_ufs_sysfs_attr *param = container_of(attr, struct exynos_ufs_sysfs_attr, attr); u32 val; int ret = 0; if (kstrtou32(buf, 10, &val)) return -EINVAL; if (!param->store) { ufs->params[param->id] = val; return length; } ret = param->store(ufs, val, param->id); return (ret == 0) ? length : (ssize_t)ret; } static const struct sysfs_ops exynos_ufs_sysfs_ops = { .show = exynos_ufs_sysfs_show, .store = exynos_ufs_sysfs_store, }; static struct kobj_type ufs_s_ktype = { .sysfs_ops = &exynos_ufs_sysfs_ops, .release = NULL, }; static int exynos_ufs_sysfs_init(struct exynos_ufs *ufs) { int error = -ENOMEM; int i; struct ufs_eom_result_s *p; /* allocate memory for eom per lane */ for (i = 0; i < MAX_LANE; i++) { ufs->cal_param.eom[i] = devm_kcalloc(ufs->dev, EOM_MAX_SIZE, sizeof(struct ufs_eom_result_s), GFP_KERNEL); p = ufs->cal_param.eom[i]; if (!p) { dev_err(ufs->dev, "%s allocate eom data\n", ufs_s_str_token[UFS_S_TOKEN_FAIL]); goto fail_mem; } } /* create a path of /sys/kernel/ufs_x */ kobject_init(&ufs->sysfs_kobj, &ufs_s_ktype); error = kobject_add(&ufs->sysfs_kobj, kernel_kobj, "ufs_%c", (char)(ufs->id + '0')); if (error) { dev_err(ufs->dev, "%s register sysfs directory: %d\n", ufs_s_str_token[UFS_S_TOKEN_FAIL], error); goto fail_kobj; } /* create attributes */ error = sysfs_create_files(&ufs->sysfs_kobj, ufs_s_sysfs_attrs); if (error) { dev_err(ufs->dev, "%s create sysfs files: %d\n", ufs_s_str_token[UFS_S_TOKEN_FAIL], error); goto fail_kobj; } /* * Set sysfs params by default. The values could change or * initial configuration could be done elsewhere in the future. * * As for eom_version, you have to move it to store a value * from device tree when eom code is revised, even though I expect * it's not gonna to happen. */ ufs->params[UFS_S_PARAM_EOM_VER] = 0; ufs->params[UFS_S_PARAM_MON] = 0; ufs->params[UFS_S_PARAM_H8_D_MS] = 4; return 0; fail_kobj: kobject_put(&ufs->sysfs_kobj); fail_mem: for (i = 0; i < MAX_LANE; i++) { if (ufs->cal_param.eom[i]) devm_kfree(ufs->dev, ufs->cal_param.eom[i]); ufs->cal_param.eom[i] = NULL; } return error; } static inline void exynos_ufs_sysfs_exit(struct exynos_ufs *ufs) { kobject_put(&ufs->sysfs_kobj); } static u64 exynos_ufs_dma_mask = DMA_BIT_MASK(32); static void __ufs_resume_async(struct work_struct *work) { struct exynos_ufs *ufs = container_of(work, struct exynos_ufs, resume_work); struct ufs_hba *hba = ufs->hba; /* to block incoming commands prior to completion of resuming */ hba->ufshcd_state = UFSHCD_STATE_RESET; if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1) scsi_block_requests(hba->host); /* adding delay to guarantee vcc discharge for abnormal wakeup case */ if (!ufs->deep_suspended) msleep(12); ufshcd_system_resume(hba); if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt)) scsi_unblock_requests(hba->host); } static int exynos_ufs_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct exynos_ufs *ufs; int ret; dev_info(dev, "%s: start\n", __func__); dev_info(dev, "===============================\n"); /* allocate memory */ ufs = devm_kzalloc(dev, sizeof(*ufs), GFP_KERNEL); if (!ufs) { ret = -ENOMEM; goto out; } ufs->dev = dev; dev->platform_data = ufs; dev->dma_mask = &exynos_ufs_dma_mask; /* remap regions */ ret = exynos_ufs_ioremap(ufs, pdev); if (ret) goto out; ufs_map_vs_regions(ufs); /* populate device tree nodes */ ret = exynos_ufs_populate_dt(dev, ufs); if (ret) { dev_err(dev, "%s get dt info.\n", ufs_s_str_token[UFS_S_TOKEN_FAIL]); return ret; } /* init cal */ ufs->cal_param.handle = &ufs->handle; ret = ufs_call_cal(ufs, 1, ufs_cal_init); if (ret) return ret; dev_info(dev, "===============================\n"); /* idle ip nofification for SICD, disable by default */ #if IS_ENABLED(CONFIG_EXYNOS_CPUPM) ufs->idle_ip_index = exynos_get_idle_ip_index(dev_name(ufs->dev), 1); exynos_update_ip_idle_status(ufs->idle_ip_index, 0); #endif /* register pm qos knobs */ #if IS_ENABLED(CONFIG_EXYNOS_PM_QOS) || IS_ENABLED(CONFIG_EXYNOS_PM_QOS_MODULE) exynos_pm_qos_add_request(&ufs->pm_qos_int, PM_QOS_DEVICE_THROUGHPUT, 0); #endif /* init dbg */ exynos_ufs_init_mem_log(pdev); ret = exynos_ufs_init_dbg(&ufs->handle); if (ret) return ret; /* store ufs host symbols to analyse later */ ufs->id = ufs_host_index++; ufs_host_backup[ufs->id] = ufs; /* init sysfs */ exynos_ufs_sysfs_init(ufs); /* init specific states */ ufs->h_state = H_DISABLED; ufs->c_state = C_OFF; /* async resume */ INIT_WORK(&ufs->resume_work, __ufs_resume_async); /* register vendor hooks: compl_commmand, etc */ exynos_ufs_register_vendor_hooks(); #if IS_ENABLED(CONFIG_SEC_UFS_FEATURE) ufs_sec_init_logging(dev); #endif /* go to core driver through the glue driver */ ret = ufshcd_pltfrm_init(pdev, &exynos_ufs_ops); out: return ret; } static int exynos_ufs_remove(struct platform_device *pdev) { struct exynos_ufs *ufs = dev_get_platdata(&pdev->dev); struct ufs_hba *hba = platform_get_drvdata(pdev); ufs_host_index--; exynos_ufs_sysfs_exit(ufs); #if IS_ENABLED(CONFIG_SEC_UFS_FEATURE) ufs_remove_sec_features(hba); #endif disable_irq(hba->irq); ufshcd_remove(hba); #if IS_ENABLED(CONFIG_EXYNOS_PM_QOS) || IS_ENABLED(CONFIG_EXYNOS_PM_QOS_MODULE) exynos_pm_qos_remove_request(&ufs->pm_qos_int); #endif /* performance */ if (ufs->perf) ufs_perf_exit(ufs->perf); exynos_ufs_ctrl_phy_pwr(ufs, false); return 0; } #ifdef CONFIG_PM_SLEEP static int exynos_ufs_suspend(struct device *dev) { struct ufs_hba *hba = dev_get_drvdata(dev); struct exynos_ufs *ufs = to_exynos_ufs(hba); int ret = 0; ufs->deep_suspended = false; /* mainly for early wake-up cases */ if (work_busy(&ufs->resume_work) && work_pending(&ufs->resume_work)) flush_work(&ufs->resume_work); ret = ufshcd_system_suspend(hba); if (!ret) hba->ufshcd_state = UFSHCD_STATE_RESET; return ret; } static int exynos_ufs_suspend_noirq(struct device *dev) { struct ufs_hba *hba = dev_get_drvdata(dev); struct exynos_ufs *ufs = to_exynos_ufs(hba); ufs->deep_suspended = true; return 0; } static int exynos_ufs_resume(struct device *dev) { struct ufs_hba *hba = dev_get_drvdata(dev); struct exynos_ufs *ufs = to_exynos_ufs(hba); schedule_work(&ufs->resume_work); return 0; } #else #define exynos_ufs_suspend NULL #define exynos_ufs_resume NULL #define exynos_ufs_suspend_noirq NULL #endif /* CONFIG_PM_SLEEP */ static void exynos_ufs_shutdown(struct platform_device *pdev) { struct exynos_ufs *ufs = dev_get_platdata(&pdev->dev); struct ufs_hba *hba; hba = (struct ufs_hba *)platform_get_drvdata(pdev); if (ufs->perf) ufs_perf_exit(ufs->perf); ufshcd_shutdown(hba); hba->ufshcd_state = UFSHCD_STATE_ERROR; } static const struct dev_pm_ops exynos_ufs_dev_pm_ops = { .suspend = exynos_ufs_suspend, .resume = exynos_ufs_resume, .suspend_noirq = exynos_ufs_suspend_noirq, }; static const struct of_device_id exynos_ufs_match[] = { { .compatible = "samsung,exynos-ufs", }, {}, }; MODULE_DEVICE_TABLE(of, exynos_ufs_match); static struct platform_driver exynos_ufs_driver = { .driver = { .name = "exynos-ufs", .owner = THIS_MODULE, .pm = &exynos_ufs_dev_pm_ops, .of_match_table = exynos_ufs_match, .suppress_bind_attrs = true, }, .probe = exynos_ufs_probe, .remove = exynos_ufs_remove, .shutdown = exynos_ufs_shutdown, }; module_platform_driver(exynos_ufs_driver); MODULE_DESCRIPTION("Exynos Specific UFSHCI driver"); MODULE_AUTHOR("Seungwon Jeon "); MODULE_AUTHOR("Kiwoong Kim "); MODULE_LICENSE("GPL");