// SPDX-License-Identifier: GPL-2.0 /* * HALO (History AnaLysis Oriented) idle governor * * Copyright (C) 2022 - 2023 Samsung Corporation * Author: Youngtae Lee */ #include #include #include #include #include #include #include #include "../sched.h" #include "ems.h" #include #include #include /* previous wake-up reason */ enum waker { WAKEUP_BY_IPI, WAKEUP_BY_DET, /* determined event like periodic IRQ, timer, tick */ WAKEUP_BY_HALO_TIMER_1ST, /* expired time from 1st try */ WAKEUP_BY_HALO_TIMER_2ND, /* expired time from 2nd try */ WAKEUP_BY_SKIP_REFLECT, WAKEUP_BY_POLL, }; /* C-state selector */ enum selector { BY_IPI, /* matches with IPI */ BY_DET, BY_CONSTRAINT, BY_ECS, BY_SKIP_REFLECT, }; /* C-state selector */ enum timer_type { TIMER_1ST, TIMER_2ND, TIMER_NUM, }; /* time and slen table */ #define HIST_NUM 7 struct time_table { s64 time; s64 slen; }; struct time_hist { s32 cur_idx; struct time_table tbl[HIST_NUM]; }; /* hit/miss history */ #define PULSE 1024 #define DECAY_SHIFT 2 #define IGNORE_NUM (1 << 7) struct hit_state { u64 hit; /* hit ratio */ u64 miss; /* miss ratio */ }; /* c-state data */ struct cstate_data { u64 entry_latency_ns; u64 exit_latency_ns; u64 target_residency_ns; }; /* IPI's history */ struct halo_ipi { s64 ipi_slen_ns; /* predicted sleep length to next ipi */ struct time_hist time_hist; }; /* Tick info */ struct halo_tick { int triggered; /* tick stopped */ s64 last_tick_ns; /* last tick time */ s64 tick_slen_ns; /* sleep length to next tick */ }; /* halo htimer */ struct halo_timer { struct hrtimer timer; u32 triggered; /* timer expired status */ int type; /* timer type */ struct hit_state hstate; }; /* halo sched-info */ struct halo_sched { u32 avg_nr_run; /* number of running cpus */ s32 heaviness; /* system heaviness value from sched info */ }; /* periodic irq requester */ #define IRQ_NAME_LEN 20 struct halo_irq { struct list_head list; int num; /* irq num */ char name[IRQ_NAME_LEN];/* irq name */ struct cpumask cpus; /* irq affinity mask */ u64 cnt; /* irq accumulate count */ ktime_t last_update_ns; /* last irq triggered time */ ktime_t period_ns; /* irq period */ void (*fn)(u64 *cnt, ktime_t *time); /* irq validation check function */ int enabled; /* irq validate status */ bool validated; /* irq period is valid */ }; /* periodic irq list */ struct halo_irqs { struct list_head list; raw_spinlock_t lock; s64 irq_slen_ns; }; /* halo tuanble and sysfs */ #define RATIO_UNIT 100 struct halo_tune { struct cpumask cpus; int expired_ratio; /* def-timer expired time ratio */ struct kobject kobj; }; /* * struct halo_cpu - CPU data used by the HAO Governor */ struct halo_cpu { u32 last_idx; /* last selected c-state */ enum waker last_waker; /* last wakeup source */ s64 last_slen_ns; /* last sleep length */ s64 timer_slen_ns; /* sleep length to next timer event without tick */ s64 req_latency_ns; /* requested latency */ s64 det_slen_ns; /* determined sleep length to next event like(timer, tick, periodic irqs) */ struct halo_ipi hipi; /* ipi history */ struct halo_tick htick; /* tick information */ struct halo_sched hsched; /* sched information */ struct halo_irqs hirqs; /* periodic irqs list */ struct halo_timer htimer; /* timer struct to correct c-state */ struct halo_tune *htune; /* for tunable */ struct cstate_data csd[CPUIDLE_STATE_MAX]; /* dynamic cstate data */ struct cpuidle_device *dev; }; struct kobject *halo_kobj; static DEFINE_PER_CPU(struct halo_cpu, halo_cpus); /******************************************************************************** * HALO HELPER FUNC * *******************************************************************************/ static inline struct halo_cpu *get_hcpu(int cpu) { return per_cpu_ptr(&halo_cpus, cpu); } /* return next time hist index */ static int halo_get_nxt_time_hist_idx(int idx) { if (++idx == HIST_NUM) return 0; return idx; } /* update time hist table */ static void halo_update_time_hist(struct time_hist *time_hist, s64 now, s64 slen) { int idx = halo_get_nxt_time_hist_idx(time_hist->cur_idx); time_hist->tbl[idx].time = now; time_hist->tbl[idx].slen = slen; time_hist->cur_idx = idx; } /* * compute statics from time hist table * return number of valid count */ static void halo_get_time_hist_statics(struct time_hist *time_hist, s64 now, s64 *min, s64 *max, s64 *avg) { s64 slen, tmin = LLONG_MAX, tmax = 0, sum = 0; s32 idx = 0, min_idx = 0, max_idx = 0; /* * It is noise filter. * When computing avg sleep length from IPI's intervals, * remove minimum/maximum values */ for (idx = 0; idx < HIST_NUM; idx++) { slen = time_hist->tbl[idx].slen; if (slen < tmin) { tmin = slen; min_idx = idx; } if (slen > tmax) { tmax = slen; max_idx = idx; } } /* find second min/max */ for (idx = 0; idx < HIST_NUM; idx++) { if (max_idx == idx || min_idx == idx) continue; slen = time_hist->tbl[idx].slen; sum += slen; if (slen < tmin) tmin = slen; if (slen > tmax) tmax = slen; } *min = tmin; *max = tmax; *avg = sum / (HIST_NUM - 2); } /* * calculate the time remaining between now and next periodic event */ static s64 calc_periodic_slen(s64 last_update_ns, s64 period, s64 now) { s64 offset_cnt, next_wakeup_ns; if (unlikely((now < last_update_ns) || !period)) return LLONG_MAX; offset_cnt = ((now - last_update_ns) / period) + 1; next_wakeup_ns = last_update_ns + offset_cnt * period; return next_wakeup_ns - now; } /******************************************************************************** * HALO TIMER FUNC * *******************************************************************************/ /* * halo_timer_fn - timer function */ static enum hrtimer_restart halo_timer_fn(struct hrtimer *h) { u32 cpu = raw_smp_processor_id(); struct halo_cpu *hcpu = get_hcpu(cpu); struct halo_timer *htimer = &hcpu->htimer; htimer->triggered = true; trace_halo_timer_fn(cpu, htimer->type); return HRTIMER_NORESTART; } /* * halo_timer_start * start halo timer when need a chance to change c-state in the future. * we do better change deeper c-state if cpu is in the shallower c-state * with wrong prediction. */ static void halo_timer_start(struct halo_cpu *hcpu, ktime_t expired_ns) { struct halo_timer *htimer = &hcpu->htimer; /* In this case, we don't need a timer */ if (expired_ns <= 0) { htimer->type = TIMER_NUM; return; } /* update timer type */ if (hcpu->last_waker == WAKEUP_BY_HALO_TIMER_1ST) htimer->type = TIMER_2ND; else htimer->type = TIMER_1ST; hrtimer_start(&htimer->timer, expired_ns, HRTIMER_MODE_REL_PINNED); } /* * halo_timer_cancel - if timer is until active, cancel timer after wake up */ static void halo_timer_cancel(struct halo_timer *htimer) { struct hrtimer *timer = &htimer->timer; ktime_t time_rem = hrtimer_get_remaining(timer); if (time_rem <= 0) return; hrtimer_try_to_cancel(timer); } /* * halo_need_timer * check whether time delta between predicted wake-up time and next event time * is longer than deeper c-state target residency. * return: When NEED a timer, return expired time if NOT return 0 */ static s64 halo_need_timer(struct cpuidle_driver *drv, struct cpuidle_device *dev, struct halo_cpu *hcpu, s64 det_slen_ns, enum selector selector, u32 candi_idx) { s64 delta_time, expired_time; u32 i, idx = candi_idx; /* we dont't need a timer when selected c-state is the deepest */ if (idx == drv->state_count - 1) return 0; /* timer must work to change c-state when prediction will be failed */ if (selector == BY_CONSTRAINT) return 0; /* * check a delta time when preidcton will be failed and then timer will be expired, * whether delta time is enough to enter deeper c-state or not. * To check it, need to compute delta time and timer expired time. * expired_time : htimer expired time to change c-state * delta_time : delta time between timer wakeup and next wakeup event when add a timer */ expired_time = 4 * NSEC_PER_MSEC; delta_time = ktime_sub(det_slen_ns, expired_time); if (delta_time <= 0 ) return 0; /* * When delta_time > deeper_cstate_target_residency, * we have enough time to enter deeper c-state, so need a timer to change c-state */ for (i = candi_idx + 1; i < drv->state_count; i++) { u64 target_residency_ns = hcpu->csd[i].target_residency_ns; if (dev->states_usage[i].disable) continue; if (target_residency_ns < delta_time) idx = i; } if (idx == candi_idx) return 0; return expired_time; } /* * halo_timer_update_hstate * Update timer hit/miss statics to decide whether use 2nd time or not. * When 1st timer is expired and 2nd time hit ratio higher than miss ratio, use 2nd timer. */ static void halo_timer_update_hstate(int cpu, struct halo_timer *htimer) { struct hit_state *hstate = &htimer->hstate; /* no timer */ if (htimer->type >= TIMER_NUM) return; hstate->hit -= (hstate->hit >> DECAY_SHIFT); if (hstate->hit <= IGNORE_NUM) hstate->hit = 0; hstate->miss -= (hstate->miss >> DECAY_SHIFT); if (hstate->miss <= IGNORE_NUM) hstate->miss = 0; if (htimer->triggered) hstate->miss += PULSE; else hstate->hit += PULSE; trace_halo_hstate(cpu, 0, htimer->hstate.hit, htimer->hstate.miss); } /******************************************************************************** * HALO Scheduler Information * *******************************************************************************/ /* * halo_update_sched_info - update sched information for more accurate prediction * When system more heavy than before, increase heaviness value */ #define VALID_NR_RUN_DIFF (NR_RUN_UNIT >> 2) static void halo_update_sched_info(struct cpuidle_device *dev, struct halo_cpu *hcpu) { struct halo_sched *hsched = &hcpu->hsched; s32 avg_nr_run = mlt_avg_nr_run(cpu_rq(dev->cpu)); s32 avg_nr_run_diff = avg_nr_run - hsched->avg_nr_run; hsched->avg_nr_run = avg_nr_run; if (abs(avg_nr_run_diff) < VALID_NR_RUN_DIFF) { hcpu->hsched.heaviness = 0; return; } hcpu->hsched.heaviness = min(avg_nr_run_diff, NR_RUN_UNIT); } /******************************************************************************** * HALO IPI INFO * *******************************************************************************/ /* * halo_update_ipi - update IPI's history * Gather residency when task wake-up by IPI except tick, IRQ, timer event */ static void halo_update_ipi_history(int cpu, struct cpuidle_driver *drv, struct halo_cpu *hcpu, s64 now) { struct halo_ipi *hipi = &hcpu->hipi; struct time_hist *time_hist = &hipi->time_hist; s64 min_ns = 0, max_ns = 0, avg_ns = 0; s64 correction_ns = 0; /* * When last wake-up reason is IPI, need to update IPI history, * Others, we don't need to update IPI history. Exclude last sleep time * from IPI's history. */ if (hcpu->last_waker != WAKEUP_BY_IPI) return; /* applying last sleep length */ halo_update_time_hist(time_hist, now, hcpu->last_slen_ns); /* get statics from time hist table with periodic duration, if there are valid data */ halo_get_time_hist_statics(time_hist, now, &min_ns, &max_ns, &avg_ns); /* * Computing predicted_sleep_length * correction_ns is correction weight for predicted_sleep_length according to sched_info * correction_ns decrease as the sched-load(nr_running) increases, * on the other way it increase as the sched-load decreases. */ if (hcpu->hsched.heaviness > 0) correction_ns = (min_ns - avg_ns) * hcpu->hsched.heaviness / NR_RUN_UNIT; else correction_ns = (avg_ns - max_ns) * hcpu->hsched.heaviness / NR_RUN_UNIT; hipi->ipi_slen_ns = max((s64)0, (avg_ns + correction_ns)); trace_halo_ipi(cpu, hcpu->hsched.heaviness, min_ns, max_ns, hcpu->last_slen_ns, avg_ns, correction_ns, hipi->ipi_slen_ns); } /******************************************************************************** * HALO TICK Information * *******************************************************************************/ void halo_tick(struct rq *rq) { struct halo_cpu *hcpu = get_hcpu(cpu_of(rq)); struct halo_tick *htick = &hcpu->htick; if (unlikely(!hcpu->dev)) return; /* * When last_state_idx is -1 and called this function, it means cpu is waking by tick * last_state_idx updated like below. * 1. exit idle * 2. if wake-up by tick, progress related work (execute this function) * 3. halo_reflect (update lats_state_idx to last_state 0 or 1) * 4. execute some jobs and done * 5. halo_select (select cstate and update last_state_idx to -1) * 6. enter idle */ if (hcpu->dev->last_state_idx < 0) { htick->triggered = true; htick->last_tick_ns = local_clock(); } trace_halo_tick(cpu_of(rq), htick->triggered); } /** * halo_update_tick - Update tick info(next tick time and stopped status) after wakeup */ static void halo_update_tick(struct halo_cpu *hcpu, s64 now) { struct halo_tick *htick = &hcpu->htick; htick->tick_slen_ns = calc_periodic_slen(htick->last_tick_ns, TICK_NSEC, now); } /******************************************************************************** * HALO Periodic IRQ * *******************************************************************************/ /* * halo_align_periodic_irq * align irq's last_update_ns and accomulate cnt * and compute whether period is valide from count and time delta * if period is in error range, return TRUE, if NOT return FALSE */ #define ERROR_RATIO 10 /* 10% */ static bool check_irq_validation(struct halo_cpu *hcpu, struct halo_irq *hirq) { u64 delta_cnt, prev_cnt = hirq->cnt; ktime_t delta_ns, prev_ns = hirq->last_update_ns; ktime_t error_range_ns, error_ns; ktime_t real_period_ns; unsigned long flags; raw_spin_lock_irqsave(&hcpu->hirqs.lock, flags); /* get recent cnt, last_update_ns */ hirq->fn(&hirq->cnt, &hirq->last_update_ns); /* check whether period is validated or not */ delta_cnt = hirq->cnt - prev_cnt; delta_ns = hirq->last_update_ns - prev_ns; real_period_ns = delta_ns / delta_cnt; error_range_ns = hirq->period_ns * ERROR_RATIO / 100; error_ns = abs(real_period_ns - hirq->period_ns); if (error_ns > error_range_ns) { hirq->validated = false; raw_spin_unlock_irqrestore(&hcpu->hirqs.lock, flags); return false; } hirq->validated = true; raw_spin_unlock_irqrestore(&hcpu->hirqs.lock, flags); return true; } static void halo_update_irq_slen(struct halo_cpu *hcpu, s64 now) { struct halo_irq *hirq; s64 min_slen = LLONG_MAX; list_for_each_entry(hirq, &hcpu->hirqs.list, list) { s64 slen; if (!hirq->enabled || hirq->period_ns == 0) continue; /* If last_update_ns is before 1 sec, try to align time */ if ((now - hirq->last_update_ns) > NSEC_PER_SEC) { /* if irq is not validated, don't use this irq */ if (!check_irq_validation(hcpu, hirq)) continue; } slen = calc_periodic_slen(hirq->last_update_ns, hirq->period_ns, now); if (slen < min_slen) min_slen = slen; } hcpu->hirqs.irq_slen_ns = min_slen;; } /* * check_irq_registable - return whether this irq is registable or not */ static int check_irq_registable(struct halo_cpu *hcpu, int cpu, int irq_num, struct cpumask *cpus, void (*fn)(u64 *cnt, ktime_t *time)) { struct halo_irq *hirq; if (cpumask_empty(cpus) || !fn) return 0; list_for_each_entry(hirq, &hcpu->hirqs.list, list) if (hirq->num == irq_num) { pr_info("%s: cpu%d already registered duplicated irq(%d)\n", cpu, irq_num); return -1; } return 0; } static struct halo_irq *find_hirq(struct halo_cpu *hcpu, int irq_num) { struct halo_irq *hirq; list_for_each_entry(hirq, &hcpu->hirqs.list, list) if (hirq->num == irq_num) return hirq; return NULL; } /* * halo_register_periodic_irq - register irq to periodic irq list */ void halo_register_periodic_irq(int irq_num, const char *name, struct cpumask *cpus, void (*fn)(u64 *cnt, ktime_t *time)) { unsigned long flags; int cpu; for_each_cpu(cpu, cpus){ struct halo_cpu *hcpu = get_hcpu(cpu); struct halo_irq *hirq; raw_spin_lock_irqsave(&hcpu->hirqs.lock, flags); /* check irq registable */ if (check_irq_registable(hcpu, cpu, irq_num, cpus, fn)) { raw_spin_unlock_irqrestore(&hcpu->hirqs.lock, flags); continue; } /* allocate irq information */ hirq = kzalloc(sizeof(struct halo_irq), GFP_KERNEL); if (!hirq) { pr_err("%s: failed to allocate cpu%d irq(%d)\n", cpu, irq_num); raw_spin_unlock_irqrestore(&hcpu->hirqs.lock, flags); continue; } hirq->num = irq_num; strcpy(hirq->name, name); cpumask_copy(&hirq->cpus, cpus); hirq->fn = fn; hirq->enabled = false; hirq->validated = false; /* reigster irq to list */ list_add(&hirq->list, &hcpu->hirqs.list); raw_spin_unlock_irqrestore(&hcpu->hirqs.lock, flags); } } EXPORT_SYMBOL_GPL(halo_register_periodic_irq); /* * halo_update_periodic_irq - Requester notifies irq operation information */ void halo_update_periodic_irq(int irq_num, u64 cnt, ktime_t last_update_ns, ktime_t period_ns) { unsigned long flags; int cpu; for_each_possible_cpu(cpu) { struct halo_cpu *hcpu = get_hcpu(cpu); struct halo_irq *hirq; raw_spin_lock_irqsave(&hcpu->hirqs.lock, flags); hirq = find_hirq(hcpu, irq_num); if (!hirq) { raw_spin_unlock_irqrestore(&hcpu->hirqs.lock, flags); continue; } /* update irq recent information */ hirq->cnt = cnt; hirq->last_update_ns = last_update_ns; hirq->period_ns = period_ns; /* enable periodic irq */ if (period_ns) { hirq->enabled = true; hirq->validated = true; } else { hirq->enabled = false; hirq->validated = false; } raw_spin_unlock_irqrestore(&hcpu->hirqs.lock, flags); } } EXPORT_SYMBOL_GPL(halo_update_periodic_irq); /* * halo_unregister_periodic_irq - unregister irq to periodic irq list */ void halo_unregister_periodic_irq(int irq_num) { unsigned long flags; int cpu; for_each_possible_cpu(cpu) { struct halo_cpu *hcpu = get_hcpu(cpu); struct halo_irq *hirq; raw_spin_lock_irqsave(&hcpu->hirqs.lock, flags); hirq = find_hirq(hcpu, irq_num); if (!hirq) { raw_spin_unlock_irqrestore(&hcpu->hirqs.lock, flags); continue; } list_del(&hirq->list); kfree(hirq); raw_spin_unlock_irqrestore(&hcpu->hirqs.lock, flags); } } EXPORT_SYMBOL_GPL(halo_unregister_periodic_irq); /******************************************************************************** * HALO MAIN * *******************************************************************************/ /* halo_get_cstate_data - gathering residency, entry/exit latency */ static void halo_get_cstate_data(struct cpuidle_driver *drv, struct cpuidle_device *dev, struct halo_cpu *hcpu) { int i; for (i = 0; i < drv->state_count; i++) { struct cpuidle_state *s = &drv->states[i]; u64 val; val = fv_get_residency(dev->cpu, i); hcpu->csd[i].target_residency_ns = val ? val : s->target_residency_ns; val = fv_get_exit_latency(dev->cpu, i); hcpu->csd[i].exit_latency_ns = val ? val : s->exit_latency_ns; } } /* * halo_update - Update CPU metrics after wakeup. * @drv: cpuidle driver containing state data. * @dev: Target CPU. */ static void halo_update(struct cpuidle_driver *drv, struct cpuidle_device *dev, s64 now) { struct halo_cpu *hcpu = get_hcpu(dev->cpu); s64 tick_slen_ns; /* get latency */ hcpu->req_latency_ns = cpuidle_governor_latency_req(dev->cpu); /* sleep length to next timer event */ hcpu->timer_slen_ns = tick_nohz_get_sleep_length(&tick_slen_ns); /* Update Sched-Tick information */ halo_update_tick(hcpu, now); /* update periodic irq sleep length */ halo_update_irq_slen(hcpu, now); /* compute det_slen_ns */ hcpu->det_slen_ns = min(hcpu->timer_slen_ns, hcpu->hirqs.irq_slen_ns); /* Update sched information */ halo_update_sched_info(dev, hcpu); /* Update IPI's history */ halo_update_ipi_history(dev->cpu, drv, hcpu, now); } /* halo_check_valid_state - return the available state number */ static int halo_get_valid_state(struct cpuidle_driver *drv, struct cpuidle_device *dev, int *valid_idx) { int idx = 0, valid_cnt = 0; /* Check if there is any choice in the first place. */ if (unlikely(drv->state_count < 2)) { *valid_idx = 0; valid_cnt = 1; return valid_cnt; } /* check enabled state */ for (idx = 0; idx < drv->state_count; idx++) { if (!dev->states_usage[idx].disable) { *valid_idx = idx; valid_cnt++; } } return valid_cnt; } /* * halo_compute_pred_slen - computing predicted sleep length */ static inline s64 halo_compute_pred_slen(struct halo_cpu *hcpu, struct cpuidle_driver *drv) { if (hcpu->last_waker == WAKEUP_BY_IPI || hcpu->last_waker == WAKEUP_BY_DET || hcpu->last_waker == WAKEUP_BY_SKIP_REFLECT) return hcpu->hipi.ipi_slen_ns; else if (hcpu->last_waker == WAKEUP_BY_HALO_TIMER_1ST) { /* * When previous IPI prediction failed and second time timer hit ratio is high, * give a change to use predicted_sleep_length one more */ struct hit_state *hstate = &hcpu->htimer.hstate; struct halo_sched *hsched = &hcpu->hsched; if (hstate->hit > hstate->miss && hsched->avg_nr_run > VALID_NR_RUN_DIFF) return hcpu->hipi.ipi_slen_ns; } return LLONG_MAX; } /* * apply tunable value * * EXPIRED_RATIO tunes def-timere expired time length * default value is 100%. it means expired time is same with deeper c-state target rsidency. * The higher the ratio, higher the lower c-state entry rate. * The lower ratio, lower the high c-state entry rate. */ static s64 apply_tunable(struct halo_cpu *hcpu, s64 expird_time) { struct halo_tune *htune = hcpu->htune; if (unlikely(!htune)) return expird_time; return expird_time * hcpu->htune->expired_ratio / RATIO_UNIT; } /* * halo_select - Selects the next idle state to enter. * @drv: cpuidle driver containing state data. * @dev: Target CPU. * @stop_tick: Indication on whether or not to stop the scheduler tick. */ static int halo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, bool *stop_tick) { struct halo_cpu *hcpu = get_hcpu(dev->cpu); s32 i, candi_idx = 0, pred_idx = 0, const_idx = 0; s32 deepest_idx = drv->state_count - 1; s64 pred_slen_ns = 0, expired_ns = 0; s64 now = local_clock(); enum selector selector = 0; /* if valid count lower than two, we don't need to compute next state */ if (halo_get_valid_state(drv, dev, &candi_idx) < 2) goto end; /* get cstate data */ halo_get_cstate_data(drv, dev, hcpu); /* update last history */ if (dev->last_state_idx >= 0) { halo_update(drv, dev, now); } else { hcpu->last_waker = WAKEUP_BY_SKIP_REFLECT; selector = BY_SKIP_REFLECT; halo_timer_cancel(&hcpu->htimer); } dev->last_state_idx = -1; if (!cpumask_test_cpu(dev->cpu, ecs_available_cpus())) { selector = BY_ECS; candi_idx = deepest_idx; goto end; } /* traversing c-state */ pred_slen_ns = halo_compute_pred_slen(hcpu, drv); deepest_idx = 0; for (i = 0; i < drv->state_count; i++) { u64 target_residency_ns = hcpu->csd[i].target_residency_ns; u64 exit_latency_ns = hcpu->csd[i].exit_latency_ns; if (dev->states_usage[i].disable) continue; /* for performance, exit_latency should be short than req_latency */ if (exit_latency_ns < hcpu->req_latency_ns) const_idx = i; /* for power, sleep length should be longer than target residency */ if (target_residency_ns < pred_slen_ns) pred_idx = i; /* deepest c-state with next determined sleep length */ if (target_residency_ns < hcpu->det_slen_ns) deepest_idx = i; } /* find c-state selector */ if (pred_slen_ns < hcpu->det_slen_ns) { selector = BY_IPI; candi_idx = pred_idx; } else { selector = BY_DET; candi_idx = deepest_idx; } /* limit c-state depth to guarantee latency */ if (candi_idx > const_idx) { struct rq *rq = cpu_rq(dev->cpu); selector = BY_CONSTRAINT; candi_idx = const_idx; if (rq->avg_idle < hcpu->csd[const_idx].target_residency_ns) *stop_tick = false; } /* When c-state is not constrainted, use deferred timer to change deeper c-state later */ expired_ns = halo_need_timer(drv, dev, hcpu, hcpu->det_slen_ns, selector, candi_idx); expired_ns = apply_tunable(hcpu, expired_ns); halo_timer_start(hcpu, expired_ns); end: trace_halo_select(hcpu->last_waker, selector, hcpu->last_slen_ns, hcpu->req_latency_ns, hcpu->htick.tick_slen_ns, hcpu->timer_slen_ns, hcpu->hirqs.irq_slen_ns, hcpu->det_slen_ns, pred_slen_ns, expired_ns, candi_idx); hcpu->last_slen_ns = now; hcpu->last_idx = candi_idx; return hcpu->last_idx;; } /* * halo_reflect - Note that governor data for the CPU need to be updated. * @dev: Target CPU. * @state: Entered state. */ static void halo_reflect(struct cpuidle_device *dev, int state) { struct halo_cpu *hcpu = get_hcpu(dev->cpu); struct halo_tick *htick = &hcpu->htick; struct halo_timer *htimer = &hcpu->htimer; enum waker waker; s64 slen_ns; /* update last c-stae */ dev->last_state_idx = state; /* compute last sleep length between idle_selection and wake_up */ slen_ns = local_clock() - hcpu->last_slen_ns; /* compute htimer hit state */ halo_timer_update_hstate(dev->cpu, htimer); if (htimer->triggered) { /* wake up by halo timer */ if (htimer->type == TIMER_1ST) waker = WAKEUP_BY_HALO_TIMER_1ST; else waker = WAKEUP_BY_HALO_TIMER_2ND; htimer->triggered = false; } else if (dev->poll_time_limit) { /* wake up by poll */ waker = WAKEUP_BY_POLL; dev->poll_time_limit = false; } else if (htick->triggered || slen_ns >= hcpu->det_slen_ns) { /* wake up by next determined event (timer, periodic irq, tick) */ htick->triggered = false; waker = WAKEUP_BY_DET; } else { waker = WAKEUP_BY_IPI; } trace_halo_reflect(dev->cpu, waker, slen_ns); hcpu->last_slen_ns = slen_ns; hcpu->last_waker = waker; halo_timer_cancel(htimer); } /* halo_mode_update_callback - change tunable valuee according to ems mode */ static int halo_mode_update_callback(struct notifier_block *nb, unsigned long val, void *v) { struct emstune_set *cur_set = (struct emstune_set *)v; struct halo_tune *htune; int cpu; for_each_possible_cpu(cpu) { struct halo_cpu *hcpu = get_hcpu(cpu); htune = hcpu->htune; if (cpu != cpumask_first(&htune->cpus)) continue; htune->expired_ratio = cur_set->cpuidle_gov.expired_ratio[cpu]; } return NOTIFY_OK; } static struct notifier_block halo_mode_update_notifier = { .notifier_call = halo_mode_update_callback, }; /******************************************************************************** * HALO SYSFS * *******************************************************************************/ struct halo_attr { struct attribute attr; ssize_t (*show)(struct kobject *, char *); ssize_t (*store)(struct kobject *, const char *, size_t count); }; #define halo_attr_r(name) \ static struct halo_attr name##_attr = \ __ATTR(name, 0444, show_##name, NULL) #define halo_attr_rw(name) \ static struct halo_attr name##_attr = \ __ATTR(name, 0644, show_##name, store_##name) #define halo_show(name) \ static ssize_t show_##name(struct kobject *k, char *buf) \ { \ struct halo_tune *htune = \ container_of(k, struct halo_tune, kobj); \ \ return sprintf(buf, "%d\n", htune->name); \ } \ #define halo_store(name) \ static ssize_t store_##name(struct kobject *k, const char *buf, size_t count) \ { \ struct halo_tune *htune = \ container_of(k, struct halo_tune, kobj); \ int data; \ \ if (!sscanf(buf, "%d", &data)) \ return -EINVAL; \ \ htune->name = data; \ return count; \ } halo_show(expired_ratio); halo_store(expired_ratio); halo_attr_rw(expired_ratio); static ssize_t show_irqs_list(struct kobject *k, char *buf) { struct halo_tune *htune = container_of(k, struct halo_tune, kobj); unsigned long flags; int cpu, ret = 0; for_each_cpu(cpu, &htune->cpus) { struct halo_cpu *hcpu = get_hcpu(cpu); struct halo_irq *hirq; ret += sprintf(buf + ret, "cpu%d: ", cpu); raw_spin_lock_irqsave(&hcpu->hirqs.lock, flags); list_for_each_entry(hirq, &hcpu->hirqs.list, list) ret += sprintf(buf + ret, "(%d, %s period=%llu en=%d v=%d) ", hirq->num, hirq->name, hirq->period_ns, hirq->enabled, hirq->validated); ret += sprintf(buf + ret, "\n"); raw_spin_unlock_irqrestore(&hcpu->hirqs.lock, flags); } return ret; } halo_attr_r(irqs_list); static ssize_t show(struct kobject *kobj, struct attribute *at, char *buf) { struct halo_attr *attr = container_of(at, struct halo_attr, attr); return attr->show(kobj, buf); } static ssize_t store(struct kobject *kobj, struct attribute *at, const char *buf, size_t count) { struct halo_attr *attr = container_of(at, struct halo_attr, attr); return attr->store(kobj, buf, count); } static const struct sysfs_ops halo_sysfs_ops = { .show = show, .store = store, }; static struct attribute *halo_attrs[] = { &expired_ratio_attr.attr, &irqs_list_attr.attr, NULL }; static struct kobj_type ktype_halo = { .sysfs_ops = &halo_sysfs_ops, .default_attrs = halo_attrs, }; /** * halo_enable_device - Initialize the governor's data for the target CPU. * @drv: cpuidle driver (not used). * @dev: Target CPU. */ static int halo_enable_device(struct cpuidle_driver *drv, struct cpuidle_device *dev) { struct halo_cpu *hcpu = get_hcpu(dev->cpu); hcpu->dev = dev; return 0; } static struct cpuidle_governor halo_governor = { .name = "halo", .rating = 19, .enable = halo_enable_device, .select = halo_select, .reflect = halo_reflect, }; static void halo_tunable_init(struct kobject *ems_kobj) { struct device_node *dn, *child; int cpu; halo_kobj = kobject_create_and_add("halo", ems_kobj); if (!halo_kobj) return; dn = of_find_node_by_path("/ems/halo"); if (!dn) return; for_each_child_of_node(dn, child) { struct halo_tune *htune; const char *buf; htune = kzalloc(sizeof(struct halo_tune), GFP_KERNEL); if (!htune) { pr_err("%s: failed to alloc halo_tune\n", __func__); return; } if (of_property_read_string(child, "cpus", &buf)) { pr_err("%s: cpus property is omitted\n", __func__); return; } else { cpulist_parse(buf, &htune->cpus); } if (of_property_read_u32(child, "expired-ratio", &htune->expired_ratio)) htune->expired_ratio = RATIO_UNIT; if (kobject_init_and_add(&htune->kobj, &ktype_halo, halo_kobj, "coregroup%d", cpumask_first(&htune->cpus))) return; for_each_cpu(cpu, &htune->cpus) { struct halo_cpu *hcpu = get_hcpu(cpu); hcpu->htune = htune; } } emstune_register_notifier(&halo_mode_update_notifier); } /* * halo_cpu_init - initialize halo cpu data */ static void halo_cpu_init(void) { s32 cpu; for_each_possible_cpu(cpu) { struct halo_cpu *hcpu = get_hcpu(cpu); struct hrtimer *timer; /* init deferred time */ timer = &hcpu->htimer.timer; hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); timer->function = halo_timer_fn; /* init periodic irq list */ raw_spin_lock_init(&hcpu->hirqs.lock); INIT_LIST_HEAD(&hcpu->hirqs.list); } } int halo_governor_init(struct kobject *ems_kobj) { halo_tunable_init(ems_kobj); halo_cpu_init(); return cpuidle_register_governor(&halo_governor); }