From f0f9800398c72085b36a8bb366ecfc62242b7760 Mon Sep 17 00:00:00 2001 From: zhujingpeng Date: Thu, 18 Apr 2024 00:57:11 +0800 Subject: [PATCH] ANDROID: vendor_hooks: add hooks in rwsem these hooks are required by the following features: 1.For rwsem readers, currently only the latest reader will be recorded in sem->owner. We add hooks to record all readers which have acquired the lock, once there are UX threads blocked in the rwsem waiting list, these read_owners will be given high priority in scheduling. 2.For rwsem writer, when a writer acquires the lock, we check whether there are UX threads blocked in the rwsem wait list. If so, we give this writer a high priority in scheduling so that it can release the lock as soon as possible. Both of these features can optimize the priority inversion problem caused by rwsem and improve system responsiveness and performance. There is already a hook android_vh_rwsem_set_owner in rwsem_set_owner() to record writer owned, so the hook android_vh_record_rwsem_writer_owned in cherry pick is removed. Bug: 335408185 Change-Id: I82a6fbb6acd2ce05d049e686b61e34e4d3b39a5e Signed-off-by: zhujingpeng [jstultz: Rebased and resolved minor conflict] Signed-off-by: John Stultz (cherry picked from commit 188c41744ddcccef6daf6dcd4d6a444dabdc2f94) (cherry picked from commit 869fc79d3abf1a1bcd97925fa3e0ee7188bb737c) --- drivers/android/vendor_hooks.c | 44 ++++++++++++++++++++++++++++++++++ include/trace/hooks/rwsem.h | 10 ++++++++ kernel/locking/rwsem.c | 4 ++++ 3 files changed, 58 insertions(+) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 58a77d5e6..9ee6f9841 100755 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -501,6 +501,50 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_wakeup_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_skip_swapcache); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mz_exit); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_proc_transaction_finish); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alloc_oem_binder_struct); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_transaction_received); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_free_oem_binder_struct); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_special_task); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_free_buf); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_buffer_release); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_perf_huristic_ctrl); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_send_command_post_change); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_abort_success_ctrl); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_compl_rsp_check_done); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_err_handler); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_err_check_ctrl); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_err_print_ctrl); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_vmscan_kswapd_done); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_do_swap_page_spf); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_exit_check); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_bio_free); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_internal_blk_mq_alloc_request); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_internal_blk_mq_free_request); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_complete_request); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_add_to_requeue_list); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_blk_mq_delay_run_hw_queue); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_run_hw_queue); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_insert_request); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_blk_mq_alloc_rq_map); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_blk_mq_init_allocated_queue); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_exit_queue); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_alloc_tag_set); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_blk_allocated_queue_init); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_blk_flush_plug_list); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_alloc_flush_queue); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_all_tag_iter); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_queue_tag_busy_iter); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_free_tags); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_sched_insert_request); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_shmem_swapin_page); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_do_wp_page); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_do_swap_page); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_do_anonymous_page); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_uprobes_replace_page); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_rwsem_reader_owned); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_clear_rwsem_reader_owned); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_clear_rwsem_writer_owned); /* * For type visibility */ diff --git a/include/trace/hooks/rwsem.h b/include/trace/hooks/rwsem.h index ed9daf91f..72c29b93a 100755 --- a/include/trace/hooks/rwsem.h +++ b/include/trace/hooks/rwsem.h @@ -49,6 +49,16 @@ DECLARE_HOOK(android_vh_rwsem_up_read_end, DECLARE_HOOK(android_vh_rwsem_mark_wake_readers, TP_PROTO(struct rw_semaphore *sem, struct rwsem_waiter *waiter), TP_ARGS(sem, waiter)); +DECLARE_HOOK(android_vh_record_rwsem_reader_owned, + TP_PROTO(struct rw_semaphore *sem, + struct list_head *wlist), + TP_ARGS(sem, wlist)); +DECLARE_HOOK(android_vh_clear_rwsem_reader_owned, + TP_PROTO(struct rw_semaphore *sem), + TP_ARGS(sem)); +DECLARE_HOOK(android_vh_clear_rwsem_writer_owned, + TP_PROTO(struct rw_semaphore *sem), + TP_ARGS(sem)); /* macro versions of hooks are no longer required */ #endif /* _TRACE_HOOK_RWSEM_H */ diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index acd1fc78d..1f69c9371 100755 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -182,6 +182,7 @@ static inline void rwsem_set_owner(struct rw_semaphore *sem) static inline void rwsem_clear_owner(struct rw_semaphore *sem) { atomic_long_set(&sem->owner, 0); + trace_android_vh_clear_rwsem_writer_owned(sem); } /* @@ -245,6 +246,7 @@ static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) { unsigned long val = atomic_long_read(&sem->owner); + trace_android_vh_clear_rwsem_reader_owned(sem); while ((val & ~RWSEM_OWNER_FLAGS_MASK) == (unsigned long)current) { if (atomic_long_try_cmpxchg(&sem->owner, &val, val & RWSEM_OWNER_FLAGS_MASK)) @@ -254,6 +256,7 @@ static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) #else static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) { + trace_android_vh_clear_rwsem_reader_owned(sem); } #endif @@ -526,6 +529,7 @@ static void rwsem_mark_wake(struct rw_semaphore *sem, if (adjustment) atomic_long_add(adjustment, &sem->count); + trace_android_vh_record_rwsem_reader_owned(sem, &wlist); /* 2nd pass */ list_for_each_entry_safe(waiter, tmp, &wlist, list) {