From 931786af089f723615525f8c531373171bd93bb4 Mon Sep 17 00:00:00 2001 From: Steven Noonan Date: Fri, 8 Feb 2019 15:28:40 +0000 Subject: [PATCH 5/6] tsc: allow directly synchronizing TSC if TSC_ADJUST is absent Signed-off-by: Steven Noonan --- .../admin-guide/kernel-parameters.txt | 2 + arch/x86/kernel/tsc.c | 3 ++ arch/x86/kernel/tsc_sync.c | 48 +++++++++++++++---- 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 296c49317f0b..37fc1019e3ae 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -4821,6 +4821,8 @@ in situations with strict latency requirements (where interruptions from clocksource watchdog are not acceptable). + [x86] directsync: attempt to sync the tsc via direct + writes if MSR_IA32_TSC_ADJUST isn't available turbografx.map[2|3]= [HW,JOY] TurboGraFX parallel port interface diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 57d87f79558f..5e5f56e1f9d8 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -45,6 +45,7 @@ static int __read_mostly tsc_unstable; static DEFINE_STATIC_KEY_FALSE(__use_tsc); int tsc_clocksource_reliable; +int tsc_allow_direct_sync; static u32 art_to_tsc_numerator; static u32 art_to_tsc_denominator; @@ -295,6 +296,8 @@ static int __init tsc_setup(char *str) mark_tsc_unstable("boot parameter"); if (!strcmp(str, "nowatchdog")) no_tsc_watchdog = 1; + if (!strcmp(str, "directsync")) + tsc_allow_direct_sync = 1; return 1; } diff --git a/arch/x86/kernel/tsc_sync.c b/arch/x86/kernel/tsc_sync.c index ec534f978867..ff3761303a0e 100644 --- a/arch/x86/kernel/tsc_sync.c +++ b/arch/x86/kernel/tsc_sync.c @@ -31,6 +31,8 @@ struct tsc_adjust { static DEFINE_PER_CPU(struct tsc_adjust, tsc_adjust); +extern int tsc_allow_direct_sync; + /* * TSC's on different sockets may be reset asynchronously. * This may cause the TSC ADJUST value on socket 0 to be NOT 0. @@ -300,6 +302,8 @@ static cycles_t check_tsc_warp(unsigned int timeout) */ static inline unsigned int loop_timeout(int cpu) { + if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) + return 30; return (cpumask_weight(topology_core_cpumask(cpu)) > 1) ? 2 : 20; } @@ -320,13 +324,16 @@ void check_tsc_sync_source(int cpu) /* * Set the maximum number of test runs to - * 1 if the CPU does not provide the TSC_ADJUST MSR - * 3 if the MSR is available, so the target can try to adjust + * 5 if we can write TSC_ADJUST to compensate + * 10 if we are allowed to write to the TSC MSR to compensate + * 1 if we cannot write MSRs to synchronize TSCs */ - if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) - atomic_set(&test_runs, 1); - else + if (boot_cpu_has(X86_FEATURE_TSC_ADJUST)) atomic_set(&test_runs, 3); + else if (tsc_allow_direct_sync) + atomic_set(&test_runs, 10); + else + atomic_set(&test_runs, 1); retry: /* * Wait for the target to start or to skip the test: @@ -394,6 +401,21 @@ void check_tsc_sync_source(int cpu) goto retry; } +static inline cycles_t write_tsc_adjustment(s64 adjustment) +{ + cycles_t adjval, nextval; + + rdmsrl(MSR_IA32_TSC, adjval); + adjval += adjustment; + wrmsrl(MSR_IA32_TSC, adjval); + rdmsrl(MSR_IA32_TSC, nextval); + + /* + * Estimated clock cycle overhead for wrmsr + rdmsr + */ + return nextval - adjval; +} + /* * Freshly booted CPUs call into this: */ @@ -401,7 +423,7 @@ void check_tsc_sync_target(void) { struct tsc_adjust *cur = this_cpu_ptr(&tsc_adjust); unsigned int cpu = smp_processor_id(); - cycles_t cur_max_warp, gbl_max_warp; + cycles_t cur_max_warp, gbl_max_warp, est_overhead = 0; int cpus = 2; /* Also aborts if there is no TSC. */ @@ -481,12 +503,18 @@ void check_tsc_sync_target(void) * value is used. In the worst case the adjustment needs to go * through a 3rd run for fine tuning. */ - cur->adjusted += cur_max_warp; + if (boot_cpu_has(X86_FEATURE_TSC_ADJUST)) { + cur->adjusted += cur_max_warp + est_overhead; - pr_warn("TSC ADJUST compensate: CPU%u observed %lld warp. Adjust: %lld\n", - cpu, cur_max_warp, cur->adjusted); + pr_warn("TSC ADJUST compensate: CPU%u observed %lld warp. Adjust: %lld\n", + cpu, cur_max_warp, cur->adjusted); - wrmsrl(MSR_IA32_TSC_ADJUST, cur->adjusted); + wrmsrl(MSR_IA32_TSC_ADJUST, cur->adjusted); + } else { + pr_warn("TSC direct sync: CPU%u observed %lld warp. Overhead: %lld\n", + cpu, cur_max_warp, est_overhead); + est_overhead = write_tsc_adjustment(cur_max_warp + est_overhead); + } goto retry; } -- 2.23.0