From: jbeulich@novell.com
Subject: fold per-CPU accounting data into a structure
Patch-mainline: n/a
... to simplify generated code, especially in timer_interrupt(). This
becomes more important with more such data elements added (i.e. by
patches.xen/xen-x86-xtime-lock).
--- sle11sp1-2010-02-17.orig/arch/x86/kernel/time-xen.c 2010-02-18 17:30:48.000000000 +0100
+++ sle11sp1-2010-02-17/arch/x86/kernel/time-xen.c 2010-02-18 17:32:00.000000000 +0100
@@ -57,12 +57,15 @@ static u32 shadow_tv_version;
/* Keep track of last time we did processing/updating of jiffies and xtime. */
static u64 processed_system_time; /* System time (ns) at last processing. */
-static DEFINE_PER_CPU(u64, processed_system_time);
-static DEFINE_PER_CPU(u64, accounted_system_time);
-/* How much CPU time was spent blocked and how much was 'stolen'? */
-static DEFINE_PER_CPU(u64, processed_stolen_time);
-static DEFINE_PER_CPU(u64, processed_blocked_time);
+struct local_time_info {
+ u64 processed_system;
+ u64 accounted_system;
+ /* How much CPU time was spent blocked and how much was 'stolen'? */
+ u64 accounted_stolen;
+ u64 accounted_blocked;
+};
+static DEFINE_PER_CPU(struct local_time_info, local_time);
/* Current runstate of each CPU (updated automatically by the hypervisor). */
DEFINE_PER_CPU(struct vcpu_runstate_info, runstate);
@@ -440,6 +443,7 @@ static irqreturn_t timer_interrupt(int i
s64 delta, delta_cpu, stolen, blocked;
unsigned int i, cpu = smp_processor_id();
struct shadow_time_info *shadow = &per_cpu(shadow_time, cpu);
+ struct local_time_info *local = &per_cpu(local_time, cpu);
bool duty = false;
struct vcpu_runstate_info runstate;
@@ -468,7 +472,7 @@ static irqreturn_t timer_interrupt(int i
delta = delta_cpu =
shadow->system_timestamp + get_nsec_offset(shadow);
delta -= processed_system_time;
- delta_cpu -= per_cpu(processed_system_time, cpu);
+ delta_cpu -= local->processed_system;
get_runstate_snapshot(&runstate);
} while (!time_values_up_to_date());
@@ -482,10 +486,10 @@ static irqreturn_t timer_interrupt(int i
"processed=%Lx/%Lx\n",
cpu, delta, delta_cpu, shadow->system_timestamp,
get_nsec_offset(shadow), blocked,
- per_cpu(processed_system_time, cpu));
+ local->processed_system);
for_each_cpu_and(i, cpu_online_mask, cpumask_of(cpu))
printk(" %u: %Lx\n", i,
- per_cpu(processed_system_time, i));
+ per_cpu(local_time.processed_system, i));
}
} else if (unlikely(delta_cpu < -(s64)permitted_clock_jitter)) {
blocked = processed_system_time;
@@ -496,10 +500,10 @@ static irqreturn_t timer_interrupt(int i
" shadow=%Lx off=%Lx processed=%Lx/%Lx\n",
cpu, delta_cpu, shadow->system_timestamp,
get_nsec_offset(shadow), blocked,
- per_cpu(processed_system_time, cpu));
+ local->processed_system);
for_each_cpu_and(i, cpu_online_mask, cpumask_of(cpu))
printk(" %u: %Lx\n", i,
- per_cpu(processed_system_time, i));
+ per_cpu(local_time.processed_system, i));
}
} else if (duty) {
/* System-wide jiffy work. */
@@ -524,11 +528,10 @@ static irqreturn_t timer_interrupt(int i
}
delta = delta_cpu;
- delta_cpu += per_cpu(processed_system_time, cpu)
- - per_cpu(accounted_system_time, cpu);
+ delta_cpu += local->processed_system - local->accounted_system;
if (delta >= NS_PER_TICK) {
do_div(delta, NS_PER_TICK);
- per_cpu(processed_system_time, cpu) += delta * NS_PER_TICK;
+ local->processed_system += delta * NS_PER_TICK;
}
/*
@@ -537,14 +540,14 @@ static irqreturn_t timer_interrupt(int i
*/
stolen = runstate.time[RUNSTATE_runnable]
+ runstate.time[RUNSTATE_offline]
- - per_cpu(processed_stolen_time, cpu);
+ - local->accounted_stolen;
if ((stolen > 0) && (delta_cpu > 0)) {
delta_cpu -= stolen;
if (unlikely(delta_cpu < 0))
stolen += delta_cpu; /* clamp local-time progress */
do_div(stolen, NS_PER_TICK);
- per_cpu(processed_stolen_time, cpu) += stolen * NS_PER_TICK;
- per_cpu(accounted_system_time, cpu) += stolen * NS_PER_TICK;
+ local->accounted_stolen += stolen * NS_PER_TICK;
+ local->accounted_system += stolen * NS_PER_TICK;
account_steal_ticks(stolen);
}
@@ -553,21 +556,21 @@ static irqreturn_t timer_interrupt(int i
* ensures that the ticks are accounted as idle/wait.
*/
blocked = runstate.time[RUNSTATE_blocked]
- - per_cpu(processed_blocked_time, cpu);
+ - local->accounted_blocked;
if ((blocked > 0) && (delta_cpu > 0)) {
delta_cpu -= blocked;
if (unlikely(delta_cpu < 0))
blocked += delta_cpu; /* clamp local-time progress */
do_div(blocked, NS_PER_TICK);
- per_cpu(processed_blocked_time, cpu) += blocked * NS_PER_TICK;
- per_cpu(accounted_system_time, cpu) += blocked * NS_PER_TICK;
+ local->accounted_blocked += blocked * NS_PER_TICK;
+ local->accounted_system += blocked * NS_PER_TICK;
account_idle_ticks(blocked);
}
/* Account user/system ticks. */
if (delta_cpu > 0) {
do_div(delta_cpu, NS_PER_TICK);
- per_cpu(accounted_system_time, cpu) += delta_cpu * NS_PER_TICK;
+ local->accounted_system += delta_cpu * NS_PER_TICK;
if (user_mode_vm(get_irq_regs()))
account_user_time(current, (cputime_t)delta_cpu,
(cputime_t)delta_cpu);
@@ -606,9 +609,9 @@ static void init_missing_ticks_accountin
{
struct vcpu_runstate_info *runstate = setup_runstate_area(cpu);
- per_cpu(processed_blocked_time, cpu) =
+ per_cpu(local_time.accounted_blocked, cpu) =
runstate->time[RUNSTATE_blocked];
- per_cpu(processed_stolen_time, cpu) =
+ per_cpu(local_time.accounted_stolen, cpu) =
runstate->time[RUNSTATE_runnable] +
runstate->time[RUNSTATE_offline];
}
@@ -668,8 +671,8 @@ static void xen_clocksource_resume(void)
BUG();
}
get_time_values_from_xen(cpu);
- per_cpu(accounted_system_time, cpu) =
- per_cpu(processed_system_time, cpu) =
+ per_cpu(local_time.accounted_system, cpu) =
+ per_cpu(local_time.processed_system, cpu) =
per_cpu(shadow_time, 0).system_timestamp;
init_missing_ticks_accounting(cpu);
}
@@ -770,8 +773,8 @@ void __init time_init(void)
get_time_values_from_xen(0);
processed_system_time = per_cpu(shadow_time, 0).system_timestamp;
- per_cpu(processed_system_time, 0) = processed_system_time;
- per_cpu(accounted_system_time, 0) = processed_system_time;
+ per_cpu(local_time.processed_system, 0) = processed_system_time;
+ per_cpu(local_time.accounted_system, 0) = processed_system_time;
init_missing_ticks_accounting(0);
clocksource_register(&clocksource_xen);
@@ -849,7 +852,7 @@ static void stop_hz_timer(void)
singleshot.timeout_abs_ns = jiffies_to_st(j);
if (!singleshot.timeout_abs_ns)
return;
- local = per_cpu(processed_system_time, cpu);
+ local = per_cpu(local_time.processed_system, cpu);
if ((s64)(singleshot.timeout_abs_ns - local) <= NS_PER_TICK) {
cpumask_clear_cpu(cpu, nohz_cpu_mask);
singleshot.timeout_abs_ns = local + NS_PER_TICK;
@@ -918,8 +921,8 @@ int __cpuinit local_setup_timer(unsigned
do {
seq = read_seqbegin(&xtime_lock);
/* Use cpu0 timestamp: cpu's shadow is not initialised yet. */
- per_cpu(accounted_system_time, cpu) =
- per_cpu(processed_system_time, cpu) =
+ per_cpu(local_time.accounted_system, cpu) =
+ per_cpu(local_time.processed_system, cpu) =
per_cpu(shadow_time, 0).system_timestamp;
init_missing_ticks_accounting(cpu);
} while (read_seqretry(&xtime_lock, seq));
