sched: Add SD_PREFER_LOCAL

And turn it on for NUMA and MC domains. This improves
locality in balancing decisions by keeping up to
capacity amount of tasks local before looking for idle
CPUs. (and twice the capacity if SD_POWERSAVINGS_BALANCE
is set.)

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/include/linux/sched.h b/include/linux/sched.h
index ee1f88993097f83cbde861a3040640258347d505..b4a39bb2b4a464f567ee91879dd7a44d3466f6f1 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -805,7 +805,7 @@ enum cpu_idle_type {
 #define SD_BALANCE_FORK                0x0008  /* Balance on fork, clone */
 #define SD_BALANCE_WAKE                0x0010  /* Balance on wakeup */
 #define SD_WAKE_AFFINE         0x0020  /* Wake task to waking CPU */
-
+#define SD_PREFER_LOCAL                0x0040  /* Prefer to keep tasks local to this domain */
 #define SD_SHARE_CPUPOWER      0x0080  /* Domain members share cpu power */
 #define SD_POWERSAVINGS_BALANCE        0x0100  /* Balance for power savings */
 #define SD_SHARE_PKG_RESOURCES 0x0200  /* Domain members share cpu pkg resources */

diff --git a/include/linux/topology.h b/include/linux/topology.h
index 936ab2b3768356c6e6e909a5bae65ca5730a1181..a6614b0242a9a34331aa4418b07a0629dbbe1464 100644 (file)
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -129,6 +129,7 @@ int arch_update_cpu_topology(void);
                                | 1*SD_BALANCE_FORK                     \
                                | 1*SD_BALANCE_WAKE                     \
                                | 1*SD_WAKE_AFFINE                      \
+                               | 1*SD_PREFER_LOCAL                     \
                                | 0*SD_SHARE_CPUPOWER                   \
                                | 1*SD_SHARE_PKG_RESOURCES              \
                                | 0*SD_SERIALIZE                        \
@@ -161,6 +162,7 @@ int arch_update_cpu_topology(void);
                                | 1*SD_BALANCE_FORK                     \
                                | 1*SD_BALANCE_WAKE                     \
                                | 1*SD_WAKE_AFFINE                      \
+                               | 1*SD_PREFER_LOCAL                     \
                                | 0*SD_SHARE_CPUPOWER                   \
                                | 0*SD_SHARE_PKG_RESOURCES              \
                                | 0*SD_SERIALIZE                        \

diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 280892e9d85e3543564a1397cd51ad82158f7545..a37f311f436e4e0d54f97f207b6f40d9b8a7c445 100644 (file)
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -1360,7 +1360,7 @@ static int select_task_rq_fair(struct task_struct *p, int sd_flag, int flags)
                 * If power savings logic is enabled for a domain, see if we
                 * are not overloaded, if so, don't balance wider.
                 */
-               if (tmp->flags & SD_POWERSAVINGS_BALANCE) {
+               if (tmp->flags & (SD_POWERSAVINGS_BALANCE|SD_PREFER_LOCAL)) {
                        unsigned long power = 0;
                        unsigned long nr_running = 0;
                        unsigned long capacity;
@@ -1373,7 +1373,10 @@ static int select_task_rq_fair(struct task_struct *p, int sd_flag, int flags)
 
                        capacity = DIV_ROUND_CLOSEST(power, SCHED_LOAD_SCALE);
 
-                       if (nr_running/2 < capacity)
+                       if (tmp->flags & SD_POWERSAVINGS_BALANCE)
+                               nr_running /= 2;
+
+                       if (nr_running < capacity)
                                break;
                }