writeback: charge leaked page dirties to active tasks

It's a years long problem that a large number of short-lived dirtiers
(eg. gcc instances in a fast kernel build) may starve long-run dirtiers
(eg. dd) as well as pushing the dirty pages to the global hard limit.

The solution is to charge the pages dirtied by the exited gcc to the
other random dirtying tasks. It sounds not perfect, however should
behave good enough in practice, seeing as that throttled tasks aren't
actually running so those that are running are more likely to pick it up
and get throttled, therefore promoting an equal spread.

Randy: fix compile error: 'dirty_throttle_leaks' undeclared in exit.c

Acked-by: Jan Kara <jack@suse.cz>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Randy Dunlap <rdunlap@xenotime.net>
Signed-off-by: Wu Fengguang <fengguang.wu@intel.com>

diff --git a/include/linux/writeback.h b/include/linux/writeback.h
index a378c295851f8cf9fe4d804080b30f72cf5101cd..05eaf5e3aad75d7dc8b4c4d0bce1a2e0114a0999 100644 (file)
--- a/include/linux/writeback.h
+++ b/include/linux/writeback.h
@@ -7,6 +7,8 @@
 #include <linux/sched.h>
 #include <linux/fs.h>
 
+DECLARE_PER_CPU(int, dirty_throttle_leaks);
+
 /*
  * The 1/4 region under the global dirty thresh is for smooth dirty throttling:
  *

diff --git a/kernel/exit.c b/kernel/exit.c
index d0b7d988f8735beb6e1e7b5f7b42ecbd04b91bf5..d4aac24cc46914165b99d41d67011a85640839ef 100644 (file)
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -51,6 +51,7 @@
 #include <trace/events/sched.h>
 #include <linux/hw_breakpoint.h>
 #include <linux/oom.h>
+#include <linux/writeback.h>
 
 #include <asm/uaccess.h>
 #include <asm/unistd.h>
@@ -1037,6 +1038,8 @@ NORET_TYPE void do_exit(long code)
        validate_creds_for_do_exit(tsk);
 
        preempt_disable();
+       if (tsk->nr_dirtied)
+               __this_cpu_add(dirty_throttle_leaks, tsk->nr_dirtied);
        exit_rcu();
        /* causes final put_task_struct in finish_task_switch(). */
        tsk->state = TASK_DEAD;

diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index 50f08241f9815668d73b1adfcd5c9585f11bbc15..619c445fc03c4daa453101e01d2ce6ee1740b24c 100644 (file)
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -1214,6 +1214,22 @@ void set_page_dirty_balance(struct page *page, int page_mkwrite)
 
 static DEFINE_PER_CPU(int, bdp_ratelimits);
 
+/*
+ * Normal tasks are throttled by
+ *     loop {
+ *             dirty tsk->nr_dirtied_pause pages;
+ *             take a snap in balance_dirty_pages();
+ *     }
+ * However there is a worst case. If every task exit immediately when dirtied
+ * (tsk->nr_dirtied_pause - 1) pages, balance_dirty_pages() will never be
+ * called to throttle the page dirties. The solution is to save the not yet
+ * throttled page dirties in dirty_throttle_leaks on task exit and charge them
+ * randomly into the running tasks. This works well for the above worst case,
+ * as the new task will pick up and accumulate the old task's leaked dirty
+ * count and eventually get throttled.
+ */
+DEFINE_PER_CPU(int, dirty_throttle_leaks) = 0;
+
 /**
  * balance_dirty_pages_ratelimited_nr - balance dirty memory state
  * @mapping: address_space which was dirtied
@@ -1261,6 +1277,17 @@ void balance_dirty_pages_ratelimited_nr(struct address_space *mapping,
                        ratelimit = 0;
                }
        }
+       /*
+        * Pick up the dirtied pages by the exited tasks. This avoids lots of
+        * short-lived tasks (eg. gcc invocations in a kernel build) escaping
+        * the dirty throttling and livelock other long-run dirtiers.
+        */
+       p = &__get_cpu_var(dirty_throttle_leaks);
+       if (*p > 0 && current->nr_dirtied < ratelimit) {
+               nr_pages_dirtied = min(*p, ratelimit - current->nr_dirtied);
+               *p -= nr_pages_dirtied;
+               current->nr_dirtied += nr_pages_dirtied;
+       }
        preempt_enable();
 
        if (unlikely(current->nr_dirtied >= ratelimit))