Welcome to little lamb

Code » anopa » commit ecab222

start/stop: Add timeout support

author Olivier Brunel
2015-03-23 21:11:59 UTC
committer Olivier Brunel
2015-04-04 12:47:36 UTC
parent b667484d0a54a4adfa2b1e4eeada852fa6812d3f

start/stop: Add timeout support

doc/aa-start.pod +33 -1
doc/aa-stop.pod +6 -1
doc/anopa.pod +12 -0
src/anopa/aa-start.c +11 -1
src/anopa/aa-stop.c +12 -1
src/anopa/start-stop.c +236 -47
src/anopa/start-stop.h +5 -2
src/include/anopa/service.h +3 -0
src/libanopa/service.c +19 -0
src/libanopa/services.c +1 -0

diff --git a/doc/aa-start.pod b/doc/aa-start.pod
index f426f97..f75488a 100644
--- a/doc/aa-start.pod
+++ b/doc/aa-start.pod
@@ -5,7 +5,7 @@ aa-start - Start services
 =head1 SYNOPSIS
 
 B<aa-start> [B<-D>] [B<-r> I<repodir>] [B<-l> I<listdir>] [B<-w>]
-[I<service...>]
+[B<-t> I<timeout>] [I<service...>]
 
 =head1 OPTIONS
 
@@ -32,6 +32,11 @@ than once the last one will be used.
 Use I<dir> as repository directory. This is where servicedirs will be looked
 for.
 
+=item B<-t, --timeout> I<timeout>
+
+Set default timeout to I<timeout> seconds. You can use 0 for no timeout.
+Timeout can also be set in service in a file I<timeout> in its servicedir.
+
 =item B<-V, --version>
 
 Show version information and exit.
@@ -51,6 +56,29 @@ possible not to auto-start the ones from I<wants> via B<--no-wants>.
 
 Refer to B<anopa>(1) for descriptions of servicedirs and service dependencies.
 
+=head1 TIMEOUTS
+
+When starting a service, a timestamp is collected. If the service fails to be
+started/ready before the number of seconds specified either in a file I<timeout>
+in its servicedir or using the default value (300 (5 min) by default, unless
+set via B<--timeout>), the service is considered to have timed out.
+
+For long-run services, B<aa-start>(1) will simply stop waiting for event and
+mark the service as timed out.
+
+For one-shot services, signal TERM will be sent to the process, which is then
+given up to 2 seconds to react. If it exits during that time, it will be
+processed as usual, with the exception that if SIGTERM is the cause the service
+will then be marked as timed out (i.e. it could exit with a specified exit code,
+and be then marked failed, or even return 0 and be successfully started).
+
+If the process hasn't ended after those 2 seconds, signal KILL is sent to the
+process and the service marked as timed out.
+
+Note that using a value of 0 as time out (either via I<timeout> file or
+B<--timeout> option) means it never times out, and B<aa-start>(1) will wait for
+it forever; Use with caution.
+
 =head1 STARTING A LONG-RUN SERVICE
 
 When starting a long-run service, B<aa-start>(1) first connects to the I<event>
@@ -141,3 +169,7 @@ simply doing the same again.
 
 Note that it is not possible to show a progress bar while asking for user input,
 not that there should be a need for it anyways.
+
+Also note that once B<aa-start>(1) received a demand of password input, it will
+disable the service's timeout, restoring it once it has been processed (e.g.
+user input has been written to the service's stdin) and resetting its timer.
diff --git a/doc/aa-stop.pod b/doc/aa-stop.pod
index 42f8a5e..5ee6622 100644
--- a/doc/aa-stop.pod
+++ b/doc/aa-stop.pod
@@ -5,7 +5,7 @@ aa-stop - Stop services
 =head1 SYNOPSIS
 
 B<aa-stop> [B<-D>] [B<-r> I<repodir>] [B<-a>] [B<-k> I<service>]
-[I<service...>]
+[B<-t> I<timeout>] [I<service...>]
 
 =head1 OPTIONS
 
@@ -39,6 +39,11 @@ possible (It will be stopped when sending SIGTERM to all running processes.).
 Use I<dir> as repository directory. This is where servicedirs will be looked
 for.
 
+=item B<-t, --timeout> I<timeout>
+
+Set default timeout to I<timeout> seconds. You can use 0 for no timeout.
+Timeout can also be set in service in a file I<timeout> in its servicedir.
+
 =item B<-V, --version>
 
 Show version information and exit.
diff --git a/doc/anopa.pod b/doc/anopa.pod
index 7c29808..8d969aa 100644
--- a/doc/anopa.pod
+++ b/doc/anopa.pod
@@ -110,6 +110,12 @@ be logged; the I<log> service is called the service's logger.
 See B<aa-enable>(1) for how you can use a file instead, to automatically use the
 same logger for all services.
 
+=item An optional regular file I<timeout>
+
+If such a file exists, it should contain the number of seconds before the
+service is considered to be in time out; i.e. B<aa-start>(1)/B<aa-stop>(1) will
+stop waiting for them (killing the process for one-shot services).
+
 =back
 
 For completeness, the following "internals" are also supported.
@@ -170,6 +176,12 @@ end successfully (return non-zero).
 You would usually use such a file for the service mounting the root filesystem
 in initramfs, or launching getty.
 
+=item An optional regular file I<timeout>
+
+If such a file exists, it should contain the number of seconds before the
+service is considered to be in time out; i.e. B<aa-start>(1)/B<aa-stop>(1) will
+stop waiting for them (killing the process for one-shot services).
+
 =back
 
 Note that a one-shot service can have only a I<start> script, only a I<stop>
diff --git a/src/anopa/aa-start.c b/src/anopa/aa-start.c
index d9a5899..913223e 100644
--- a/src/anopa/aa-start.c
+++ b/src/anopa/aa-start.c
@@ -193,6 +193,7 @@ dieusage (int rc)
             " -r, --repodir DIR             Use DIR as repository directory\n"
             " -l, --listdir DIR             Use DIR to list services to start\n"
             " -w, --no-wants                Don't auto-start services from 'wants'\n"
+            " -t, --timeout SECS            Use SECS seconds as default timeout\n"
             " -h, --help                    Show this help screen and exit\n"
             " -V, --version                 Show version information and exit\n"
             );
@@ -213,6 +214,7 @@ main (int argc, char * const argv[])
     int mode_both = 0;
     int i;
 
+    aa_secs_timeout = DEFAULT_TIMEOUT_SECS;
     for (;;)
     {
         struct option longopts[] = {
@@ -220,13 +222,14 @@ main (int argc, char * const argv[])
             { "help",               no_argument,        NULL,   'h' },
             { "listdir",            required_argument,  NULL,   'l' },
             { "repodir",            required_argument,  NULL,   'r' },
+            { "timeout",            required_argument,  NULL,   't' },
             { "version",            no_argument,        NULL,   'V' },
             { "no-wants",           no_argument,        NULL,   'w' },
             { NULL, 0, 0, 0 }
         };
         int c;
 
-        c = getopt_long (argc, argv, "Dhl:r:Vw", longopts, NULL);
+        c = getopt_long (argc, argv, "Dhl:r:t:Vw", longopts, NULL);
         if (c == -1)
             break;
         switch (c)
@@ -248,6 +251,11 @@ main (int argc, char * const argv[])
                 path_repo = optarg;
                 break;
 
+            case 't':
+                if (!uint0_scan (optarg, &aa_secs_timeout))
+                    strerr_diefu2sys (ERR_IO, "set default timeout to ", optarg);
+                break;
+
             case 'V':
                 aa_die_version ();
 
@@ -295,12 +303,14 @@ main (int argc, char * const argv[])
     put_title (1, PROG, "Completed.", 1);
     aa_show_stat_nb (nb_already, "Already up", ANSI_HIGHLIGHT_GREEN_ON);
     aa_show_stat_nb (nb_done, "Started", ANSI_HIGHLIGHT_GREEN_ON);
+    show_stat_service_names (&ga_timedout, "Timed out", ANSI_HIGHLIGHT_RED_ON);
     show_stat_service_names (&ga_failed, "Failed", ANSI_HIGHLIGHT_RED_ON);
     show_stat_service_names (&ga_depend, "Dependency failed", ANSI_HIGHLIGHT_RED_ON);
     aa_show_stat_names (aa_names.s, &ga_io, "I/O error", ANSI_HIGHLIGHT_RED_ON);
     aa_show_stat_names (aa_names.s, &ga_unknown, "Unknown", ANSI_HIGHLIGHT_RED_ON);
     aa_show_stat_names (aa_names.s, &ga_skipped, "Skipped", ANSI_HIGHLIGHT_YELLOW_ON);
 
+    genalloc_free (int, &ga_timedout);
     genalloc_free (int, &ga_failed);
     genalloc_free (int, &ga_depend);
     genalloc_free (int, &ga_unknown);
diff --git a/src/anopa/aa-stop.c b/src/anopa/aa-stop.c
index 0aff6e1..9767547 100644
--- a/src/anopa/aa-stop.c
+++ b/src/anopa/aa-stop.c
@@ -12,6 +12,7 @@
 #include <skalibs/genalloc.h>
 #include <skalibs/strerr2.h>
 #include <skalibs/error.h>
+#include <skalibs/uint.h>
 #include <skalibs/tai.h>
 #include <skalibs/djbunix.h>
 #include <anopa/common.h>
@@ -150,6 +151,7 @@ dieusage (int rc)
             " -D, --double-output           Enable double-output mode\n"
             " -r, --repodir DIR             Use DIR as repository directory\n"
             " -k, --skip SERVICE            Skip (do not stop) SERVICE\n"
+            " -t, --timeout SECS            Use SECS seconds as default timeout\n"
             " -a, --all                     Stop all running services\n"
             " -h, --help                    Show this help screen and exit\n"
             " -V, --version                 Show version information and exit\n"
@@ -170,6 +172,7 @@ main (int argc, char * const argv[])
     int mode_both = 0;
     int i;
 
+    aa_secs_timeout = DEFAULT_TIMEOUT_SECS;
     for (;;)
     {
         struct option longopts[] = {
@@ -178,12 +181,13 @@ main (int argc, char * const argv[])
             { "help",               no_argument,        NULL,   'h' },
             { "skip",               required_argument,  NULL,   'k' },
             { "repodir",            required_argument,  NULL,   'r' },
+            { "timeout",            required_argument,  NULL,   't' },
             { "version",            no_argument,        NULL,   'V' },
             { NULL, 0, 0, 0 }
         };
         int c;
 
-        c = getopt_long (argc, argv, "aDhk:r:V", longopts, NULL);
+        c = getopt_long (argc, argv, "aDhk:r:t:V", longopts, NULL);
         if (c == -1)
             break;
         switch (c)
@@ -208,6 +212,11 @@ main (int argc, char * const argv[])
                 path_repo = optarg;
                 break;
 
+            case 't':
+                if (!uint0_scan (optarg, &aa_secs_timeout))
+                    strerr_diefu2sys (ERR_IO, "set default timeout to ", optarg);
+                break;
+
             case 'V':
                 aa_die_version ();
 
@@ -251,10 +260,12 @@ main (int argc, char * const argv[])
     put_title (1, PROG, "Completed.", 1);
     aa_show_stat_nb (nb_already, "Not up", ANSI_HIGHLIGHT_GREEN_ON);
     aa_show_stat_nb (nb_done, "Stopped", ANSI_HIGHLIGHT_GREEN_ON);
+    show_stat_service_names (&ga_timedout, "Timed out", ANSI_HIGHLIGHT_RED_ON);
     show_stat_service_names (&ga_failed, "Failed", ANSI_HIGHLIGHT_RED_ON);
     aa_show_stat_names (aa_names.s, &ga_io, "I/O error", ANSI_HIGHLIGHT_RED_ON);
     aa_show_stat_names (aa_names.s, &ga_unknown, "Unknown", ANSI_HIGHLIGHT_RED_ON);
 
+    genalloc_free (int, &ga_timedout);
     genalloc_free (int, &ga_failed);
     genalloc_free (int, &ga_unknown);
     genalloc_free (int, &ga_io);
diff --git a/src/anopa/start-stop.c b/src/anopa/start-stop.c
index 3aecceb..747ed11 100644
--- a/src/anopa/start-stop.c
+++ b/src/anopa/start-stop.c
@@ -31,6 +31,7 @@ int nb_already = 0;
 int nb_done = 0;
 int nb_wait_longrun = 0;
 genalloc ga_failed = GENALLOC_ZERO;
+genalloc ga_timedout = GENALLOC_ZERO;
 int cols = 80;
 int is_utf8 = 0;
 int ioloop = 1;
@@ -104,6 +105,29 @@ draw_password ()
     draw |= DRAW_CUR_PASSWORD;
 }
 
+static void
+is_noflush_time (int secs)
+{
+    char buf[UINT_FMT];
+    int mins;
+
+    mins = secs / 60;
+    secs -= 60 * mins;
+
+    if (mins > 0)
+    {
+        buf[uint_fmt (buf, mins)] = '\0';
+        aa_is_noflush (AA_OUT, buf);
+        aa_is_noflush (AA_OUT, "m");
+    }
+    if (secs > 0)
+    {
+        buf[uint_fmt (buf, secs)] = '\0';
+        aa_is_noflush (AA_OUT, buf);
+        aa_is_noflush (AA_OUT, "s");
+    }
+}
+
 void
 draw_waiting (int already_drawn)
 {
@@ -177,20 +201,14 @@ draw_waiting (int already_drawn)
 
     if (secs >= 0)
     {
-        int mins;
-
-        mins = secs / 60;
-        secs -= 60 * mins;
-
-        if (mins > 0)
-        {
-            buf[uint_fmt (buf, mins)] = '\0';
-            aa_is_noflush (AA_OUT, buf);
-            aa_is_noflush (AA_OUT, "m");
-        }
-        buf[uint_fmt (buf, secs)] = '\0';
-        aa_is_noflush (AA_OUT, buf);
-        aa_is_noflush (AA_OUT, "s");
+        is_noflush_time (secs);
+        aa_is_noflush (AA_OUT, "/");
+        if (aa_service (si)->secs_timeout > 0)
+            is_noflush_time (aa_service (si)->secs_timeout);
+        else if (is_utf8)
+            aa_is_noflush (AA_OUT, "\u221e"); /* infinity sign */
+        else
+            aa_is_noflush (AA_OUT, "Inf");
     }
 
     if (nb > 1 || secs >= 0)
@@ -227,7 +245,7 @@ term_set_echo (int on)
     return tcsetattr (0, TCSANOW, &termios);
 }
 
-void
+int
 refresh_draw ()
 {
     unsigned int old_draw = draw;
@@ -300,10 +318,7 @@ refresh_draw ()
     if (draw & DRAW_NEED_WAITING)
         draw_waiting ((old_draw & DRAW_CUR_WAITING) && !(draw & DRAW_CUR_PROGRESS));
 
-    if (draw & DRAW_CUR_WAITING)
-        iol_deadline_addsec (1);
-    else
-        iol_deadline_addsec (TIMEOUT_SECS);
+    return 1000 * ((draw & DRAW_CUR_WAITING) ? 1 : SECS_BEFORE_WAITING);
 }
 
 void
@@ -313,12 +328,6 @@ add_name_to_ga (const char *name, genalloc *ga)
     genalloc_append (int, ga, &offset);
 }
 
-void
-iol_deadline_addsec (int n)
-{
-    tain_addsec_g (&iol_deadline, n);
-}
-
 void
 remove_fd_from_iop (int fd)
 {
@@ -493,6 +502,9 @@ handle_fd_progress (int si)
         draw |= DRAW_NEED_PASSWORD;
         /* clear in order to "reset" any WAITING */
         clear_draw ();
+        /* store timeout and disable it for now */
+        pg->secs_timeout = s->secs_timeout;
+        s->secs_timeout = 0;
         return 0;
     }
     else if (pg->is_drawn < 0)
@@ -604,6 +616,11 @@ end_si_password (void)
     pg->aa_pg.sa.len = 0;
     s->pi = -1;
     si_password = -1;
+    /* restore timeout */
+    s->secs_timeout = pg->secs_timeout;
+    pg->secs_timeout = 0;
+    /* reset ts */
+    tain_copynow (&s->ts_exec);
 
     r = term_set_echo (1);
     if (r < 0)
@@ -695,34 +712,52 @@ handle_oneshot (int is_start)
     else
     {
         aa_service_status *svst = &aa_service (si)->st;
-        char buf[20];
 
-        svst->event = (is_start) ? AA_EVT_START_FAILED: AA_EVT_STOP_FAILED;
-        svst->code = wstat;
-        tain_copynow (&svst->stamp);
-        aa_service_status_set_msg (svst, "");
-        if (aa_service_status_write (svst, aa_service_name (aa_service (si))) < 0)
-            strerr_warnwu2sys ("write service status file for ", aa_service_name (aa_service (si)));
-
-        if (WIFEXITED (wstat))
+        /* if this is the SIGTERM we sent on timeout, treat it as timed out */
+        if (aa_service (si)->timedout && !WIFEXITED (wstat) && WTERMSIG (wstat) == SIGTERM)
         {
-            byte_copy (buf, 9, "exitcode ");
-            buf[9 + uint_fmt (buf + 9, WEXITSTATUS (wstat))] = '\0';
+            svst->event = (is_start) ? AA_EVT_STARTING_FAILED: AA_EVT_STOPPING_FAILED;
+            svst->code = ERR_TIMEDOUT;
+            tain_copynow (&svst->stamp);
+            aa_service_status_set_msg (svst, "");
+            if (aa_service_status_write (svst, aa_service_name (aa_service (si))) < 0)
+                strerr_warnwu2sys ("write service status file for ", aa_service_name (aa_service (si)));
+
+            put_err_service (aa_service_name (aa_service (si)), ERR_TIMEDOUT, 1);
+            genalloc_append (int, &ga_timedout, &si);
         }
         else
         {
-            const char *name;
+            char buf[20];
+
+            svst->event = (is_start) ? AA_EVT_START_FAILED: AA_EVT_STOP_FAILED;
+            svst->code = wstat;
+            tain_copynow (&svst->stamp);
+            aa_service_status_set_msg (svst, "");
+            if (aa_service_status_write (svst, aa_service_name (aa_service (si))) < 0)
+                strerr_warnwu2sys ("write service status file for ", aa_service_name (aa_service (si)));
 
-            name = sig_name (WTERMSIG (wstat));
-            byte_copy (buf, 10, "signal SIG");
-            byte_copy (buf + 10, strlen (name) + 1, name);
+            if (WIFEXITED (wstat))
+            {
+                byte_copy (buf, 9, "exitcode ");
+                buf[9 + uint_fmt (buf + 9, WEXITSTATUS (wstat))] = '\0';
+            }
+            else
+            {
+                const char *name;
+
+                name = sig_name (WTERMSIG (wstat));
+                byte_copy (buf, 10, "signal SIG");
+                byte_copy (buf + 10, strlen (name) + 1, name);
+            }
+
+            put_err_service (aa_service_name (aa_service (si)), ERR_FAILED, 0);
+            add_err (": ");
+            add_err (buf);
+            end_err ();
+            genalloc_append (int, &ga_failed, &si);
         }
 
-        put_err_service (aa_service_name (aa_service (si)), ERR_FAILED, 0);
-        add_err (": ");
-        add_err (buf);
-        end_err ();
-        genalloc_append (int, &ga_failed, &si);
         if (is_start)
             check_essential (si);
     }
@@ -959,6 +994,152 @@ exec_cb (int si, aa_evt evt, pid_t pid)
     }
 }
 
+int
+process_timeouts (aa_mode mode, aa_scan_cb scan_cb)
+{
+    int si;
+    int l;
+    int i;
+    tain_t ts_timeout;
+    tain_t ts;
+    tain_t tms;
+    int ms = -1;
+    int scan = 0;
+
+    l = genalloc_len (int, &aa_tmp_list);
+    for (i = 0; i < l; ++i)
+    {
+        si = list_get (&aa_tmp_list, i);
+        /* no limit? */
+        if (aa_service (si)->secs_timeout == 0)
+            continue;
+
+        tain_from_millisecs (&ts_timeout, 1000 * aa_service (si)->secs_timeout);
+        tain_add (&ts, &aa_service (si)->ts_exec, &ts_timeout);
+        /* timeout expired? */
+        if (tain_less (&ts, &STAMP))
+        {
+            /* not yet signaled? */
+            if (!aa_service (si)->timedout)
+            {
+                kill (genalloc_s (pid_t, &ga_pid)[i], SIGTERM);
+                aa_service (si)->timedout = 1;
+            }
+
+            tain_addsec (&tms, &ts, 2);
+            if (!tain_less (&tms, &STAMP))
+            {
+                aa_service_status *svst = &aa_service (si)->st;
+
+                kill (genalloc_s (pid_t, &ga_pid)[i], SIGKILL);
+
+                remove_from_list (&aa_tmp_list, si);
+                ga_remove (pid_t, &ga_pid, i);
+                if (si == si_password)
+                    end_si_password ();
+                if (aa_service (si)->fd_in > 0)
+                    close_fd_for (aa_service (si)->fd_in, si);
+                if (aa_service (si)->fd_out > 0)
+                    close_fd_for (aa_service (si)->fd_out, si);
+                if (aa_service (si)->fd_progress > 0)
+                    close_fd_for (aa_service (si)->fd_progress, si);
+
+                svst->event = (mode == AA_MODE_START) ? AA_EVT_STARTING_FAILED: AA_EVT_STOPPING_FAILED;
+                svst->code = ERR_TIMEDOUT;
+                tain_copynow (&svst->stamp);
+                aa_service_status_set_msg (svst, "");
+                if (aa_service_status_write (svst, aa_service_name (aa_service (si))) < 0)
+                    strerr_warnwu2sys ("write service status file for ", aa_service_name (aa_service (si)));
+
+                put_err_service (aa_service_name (aa_service (si)), ERR_TIMEDOUT, 1);
+                genalloc_append (int, &ga_timedout, &si);
+                if (mode == AA_MODE_START)
+                    check_essential (si);
+
+                remove_from_list (&aa_main_list, si);
+                scan = 1;
+            }
+            else
+            {
+                int _ms;
+
+                ts = tms;
+                tain_sub (&tms, &ts, &STAMP);
+                _ms = tain_to_millisecs (&tms);
+                if (_ms > 0 && (ms < 0 || _ms < ms))
+                    ms = _ms;
+            }
+        }
+        else
+        {
+            int _ms;
+
+            tain_sub (&tms, &ts, &STAMP);
+            _ms = tain_to_millisecs (&tms);
+            if (ms < 0 || _ms < ms)
+                ms = _ms;
+        }
+    }
+
+    if (nb_wait_longrun > 0)
+    {
+        int j = 0;
+
+        l = genalloc_len (int, &aa_main_list);
+
+        for (i = 0; i < l && j < nb_wait_longrun; ++i)
+            if (aa_service (list_get (&aa_main_list, i))->ft_id > 0)
+            {
+                ++j;
+                si = list_get (&aa_main_list, i);
+                /* no limit? */
+                if (aa_service (si)->secs_timeout == 0)
+                    continue;
+
+                tain_from_millisecs (&ts_timeout, 1000 * aa_service (si)->secs_timeout);
+                tain_add (&ts, &aa_service (si)->ts_exec, &ts_timeout);
+                /* timeout expired? */
+                if (tain_less (&ts, &STAMP))
+                {
+                    aa_service_status *svst = &aa_service (si)->st;
+
+                    aa_unsubscribe_for (aa_service (si)->ft_id);
+                    aa_service (si)->ft_id = 0;
+                    --nb_wait_longrun;
+
+                    svst->event = (mode == AA_MODE_START) ? AA_EVT_STARTING_FAILED: AA_EVT_STOPPING_FAILED;
+                    svst->code = ERR_TIMEDOUT;
+                    tain_copynow (&svst->stamp);
+                    aa_service_status_set_msg (svst, "");
+                    if (aa_service_status_write (svst, aa_service_name (aa_service (si))) < 0)
+                        strerr_warnwu2sys ("write service status file for ", aa_service_name (aa_service (si)));
+
+                    put_err_service (aa_service_name (aa_service (si)), ERR_TIMEDOUT, 1);
+                    genalloc_append (int, &ga_timedout, &si);
+                    if (mode == AA_MODE_START)
+                        check_essential (si);
+
+                    remove_from_list (&aa_main_list, si);
+                    scan = 1;
+                }
+                else
+                {
+                    int _ms;
+
+                    tain_sub (&tms, &ts, &STAMP);
+                    _ms = tain_to_millisecs (&tms);
+                    if (ms < 0 || _ms < ms)
+                        ms = _ms;
+                }
+            }
+    }
+
+    if (scan)
+        aa_scan_mainlist (scan_cb, mode);
+
+    return ms;
+}
+
 void
 mainloop (aa_mode mode, aa_scan_cb scan_cb)
 {
@@ -1003,15 +1184,23 @@ mainloop (aa_mode mode, aa_scan_cb scan_cb)
     {
         int nb_iop;
         int r;
+        int ms1, ms2;
+        tain_t tms;
 
-        refresh_draw ();
+        ms1 = process_timeouts (mode, scan_cb);
+        ms2 = refresh_draw ();
+        tain_from_millisecs (&tms, (ms1 < 0 || ms2 < ms1) ? ms2 : ms1);
+        tain_add (&iol_deadline, &STAMP, &tms);
 
         nb_iop = genalloc_len (iopause_fd, &ga_iop);
         r = iopause_g (genalloc_s (iopause_fd, &ga_iop), nb_iop, &iol_deadline);
         if (r < 0)
             strerr_diefu1sys (ERR_IO, "iopause");
         else if (r == 0)
-            draw |= DRAW_NEED_WAITING;
+        {
+            if (ms1 < 0 || ms2 < ms1)
+                draw |= DRAW_NEED_WAITING;
+        }
         else
         {
             iopause_fd *iofd;
diff --git a/src/anopa/start-stop.h b/src/anopa/start-stop.h
index 05d4163..7c14ddb 100644
--- a/src/anopa/start-stop.h
+++ b/src/anopa/start-stop.h
@@ -9,7 +9,8 @@
 #include <anopa/progress.h>
 #include <anopa/output.h>
 
-#define TIMEOUT_SECS                3
+#define SECS_BEFORE_WAITING         7
+#define DEFAULT_TIMEOUT_SECS        300
 
 #define ANSI_PREV_LINE              "\x1B[F"
 #define ANSI_CLEAR_AFTER            "\x1B[K"
@@ -25,6 +26,7 @@ extern int nb_already;
 extern int nb_done;
 extern int nb_wait_longrun;
 extern genalloc ga_failed;
+extern genalloc ga_timedout;
 extern int cols;
 extern int is_utf8;
 extern int ioloop;
@@ -58,10 +60,11 @@ struct progress
     aa_progress aa_pg;
     int si;
     int is_drawn;
+    int secs_timeout;
 };
 
 void free_progress (struct progress *pg);
-void refresh_draw ();
+int refresh_draw ();
 void draw_waiting (int already_drawn);
 void draw_progress_for (int si);
 void clear_draw ();
diff --git a/src/include/anopa/service.h b/src/include/anopa/service.h
index e6d4a22..f7313f6 100644
--- a/src/include/anopa/service.h
+++ b/src/include/anopa/service.h
@@ -19,6 +19,7 @@ extern stralloc aa_names;
 extern genalloc aa_main_list;
 extern genalloc aa_tmp_list;
 extern genalloc aa_pid_list;
+extern int aa_secs_timeout;
 
 #define aa_service(i)               (&((aa_service *) aa_services.s)[i])
 #define aa_service_name(service)    (aa_names.s + (service)->offset_name)
@@ -58,6 +59,7 @@ typedef struct
     genalloc needs;
     genalloc wants;
     genalloc after;
+    int secs_timeout;
     aa_ls ls;
     aa_service_status st;
     tain_t ts_exec;
@@ -70,6 +72,7 @@ typedef struct
     stralloc sa_out;
     int fd_progress;
     int pi;
+    int timedout;
 } aa_service;
 
 typedef void (*aa_close_fd_fn) (int fd);
diff --git a/src/libanopa/service.c b/src/libanopa/service.c
index fb7889d..6e532c4 100644
--- a/src/libanopa/service.c
+++ b/src/libanopa/service.c
@@ -6,6 +6,7 @@
 #include <skalibs/genalloc.h>
 #include <skalibs/bytestr.h>
 #include <skalibs/direntry.h>
+#include <skalibs/uint.h>
 #include <skalibs/tai.h>
 #include <skalibs/strerr2.h>
 #include <s6/s6-supervise.h>
@@ -246,6 +247,24 @@ aa_ensure_service_loaded (int si, aa_mode mode, int no_wants, aa_load_fail_cb lf
     if (r < 0 && (r != -ERR_IO || errno != ENOENT))
         goto err;
 
+    {
+        stralloc sa_to = STRALLOC_ZERO;
+
+        sa.len -= strlen ("before") + 1;
+        stralloc_catb (&sa, "timeout", strlen ("timeout") + 1);
+        if (openreadclose (sa.s, &sa_to, 0) == 0 && sa_to.len > 0)
+        {
+            r = uint_scan (sa_to.s, &aa_service (si)->secs_timeout);
+            if (!r || (sa_to.s[r] != '\n' && sa_to.s[r] != '\0'))
+            {
+                strerr_warnwu3x ("read timeout for ", aa_service_name (aa_service (si)), "; using default");
+                aa_service (si)->secs_timeout = aa_secs_timeout;
+            }
+        }
+        else
+            aa_service (si)->secs_timeout = aa_secs_timeout;
+    }
+
     stralloc_free (&sa);
     aa_service (si)->ls = AA_LOAD_DONE;
     tain_now_g ();
diff --git a/src/libanopa/services.c b/src/libanopa/services.c
index 27ffee4..0935f45 100644
--- a/src/libanopa/services.c
+++ b/src/libanopa/services.c
@@ -8,6 +8,7 @@ genalloc aa_services    = GENALLOC_ZERO;
 stralloc aa_names       = STRALLOC_ZERO;
 genalloc aa_main_list   = GENALLOC_ZERO;
 genalloc aa_tmp_list    = GENALLOC_ZERO;
+int aa_secs_timeout     = 0;
 
 ftrigr_t _aa_ft         = FTRIGR_ZERO;
 aa_exec_cb _exec_cb     = NULL;