zed: Add synchronous zedlets

Historically, ZED has blindly spawned off zedlets in parallel and never
worried about their completion order.  This means that you can
potentially have zedlets for event number 2 starting before zedlets for
event number 1 had finished.  Most of the time this is fine, and it
actually helps a lot when the system is getting spammed with hundreds
of events.

However, there are times when you want your zedlets to be executed
in sequence with the event ID.  That is where synchronous zedlets
come in.

ZED will wait for all previously spawned zedlets to finish before
running a synchronous zedlet.  Synchronous zedlets are guaranteed to be
the only zedlet running.  No other zedlets may run in parallel with a
synchronous zedlet.  Users should be careful to only use synchronous
zedlets when needed, since they decrease parallelism.

To make a zedlet synchronous, simply add a "-sync-" immediately
following the event name in the zedlet's file name:

	EVENT_NAME-sync-ZEDLETNAME.sh

For example, if you wanted a synchronous statechange script:

	statechange-sync-myzedlet.sh

Signed-off-by: Tony Hutter <[email protected]>

Author: tonyhutter

cmd/zed/zed.d/deadman-slot_off.sh renamed to cmd/zed/zed.d/deadman-sync-slot_off.sh

@@ -196,38 +196,33 @@ _nop(int sig)
 	(void) sig;
 }
 
-static void *
-_reap_children(void *arg)
+static void
+wait_for_children(boolean_t do_pause, boolean_t wait)
 {
-	(void) arg;
-	struct launched_process_node node, *pnode;
 	pid_t pid;
-	int status;
 	struct rusage usage;
-	struct sigaction sa = {};
-
-	(void) sigfillset(&sa.sa_mask);
-	(void) sigdelset(&sa.sa_mask, SIGCHLD);
-	(void) pthread_sigmask(SIG_SETMASK, &sa.sa_mask, NULL);
-
-	(void) sigemptyset(&sa.sa_mask);
-	sa.sa_handler = _nop;
-	sa.sa_flags = SA_NOCLDSTOP;
-	(void) sigaction(SIGCHLD, &sa, NULL);
+	int status;
+	struct launched_process_node node, *pnode;
 
 	for (_reap_children_stop = B_FALSE; !_reap_children_stop; ) {
 		(void) pthread_mutex_lock(&_launched_processes_lock);
-		pid = wait4(0, &status, WNOHANG, &usage);
-
+		pid = wait4(0, &status, wait ? 0 : WNOHANG, &usage);
 		if (pid == 0 || pid == (pid_t)-1) {
 			(void) pthread_mutex_unlock(&_launched_processes_lock);
-			if (pid == 0 || errno == ECHILD)
-				pause();
-			else if (errno != EINTR)
+			if ((pid == 0) || (errno == ECHILD)) {
+				if (do_pause)
+					pause();
+			} else if (errno != EINTR)
 				zed_log_msg(LOG_WARNING,
 				    "Failed to wait for children: %s",
 				    strerror(errno));
+			zed_log_msg(LOG_INFO, "badpid exit %d", pid);
+			if (!do_pause)
+				return;
+
 		} else {
+			zed_log_msg(LOG_INFO, "normal pid %d", pid);
+
 			memset(&node, 0, sizeof (node));
 			node.pid = pid;
 			pnode = avl_find(&_launched_processes, &node, NULL);
@@ -278,6 +273,25 @@ _reap_children(void *arg)
 		}
 	}
 
+}
+
+static void *
+_reap_children(void *arg)
+{
+	(void) arg;
+	struct sigaction sa = {};
+
+	(void) sigfillset(&sa.sa_mask);
+	(void) sigdelset(&sa.sa_mask, SIGCHLD);
+	(void) pthread_sigmask(SIG_SETMASK, &sa.sa_mask, NULL);
+
+	(void) sigemptyset(&sa.sa_mask);
+	sa.sa_handler = _nop;
+	sa.sa_flags = SA_NOCLDSTOP;
+	(void) sigaction(SIGCHLD, &sa, NULL);
+
+	wait_for_children(B_TRUE, B_FALSE);
+
 	return (NULL);
 }
 
@@ -306,6 +320,45 @@ zed_exec_fini(void)
 	_reap_children_tid = (pthread_t)-1;
 }
 
+/*
+ * Check if the zedlet name indicates if it is a synchronous zedlet
+ *
+ * Synchronous zedlets have a "-sync-" immediately following the event name in
+ * their zedlet filename, like:
+ *
+ * EVENT_NAME-sync-ZEDLETNAME.sh
+ *
+ * For example, if you wanted a synchronous statechange script:
+ *
+ * statechange-sync-myzedlet.sh
+ *
+ * Synchronous zedlets are guaranteed to be the only zedlet running.  No other
+ * zedlets may run in parallel with a synchronous zedlet.  A synchronously
+ * zedlet will wait for all previously spawned zedlets to finish before running.
+ * Users should be careful to only use synchronous zedlets when needed, since
+ * they decrease parallelism.
+ */
+static boolean_t
+zedlet_is_sync(const char *zedlet, const char *event)
+{
+	const char *sync_str = "-sync-";
+	size_t sync_str_len;
+	size_t zedlet_len;
+	size_t event_len;
+
+	sync_str_len = strlen(sync_str);
+	zedlet_len = strlen(zedlet);
+	event_len = strlen(event);
+
+	if (event_len + sync_str_len >= zedlet_len)
+		return (B_FALSE);
+
+	if (strncmp(&zedlet[event_len], sync_str, sync_str_len) == 0)
+		return (B_TRUE);
+
+	return (B_FALSE);
+}
+
 /*
  * Process the event [eid] by synchronously invoking all zedlets with a
  * matching class prefix.
@@ -368,9 +421,28 @@ zed_exec_process(uint64_t eid, const char *class, const char *subclass,
 	    z = zed_strings_next(zcp->zedlets)) {
 		for (csp = class_strings; *csp; csp++) {
 			n = strlen(*csp);
-			if ((strncmp(z, *csp, n) == 0) && !isalpha(z[n]))
+			if ((strncmp(z, *csp, n) == 0) && !isalpha(z[n])) {
+				boolean_t is_sync = zedlet_is_sync(z, *csp);
+
+				if (is_sync) {
+					/*
+					 * Wait for previous zedlets to
+					 * finish
+					 */
+					wait_for_children(B_FALSE, B_TRUE);
+				}
+
 				_zed_exec_fork_child(eid, zcp->zedlet_dir,
 				    z, e, zcp->zevent_fd, zcp->do_foreground);
+
+				if (is_sync) {
+					/*
+					 * Wait for sync zedlet we just launched
+					 * to finish.
+					 */
+					wait_for_children(B_FALSE, B_TRUE);
+				}
+			}
 		}
 	}
 	free(e);

cmd/zed/zed.d/pool_import-led.sh renamed to cmd/zed/zed.d/pool_import-sync-led.sh

@@ -158,6 +158,8 @@ Multiple ZEDLETs may be invoked for a given zevent.
 ZEDLETs are executables invoked by the ZED in response to a given zevent.
 They should be written under the presumption they can be invoked concurrently,
 and they should use appropriate locking to access any shared resources.
+The one exception to this are "synchronous zedlets", which are described later
+in this page.
 Common variables used by ZEDLETs can be stored in the default rc file which
 is sourced by scripts; these variables should be prefixed with
 .Sy ZED_ .
@@ -233,6 +235,36 @@ and
 .Sy ZPOOL .
 These variables may be overridden in the rc file.
 .
+.Sh Synchronous ZEDLETS
+ZED's normal behavior is to spawn off zedlets in parallel and ignore their
+completion order.
+This means that ZED can potentially
+have zedlets for event ID number 2 starting before zedlets for event ID number
+1 have finished.
+Most of the time this is fine, and it actually helps when the system is getting
+hammered with hundreds of events.
+.Pp
+However, there are times when you want your zedlets to be executed in sequence
+with the event ID.
+That is where synchronous zedlets come in.
+.Pp
+ZED will wait for all previously spawned zedlets to finish before running
+a synchronous zedlet.
+Synchronous zedlets are guaranteed to be the only
+zedlet running.
+No other zedlets may run in parallel with a synchronous zedlet.
+Users should be careful to only use synchronous zedlets when needed, since
+they decrease parallelism.
+.Pp
+To make a zedlet synchronous, simply add a "-sync-" immediately following the
+event name in the zedlet's file name:
+.Pp
+.Sy EVENT_NAME-sync-ZEDLETNAME.sh
+.Pp
+For example, if you wanted a synchronous statechange script:
+.Pp
+.Sy statechange-sync-myzedlet.sh
+.
 .Sh FILES
 .Bl -tag -width "-c"
 .It Pa @sysconfdir@/zfs/zed.d

cmd/zed/zed.d/statechange-led.sh renamed to cmd/zed/zed.d/statechange-sync-led.sh

@@ -109,7 +109,8 @@ tags = ['functional', 'direct']
 [tests/functional/events:Linux]
 tests = ['events_001_pos', 'events_002_pos', 'zed_rc_filter', 'zed_fd_spill',
     'zed_cksum_reported', 'zed_cksum_config', 'zed_io_config',
-    'zed_slow_io', 'zed_slow_io_many_vdevs', 'zed_diagnose_multiple']
+    'zed_slow_io', 'zed_slow_io_many_vdevs', 'zed_diagnose_multiple',
+    'zed_synchronous_zedlet']
 tags = ['functional', 'events']
 
 [tests/functional/fadvise:Linux]

cmd/zed/zed.d/statechange-slot_off.sh renamed to cmd/zed/zed.d/statechange-sync-slot_off.sh

@@ -1518,6 +1518,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/events/zed_rc_filter.ksh \
 	functional/events/zed_slow_io.ksh \
 	functional/events/zed_slow_io_many_vdevs.ksh \
+	functional/events/zed_synchronous_zedlet.ksh \
 	functional/exec/cleanup.ksh \
 	functional/exec/exec_001_pos.ksh \
 	functional/exec/exec_002_neg.ksh \