mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Tom Zanussi <tom.zanussi@linux.intel.com>
To: rostedt@goodmis.org
Cc: tglx@linutronix.de, mhiramat@kernel.org, namhyung@kernel.org,
	vedang.patel@intel.com, bigeasy@linutronix.de,
	joel.opensrc@gmail.com, joelaf@google.com,
	mathieu.desnoyers@efficios.com, baohong.liu@intel.com,
	rajvi.jingar@intel.com, linux-kernel@vger.kernel.org,
	linux-rt-users@vger.kernel.org,
	Tom Zanussi <tom.zanussi@linux.intel.com>
Subject: [PATCH v3 28/33] tracing: Add 'last error' error facility for hist triggers
Date: Fri, 22 Sep 2017 15:00:08 -0500	[thread overview]
Message-ID: <7350044ff7277922b339df074d0a104c7955594a.1506105131.git.tom.zanussi@linux.intel.com> (raw)
In-Reply-To: <cover.1506105131.git.tom.zanussi@linux.intel.com>
In-Reply-To: <cover.1506105131.git.tom.zanussi@linux.intel.com>

With the addition of variables and actions, it's become necessary to
provide more detailed error information to users about syntax errors.

Add a 'last error' facility accessible via the erroring event's 'hist'
file.  Reading the hist file after an error will display more detailed
information about what went wrong, if information is available.  This
extended error information will be available until the next hist
trigger command for that event.

  # echo xxx > /sys/kernel/debug/tracing/events/sched/sched_wakeup/trigger
  echo: write error: Invalid argument

  # cat /sys/kernel/debug/tracing/events/sched/sched_wakeup/hist

  ERROR: Couldn't yyy: zzz
  Last command: xxx

Also add specific error messages for variable and action errors.

Signed-off-by: Tom Zanussi <tom.zanussi@linux.intel.com>
---
 Documentation/trace/events.txt   |  19 +++++
 kernel/trace/trace_events_hist.c | 162 +++++++++++++++++++++++++++++++++++----
 2 files changed, 168 insertions(+), 13 deletions(-)

diff --git a/Documentation/trace/events.txt b/Documentation/trace/events.txt
index f36fa00..ad4660e 100644
--- a/Documentation/trace/events.txt
+++ b/Documentation/trace/events.txt
@@ -686,6 +686,25 @@ The following commands are supported:
 				 interpreted as microseconds.
     cpu                    int - the cpu on which the event occurred.
 
+  Extended error information
+  --------------------------
+
+  For some error conditions encountered when invoking a hist trigger
+  command, extended error information is available via the
+  corresponding event's 'hist' file.  Reading the hist file after an
+  error will display more detailed information about what went wrong,
+  if information is available.  This extended error information will
+  be available until the next hist trigger command for that event.
+
+  If available for a given error condition, the extended error
+  information and usage takes the following form:
+
+    # echo xxx > /sys/kernel/debug/tracing/events/sched/sched_wakeup/trigger
+    echo: write error: Invalid argument
+
+    # cat /sys/kernel/debug/tracing/events/sched/sched_wakeup/hist
+    ERROR: Couldn't yyy: zzz
+      Last command: xxx
 
 6.2 'hist' trigger examples
 ---------------------------
diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c
index fad93ee..491fe74 100644
--- a/kernel/trace/trace_events_hist.c
+++ b/kernel/trace/trace_events_hist.c
@@ -351,6 +351,65 @@ struct action_data {
 	};
 };
 
+
+static char last_hist_cmd[MAX_FILTER_STR_VAL];
+static char hist_err_str[MAX_FILTER_STR_VAL];
+
+static void last_cmd_set(char *str)
+{
+	if (!str)
+		return;
+
+	strncpy(last_hist_cmd, str, MAX_FILTER_STR_VAL - 1);
+}
+
+static void hist_err(char *str, char *var)
+{
+	int maxlen = MAX_FILTER_STR_VAL - 1;
+
+	if (!str)
+		return;
+
+	if (strlen(hist_err_str))
+		return;
+
+	if (!var)
+		var = "";
+
+	if (strlen(hist_err_str) + strlen(str) + strlen(var) > maxlen)
+		return;
+
+	strcat(hist_err_str, str);
+	strcat(hist_err_str, var);
+}
+
+static void hist_err_event(char *str, char *system, char *event, char *var)
+{
+	char err[MAX_FILTER_STR_VAL];
+
+	if (system && var)
+		snprintf(err, MAX_FILTER_STR_VAL, "%s.%s.%s", system, event, var);
+	else if (system)
+		snprintf(err, MAX_FILTER_STR_VAL, "%s.%s", system, event);
+	else
+		strncpy(err, var, MAX_FILTER_STR_VAL);
+
+	hist_err(str, err);
+}
+
+static void hist_err_clear(void)
+{
+	hist_err_str[0] = '\0';
+}
+
+static bool have_hist_err(void)
+{
+	if (strlen(hist_err_str))
+		return true;
+
+	return false;
+}
+
 static LIST_HEAD(synth_event_list);
 static DEFINE_MUTEX(synth_event_mutex);
 
@@ -2132,9 +2191,18 @@ static struct hist_field *create_var_ref(struct hist_field *var_field)
 	return ref_field;
 }
 
+static bool is_common_field(char *var_name)
+{
+	if (strncmp(var_name, "$common_timestamp", strlen("$common_timestamp")) == 0)
+		return true;
+
+	return false;
+}
+
 static bool is_var_ref(char *var_name)
 {
-	if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
+	if (!var_name || strlen(var_name) < 2 || var_name[0] != '$' ||
+	    is_common_field(var_name))
 		return false;
 
 	return true;
@@ -2186,6 +2254,10 @@ static struct hist_field *parse_var_ref(struct trace_array *tr,
 	if (var_field)
 		ref_field = create_var_ref(var_field);
 
+	if (!ref_field)
+		hist_err_event("Couldn't find variable: $",
+			       system, event_name, var_name);
+
 	return ref_field;
 }
 
@@ -2431,8 +2503,10 @@ static int check_expr_operands(struct hist_field *operand1,
 	}
 
 	if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
-	    (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS))
+	    (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
+		hist_err("Timestamp units in expression don't match", NULL);
 		return -EINVAL;
+	}
 
 	return 0;
 }
@@ -2632,19 +2706,27 @@ static struct trace_event_file *event_file(struct trace_array *tr,
 	char *cmd;
 	int ret;
 
-	if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX)
+	if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
+		hist_err_event("onmatch: Too many field variables defined: ",
+			       system, event_name, field_name);
 		return ERR_PTR(-EINVAL);
+	}
 
 	file = event_file(tr, system, event_name);
 
 	if (IS_ERR(file)) {
+		hist_err_event("onmatch: Event file not found: ",
+			       system, event_name, field_name);
 		ret = PTR_ERR(file);
 		return ERR_PTR(ret);
 	}
 
 	hist_data = find_compatible_hist(target_hist_data, file);
-	if (!hist_data)
+	if (!hist_data) {
+		hist_err_event("onmatch: Matching event histogram not found: ",
+			       system, event_name, field_name);
 		return ERR_PTR(-EINVAL);
+	}
 
 	var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
 	if (!var_hist)
@@ -2692,6 +2774,8 @@ static struct trace_event_file *event_file(struct trace_array *tr,
 		kfree(cmd);
 		kfree(var_hist->cmd);
 		kfree(var_hist);
+		hist_err_event("onmatch: Couldn't create histogram for field: ",
+			       system, event_name, field_name);
 		return ERR_PTR(ret);
 	}
 
@@ -2703,6 +2787,8 @@ static struct trace_event_file *event_file(struct trace_array *tr,
 		kfree(cmd);
 		kfree(var_hist->cmd);
 		kfree(var_hist);
+		hist_err_event("onmatch: Couldn't find synthetic variable: ",
+			       system, event_name, field_name);
 		return ERR_PTR(-EINVAL);
 	}
 
@@ -2839,18 +2925,21 @@ static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
 	int ret = 0;
 
 	if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
+		hist_err("Too many field variables defined: ", field_name);
 		ret = -EINVAL;
 		goto err;
 	}
 
 	val = parse_atom(hist_data, file, field_name, &flags, NULL);
 	if (IS_ERR(val)) {
+		hist_err("Couldn't parse field variable: ", field_name);
 		ret = PTR_ERR(val);
 		goto err;
 	}
 
 	var = create_var(hist_data, file, field_name, val->size, val->type);
 	if (IS_ERR(var)) {
+		hist_err("Couldn't create or find variable: ", field_name);
 		kfree(val);
 		ret = PTR_ERR(var);
 		goto err;
@@ -2975,14 +3064,18 @@ static int onmax_create(struct hist_trigger_data *hist_data,
 	int ret = 0;
 
 	onmax_var_str = data->onmax.var_str;
-	if (onmax_var_str[0] != '$')
+	if (onmax_var_str[0] != '$') {
+		hist_err("onmax: For onmax(x), x must be a variable: ", onmax_var_str);
 		return -EINVAL;
+	}
 	onmax_var_str++;
 
 	event_name = trace_event_name(call);
 	var_field = find_target_event_var(hist_data, NULL, NULL, onmax_var_str);
-	if (!var_field)
+	if (!var_field) {
+		hist_err("onmax: Couldn't find onmax variable: ", onmax_var_str);
 		return -EINVAL;
+	}
 
 	flags = HIST_FIELD_FL_VAR_REF;
 	ref_field = create_hist_field(hist_data, NULL, flags, NULL);
@@ -3002,6 +3095,7 @@ static int onmax_create(struct hist_trigger_data *hist_data,
 	data->onmax.max_var_ref_idx = var_ref_idx;
 	max_var = create_var(hist_data, file, "max", sizeof(u64), "u64");
 	if (IS_ERR(max_var)) {
+		hist_err("onmax: Couldn't create onmax variable: ", "max");
 		ret = PTR_ERR(max_var);
 		goto out;
 	}
@@ -3016,6 +3110,7 @@ static int onmax_create(struct hist_trigger_data *hist_data,
 
 		field_var = create_target_field_var(hist_data, NULL, NULL, param);
 		if (IS_ERR(field_var)) {
+			hist_err("onmax: Couldn't create field variable: ", param);
 			ret = PTR_ERR(field_var);
 			kfree(param);
 			goto out;
@@ -3050,6 +3145,7 @@ static int parse_action_params(char *params, struct action_data *data)
 
 		param = strstrip(param);
 		if (strlen(param) < 2) {
+			hist_err("Invalid action param: ", param);
 			ret = -EINVAL;
 			goto out;
 		}
@@ -3223,6 +3319,9 @@ static int check_synth_field(struct synth_event *event,
 		hist_field = find_event_var(tr, system, event, var);
 	}
 
+	if (!hist_field)
+		hist_err_event("onmatch: Couldn't find onmatch param: $", system, event, var);
+
 	return hist_field;
 }
 
@@ -3273,6 +3372,7 @@ static int onmatch_create(struct hist_trigger_data *hist_data,
 	mutex_lock(&synth_event_mutex);
 	event = find_synth_event(data->onmatch.synth_event_name);
 	if (!event) {
+		hist_err("onmatch: Couldn't find synthetic event: ", data->onmatch.synth_event_name);
 		mutex_unlock(&synth_event_mutex);
 		return -EINVAL;
 	}
@@ -3331,12 +3431,15 @@ static int onmatch_create(struct hist_trigger_data *hist_data,
 			continue;
 		}
 
+		hist_err_event("onmatch: Param type doesn't match synthetic event field type: ",
+			       system, event_name, param);
 		kfree(p);
 		ret = -EINVAL;
 		goto out;
 	}
 
 	if (field_pos != event->n_fields) {
+		hist_err("onmatch: Param count doesn't match synthetic event field count: ", event->name);
 		ret = -EINVAL;
 		goto out;
 	}
@@ -3362,15 +3465,22 @@ static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
 		return ERR_PTR(-ENOMEM);
 
 	match_event = strsep(&str, ")");
-	if (!match_event || !str)
+	if (!match_event || !str) {
+		hist_err("onmatch: Missing closing paren: ", match_event);
 		goto free;
+	}
 
 	match_event_system = strsep(&match_event, ".");
-	if (!match_event)
+	if (!match_event) {
+		hist_err("onmatch: Missing subsystem for match event: ", match_event_system);
 		goto free;
+	}
 
-	if (IS_ERR(event_file(tr, match_event_system, match_event)))
+	if (IS_ERR(event_file(tr, match_event_system, match_event))) {
+		hist_err_event("onmatch: Invalid subsystem or event name: ",
+			       match_event_system, match_event, NULL);
 		goto free;
+	}
 
 	data->onmatch.match_event = kstrdup(match_event, GFP_KERNEL);
 	if (!data->onmatch.match_event) {
@@ -3385,12 +3495,16 @@ static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
 	}
 
 	strsep(&str, ".");
-	if (!str)
+	if (!str) {
+		hist_err("onmatch: Missing . after onmatch(): ", str);
 		goto free;
+	}
 
 	synth_event_name = strsep(&str, "(");
-	if (!synth_event_name || !str)
+	if (!synth_event_name || !str) {
+		hist_err("onmatch: Missing opening paramlist paren: ", synth_event_name);
 		goto free;
+	}
 
 	data->onmatch.synth_event_name = kstrdup(synth_event_name, GFP_KERNEL);
 	if (!data->onmatch.synth_event_name) {
@@ -3399,8 +3513,10 @@ static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
 	}
 
 	params = strsep(&str, ")");
-	if (!params || !str || (str && strlen(str)))
+	if (!params || !str || (str && strlen(str))) {
+		hist_err("onmatch: Missing closing paramlist paren: ", params);
 		goto free;
+	}
 
 	ret = parse_action_params(params, data);
 	if (ret)
@@ -3480,12 +3596,14 @@ static int create_var_field(struct hist_trigger_data *hist_data,
 		return -EINVAL;
 
 	if (find_var(file, var_name) && !hist_data->remove) {
+		hist_err("Variable already defined: ", var_name);
 		return -EINVAL;
 	}
 
 	flags |= HIST_FIELD_FL_VAR;
 	hist_data->n_vars++;
 	if (hist_data->n_vars > TRACING_MAP_VARS_MAX) {
+		hist_err("Too many variables defined: ", var_name);
 		return -EINVAL;
 	}
 
@@ -3676,6 +3794,7 @@ static int parse_var_defs(struct hist_trigger_data *hist_data)
 
 			var_name = strsep(&field_str, "=");
 			if (!var_name || !field_str) {
+				hist_err("Malformed assignment: ", var_name);
 				ret = -EINVAL;
 				goto free;
 			}
@@ -4402,6 +4521,11 @@ static int hist_show(struct seq_file *m, void *v)
 			hist_trigger_show(m, data, n++);
 	}
 
+	if (have_hist_err()) {
+		seq_printf(m, "\nERROR: %s\n", hist_err_str);
+		seq_printf(m, "  Last command: %s\n", last_hist_cmd);
+	}
+
  out_unlock:
 	mutex_unlock(&event_mutex);
 
@@ -4748,6 +4872,7 @@ static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
 		if (named_data) {
 			if (!hist_trigger_match(data, named_data, named_data,
 						true)) {
+				hist_err("Named hist trigger doesn't match existing named trigger (includes variables): ", hist_data->attrs->name);
 				ret = -EINVAL;
 				goto out;
 			}
@@ -4767,13 +4892,16 @@ static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
 				test->paused = false;
 			else if (hist_data->attrs->clear)
 				hist_clear(test);
-			else
+			else {
+				hist_err("Hist trigger already exists", NULL);
 				ret = -EEXIST;
+			}
 			goto out;
 		}
 	}
  new:
 	if (hist_data->attrs->cont || hist_data->attrs->clear) {
+		hist_err("Can't clear or continue a nonexistent hist trigger", NULL);
 		ret = -ENOENT;
 		goto out;
 	}
@@ -4941,6 +5069,11 @@ static int event_hist_trigger_func(struct event_command *cmd_ops,
 	char *trigger, *p;
 	int ret = 0;
 
+	if (glob && strlen(glob)) {
+		last_cmd_set(param);
+		hist_err_clear();
+	}
+
 	if (!param)
 		return -EINVAL;
 
@@ -5062,6 +5195,9 @@ static int event_hist_trigger_func(struct event_command *cmd_ops,
 	/* Just return zero, not the number of registered triggers */
 	ret = 0;
  out:
+	if (ret == 0)
+		hist_err_clear();
+
 	return ret;
  out_unreg:
 	cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
-- 
1.9.3

  parent reply	other threads:[~2017-09-22 20:05 UTC|newest]

Thread overview: 61+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2017-09-22 19:59 [PATCH v3 00/33] tracing: Inter-event (e.g. latency) support Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 01/33] tracing: Add support to detect and avoid duplicates Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 02/33] tracing: Remove code which merges duplicates Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 03/33] ring-buffer: Add interface for setting absolute time stamps Tom Zanussi
2017-10-04 17:14   ` Steven Rostedt
2017-10-05 11:34     ` Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 04/33] ring-buffer: Redefine the unimplemented RINGBUF_TIME_TIME_STAMP Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 05/33] tracing: Give event triggers access to ring_buffer_event Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 06/33] tracing: Add ring buffer event param to hist field functions Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 07/33] tracing: Break out hist trigger assignment parsing Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 08/33] tracing: Add hist trigger timestamp support Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 09/33] tracing: Add per-element variable support to tracing_map Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 10/33] tracing: Add hist_data member to hist_field Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 11/33] tracing: Add usecs modifier for hist trigger timestamps Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 12/33] tracing: Add variable support to hist triggers Tom Zanussi
2017-10-09  3:27   ` Namhyung Kim
2017-10-11 13:33     ` Tom Zanussi
2017-10-11 23:23       ` Namhyung Kim
2017-10-12 15:51         ` Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 13/33] tracing: Account for variables in named trigger compatibility Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 14/33] tracing: Move get_hist_field_flags() Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 15/33] tracing: Add simple expression support to hist triggers Tom Zanussi
2017-10-09 12:54   ` Namhyung Kim
2017-10-11 14:08     ` Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 16/33] tracing: Generalize per-element hist trigger data Tom Zanussi
2017-10-04 17:34   ` Steven Rostedt
2017-10-04 19:22     ` Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 17/33] tracing: Pass tracing_map_elt to hist_field accessor functions Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 18/33] tracing: Add hist_field 'type' field Tom Zanussi
2017-09-22 19:59 ` [PATCH v3 19/33] tracing: Add variable reference handling to hist triggers Tom Zanussi
2017-10-09 12:46   ` Namhyung Kim
2017-10-11 14:07     ` Tom Zanussi
2017-09-22 20:00 ` [PATCH v3 20/33] tracing: Add hist trigger action hook Tom Zanussi
2017-09-22 20:00 ` [PATCH v3 21/33] tracing: Add support for 'synthetic' events Tom Zanussi
2017-10-04 18:08   ` Steven Rostedt
2017-10-04 19:50     ` Tom Zanussi
2017-10-12  0:53       ` Namhyung Kim
2017-09-22 20:00 ` [PATCH v3 22/33] tracing: Add support for 'field variables' Tom Zanussi
2017-10-12  2:52   ` Namhyung Kim
2017-10-12 21:35     ` Tom Zanussi
2017-09-22 20:00 ` [PATCH v3 23/33] tracing: Add 'onmatch' hist trigger action support Tom Zanussi
2017-09-22 20:00 ` [PATCH v3 24/33] tracing: Add 'onmax' " Tom Zanussi
2017-09-22 20:00 ` [PATCH v3 25/33] tracing: Allow whitespace to surround hist trigger filter Tom Zanussi
2017-10-04 18:11   ` Steven Rostedt
2017-10-04 19:05     ` Tom Zanussi
2017-10-04 19:28       ` Steven Rostedt
2017-10-18  3:00       ` Namhyung Kim
2017-10-18 14:11         ` Tom Zanussi
2017-09-22 20:00 ` [PATCH v3 26/33] tracing: Add cpu field for hist triggers Tom Zanussi
2017-10-04 18:12   ` Steven Rostedt
2017-10-04 19:21     ` Tom Zanussi
2017-10-04 19:30       ` Steven Rostedt
2017-09-22 20:00 ` [PATCH v3 27/33] tracing: Add hist trigger support for variable reference aliases Tom Zanussi
2017-09-22 20:00 ` Tom Zanussi [this message]
2017-09-22 20:00 ` [PATCH v3 29/33] tracing: Add inter-event hist trigger Documentation Tom Zanussi
2017-10-18  1:36   ` Steven Rostedt
2017-10-18 14:07     ` Tom Zanussi
2017-09-22 20:00 ` [PATCH v3 30/33] tracing: Make tracing_set_clock() non-static Tom Zanussi
2017-09-22 20:00 ` [PATCH v3 31/33] tracing: Add a clock attribute for hist triggers Tom Zanussi
2017-09-22 20:00 ` [PATCH v3 32/33] tracing: Increase trace_recursive_lock() limit for synthetic events Tom Zanussi
2017-09-22 20:00 ` [PATCH v3 33/33] tracing: Add inter-event blurb to HIST_TRIGGERS config option Tom Zanussi

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=7350044ff7277922b339df074d0a104c7955594a.1506105131.git.tom.zanussi@linux.intel.com \
    --to=tom.zanussi@linux.intel.com \
    --cc=baohong.liu@intel.com \
    --cc=bigeasy@linutronix.de \
    --cc=joel.opensrc@gmail.com \
    --cc=joelaf@google.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-rt-users@vger.kernel.org \
    --cc=mathieu.desnoyers@efficios.com \
    --cc=mhiramat@kernel.org \
    --cc=namhyung@kernel.org \
    --cc=rajvi.jingar@intel.com \
    --cc=rostedt@goodmis.org \
    --cc=tglx@linutronix.de \
    --cc=vedang.patel@intel.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®