[PATCH v2] options: check *frames option values against system limit

Commit 81e99397d ("Add support for the stack() action") added code to
retrieve the system limit for stack traces and used that value to set
the 'maxframes' option.

However, a user can set the 'maxframes' option from the command line,
which would allow setting the option to value beyond the system limit.

We now record the system limit in dtrace_hdl_t as well as setting the
'maxframes' option to this value as a default.  Options processing now
ensures that *frames options cannot be set beyond the system limit.

Note: If 'stackframes', 'ustackframes', or 'jstackframes' are set
before 'maxframes' they could be given a value that exceeds 'maxframes'.
The existing immplementation in the code generator already lowers their
value to 'maxframes' as needed.

Signed-off-by: Kris Van Hees <[email protected]>
---
 libdtrace/dt_impl.h                           |  1 +
 libdtrace/dt_open.c                           |  7 +++-
 libdtrace/dt_options.c                        | 38 +++++--------------
 .../options/err.jstackframes-negative.d       | 24 ++++++++++++
 .../options/err.jstackframes-too-high.sh      | 25 ++++++++++++
 .../unittest/options/err.maxframes-negative.d | 24 ++++++++++++
 .../options/err.maxframes-too-high.sh         | 25 ++++++++++++
 .../options/err.stackframes-negative.d        | 24 ++++++++++++
 .../options/err.stackframes-too-high.sh       | 25 ++++++++++++
 .../options/err.ustackframes-negative.d       | 24 ++++++++++++
 .../options/err.ustackframes-too-high.sh      | 25 ++++++++++++
 11 files changed, 211 insertions(+), 31 deletions(-)
 create mode 100644 test/unittest/options/err.jstackframes-negative.d
 create mode 100755 test/unittest/options/err.jstackframes-too-high.sh
 create mode 100644 test/unittest/options/err.maxframes-negative.d
 create mode 100755 test/unittest/options/err.maxframes-too-high.sh
 create mode 100644 test/unittest/options/err.stackframes-negative.d
 create mode 100755 test/unittest/options/err.stackframes-too-high.sh
 create mode 100644 test/unittest/options/err.ustackframes-negative.d
 create mode 100755 test/unittest/options/err.ustackframes-too-high.sh

diff --git a/libdtrace/dt_impl.h b/libdtrace/dt_impl.h
index 4dc4c9a7..723aac8b 100644
--- a/libdtrace/dt_impl.h
+++ b/libdtrace/dt_impl.h
@@ -310,6 +310,7 @@ struct dtrace_hdl {
 	uint_t dt_maxtuplesize;	/* largest tuple across programs */
 	uint_t dt_maxlvaralloc;	/* largest lvar alloc across pcbs */
 	uint_t dt_maxaggdsize;	/* largest aggregation data size */
+	uint64_t dt_maxframes;	/* system limit on stack traces */
 	dt_tstring_t *dt_tstrings; /* temporary string slots */
 	dt_list_t dt_modlist;	/* linked list of dt_module_t's */
 	dt_htab_t *dt_mods;	/* hash table of dt_module_t's */
diff --git a/libdtrace/dt_open.c b/libdtrace/dt_open.c
index ed0b14a1..ba5463a0 100644
--- a/libdtrace/dt_open.c
+++ b/libdtrace/dt_open.c
@@ -764,13 +764,16 @@ dt_vopen(int version, int flags, int *errp,
 	dtp->dt_options[DTRACEOPT_SCRATCHSIZE] = sizeof(uint64_t) + _dtrace_scratchsize;
 
 	/*
-	 * Set the default value of maxframes.
+	 * Set the default value of maxframes.  This is the maximum of stack
+	 * frames that can be requested from the kernel.  The *frames options
+	 * cannot be set to a value that exceeds this limit.
 	 */
 	fd = fopen("/proc/sys/kernel/perf_event_max_stack", "r");
 	assert(fd);
-	if (fscanf(fd, "%lu", &dtp->dt_options[DTRACEOPT_MAXFRAMES]) != 1)
+	if (fscanf(fd, "%lu", &dtp->dt_maxframes) != 1)
 		return set_open_errno(dtp, errp, EDT_READMAXSTACK);
 	fclose(fd);
+	dtp->dt_options[DTRACEOPT_MAXFRAMES] = dtp->dt_maxframes;
 
 	/*
 	 * Set the default pcap capture size.
diff --git a/libdtrace/dt_options.c b/libdtrace/dt_options.c
index 7508f044..97516517 100644
--- a/libdtrace/dt_options.c
+++ b/libdtrace/dt_options.c
@@ -913,40 +913,20 @@ dt_opt_rate(dtrace_hdl_t *dtp, const char *arg, uintptr_t option)
 }
 
 /*
- * When setting the maxframes option, set the option in the dt_options array
- * using dt_opt_runtime() as usual, and then update the definition of the CTF
- * type for "_stack" to be an array of the corresponding size.
- * If any errors occur, reset dt_options[option] to its previous value.
+ * Set a *frames option, ensuring that its value is within the system
+ * constaints.
  */
 static int
-dt_opt_maxframes(dtrace_hdl_t *dtp, const char *arg, uintptr_t option)
+dt_opt_frames(dtrace_hdl_t *dtp, const char *arg, uintptr_t option)
 {
 	dtrace_optval_t val = dtp->dt_options[option];
-	ctf_file_t *fp = DT_STACK_CTFP(dtp);
-	ctf_id_t type = ctf_type_resolve(fp, DT_STACK_TYPE(dtp));
-	ctf_arinfo_t r;
 
 	if (dt_opt_runtime(dtp, arg, option) != 0)
 		return -1; /* dt_errno is set for us */
 
-	if (dtp->dt_options[option] > UINT_MAX) {
-		dtp->dt_options[option] = val;
-		return dt_set_errno(dtp, EOVERFLOW);
-	}
-
-	if (ctf_array_info(fp, type, &r) == CTF_ERR) {
-		dtp->dt_options[option] = val;
-		dtp->dt_ctferr = ctf_errno(fp);
-		return dt_set_errno(dtp, EDT_CTF);
-	}
-
-	r.ctr_nelems = (uint_t)dtp->dt_options[option];
-
-	if (ctf_set_array(fp, type, &r) == CTF_ERR ||
-	    ctf_update(fp) == CTF_ERR) {
+	if (dtp->dt_options[option] > dtp->dt_maxframes) {
 		dtp->dt_options[option] = val;
-		dtp->dt_ctferr = ctf_errno(fp);
-		return dt_set_errno(dtp, EDT_CTF);
+		return dt_set_errno(dtp, EDT_BADOPTVAL);
 	}
 
 	return 0;
@@ -1194,19 +1174,19 @@ static const dt_option_t _dtrace_rtoptions[] = {
 	{ "destructive", dt_opt_runtime, DTRACEOPT_DESTRUCTIVE },
 	{ "dynvarsize", dt_opt_size, DTRACEOPT_DYNVARSIZE },
 	{ "grabanon", dt_opt_runtime, DTRACEOPT_GRABANON },
-	{ "jstackframes", dt_opt_runtime, DTRACEOPT_JSTACKFRAMES },
+	{ "jstackframes", dt_opt_frames, DTRACEOPT_JSTACKFRAMES },
 	{ "jstackstrsize", dt_opt_size, DTRACEOPT_JSTACKSTRSIZE },
 	{ "lockmem", dt_opt_lockmem, DTRACEOPT_LOCKMEM },
-	{ "maxframes", dt_opt_maxframes, DTRACEOPT_MAXFRAMES },
+	{ "maxframes", dt_opt_frames, DTRACEOPT_MAXFRAMES },
 	{ "nusdtprobes", dt_opt_runtime, DTRACEOPT_NUSDTPROBES },
 	{ "nspec", dt_opt_runtime, DTRACEOPT_NSPEC },
 	{ "pcapsize", dt_opt_pcapsize, DTRACEOPT_PCAPSIZE },
 	{ "scratchsize", dt_opt_scratchsize, DTRACEOPT_SCRATCHSIZE },
 	{ "specsize", dt_opt_size, DTRACEOPT_SPECSIZE },
-	{ "stackframes", dt_opt_runtime, DTRACEOPT_STACKFRAMES },
+	{ "stackframes", dt_opt_frames, DTRACEOPT_STACKFRAMES },
 	{ "statusrate", dt_opt_rate, DTRACEOPT_STATUSRATE },
 	{ "strsize", dt_opt_strsize, DTRACEOPT_STRSIZE },
-	{ "ustackframes", dt_opt_runtime, DTRACEOPT_USTACKFRAMES },
+	{ "ustackframes", dt_opt_frames, DTRACEOPT_USTACKFRAMES },
 	{ "noresolve", dt_opt_runtime, DTRACEOPT_NORESOLVE },
 	{ NULL }
 };
diff --git a/test/unittest/options/err.jstackframes-negative.d b/test/unittest/options/err.jstackframes-negative.d
new file mode 100644
index 00000000..0fdcbb58
--- /dev/null
+++ b/test/unittest/options/err.jstackframes-negative.d
@@ -0,0 +1,24 @@
+/*
+ * Oracle Linux DTrace.
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Licensed under the Universal Permissive License v 1.0 as shown at
+ * http://oss.oracle.com/licenses/upl.
+ */
+
+/*
+ * ASSERTION: The -xjstackframes option requires a non-negative value
+ *
+ * SECTION: Options and Tunables/Consumer Options
+ */
+
+/* @@runtest-opts: -xjstackframes=-1 */
+
+BEGIN
+{
+	exit(0);
+}
+
+ERROR
+{
+	exit(1);
+}
diff --git a/test/unittest/options/err.jstackframes-too-high.sh b/test/unittest/options/err.jstackframes-too-high.sh
new file mode 100755
index 00000000..869a3d1f
--- /dev/null
+++ b/test/unittest/options/err.jstackframes-too-high.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Oracle Linux DTrace.
+# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+
+dtrace=$1
+
+#
+# ASSERTION: The -xjstackframes option value must be within the system limit.
+#
+# SECTION: Options and Tunables/Consumer Options
+#
+maxframes=$[ `cat /proc/sys/kernel/perf_event_max_stack` + 1 ]
+
+$dtrace $dt_flags -xjstackframes=$maxframes -qn '
+BEGIN, ERROR
+{
+	trace("ERROR: DTrace should fail to start.");
+	exit(0);
+}'
+
+exit $?
diff --git a/test/unittest/options/err.maxframes-negative.d b/test/unittest/options/err.maxframes-negative.d
new file mode 100644
index 00000000..62e4d21f
--- /dev/null
+++ b/test/unittest/options/err.maxframes-negative.d
@@ -0,0 +1,24 @@
+/*
+ * Oracle Linux DTrace.
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Licensed under the Universal Permissive License v 1.0 as shown at
+ * http://oss.oracle.com/licenses/upl.
+ */
+
+/*
+ * ASSERTION: The -xmaxframes option requires a non-negative value
+ *
+ * SECTION: Options and Tunables/Consumer Options
+ */
+
+/* @@runtest-opts: -xmaxframes=-1 */
+
+BEGIN
+{
+	exit(0);
+}
+
+ERROR
+{
+	exit(1);
+}
diff --git a/test/unittest/options/err.maxframes-too-high.sh b/test/unittest/options/err.maxframes-too-high.sh
new file mode 100755
index 00000000..3b2674cf
--- /dev/null
+++ b/test/unittest/options/err.maxframes-too-high.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Oracle Linux DTrace.
+# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+
+dtrace=$1
+
+#
+# ASSERTION: The -xmaxframes option value must be within the system limit.
+#
+# SECTION: Options and Tunables/Consumer Options
+#
+maxframes=$[ `cat /proc/sys/kernel/perf_event_max_stack` + 1 ]
+
+$dtrace $dt_flags -xmaxframes=$maxframes -qn '
+BEGIN, ERROR
+{
+	trace("ERROR: DTrace should fail to start.");
+	exit(0);
+}'
+
+exit $?
diff --git a/test/unittest/options/err.stackframes-negative.d b/test/unittest/options/err.stackframes-negative.d
new file mode 100644
index 00000000..e6c276d7
--- /dev/null
+++ b/test/unittest/options/err.stackframes-negative.d
@@ -0,0 +1,24 @@
+/*
+ * Oracle Linux DTrace.
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Licensed under the Universal Permissive License v 1.0 as shown at
+ * http://oss.oracle.com/licenses/upl.
+ */
+
+/*
+ * ASSERTION: The -xstackframes option requires a non-negative value
+ *
+ * SECTION: Options and Tunables/Consumer Options
+ */
+
+/* @@runtest-opts: -xstackframes=-1 */
+
+BEGIN
+{
+	exit(0);
+}
+
+ERROR
+{
+	exit(1);
+}
diff --git a/test/unittest/options/err.stackframes-too-high.sh b/test/unittest/options/err.stackframes-too-high.sh
new file mode 100755
index 00000000..0123f92b
--- /dev/null
+++ b/test/unittest/options/err.stackframes-too-high.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Oracle Linux DTrace.
+# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+
+dtrace=$1
+
+#
+# ASSERTION: The -xstackframes option value must be within the system limit.
+#
+# SECTION: Options and Tunables/Consumer Options
+#
+maxframes=$[ `cat /proc/sys/kernel/perf_event_max_stack` + 1 ]
+
+$dtrace $dt_flags -xstackframes=$maxframes -qn '
+BEGIN, ERROR
+{
+	trace("ERROR: DTrace should fail to start.");
+	exit(0);
+}'
+
+exit $?
diff --git a/test/unittest/options/err.ustackframes-negative.d b/test/unittest/options/err.ustackframes-negative.d
new file mode 100644
index 00000000..ede20425
--- /dev/null
+++ b/test/unittest/options/err.ustackframes-negative.d
@@ -0,0 +1,24 @@
+/*
+ * Oracle Linux DTrace.
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Licensed under the Universal Permissive License v 1.0 as shown at
+ * http://oss.oracle.com/licenses/upl.
+ */
+
+/*
+ * ASSERTION: The -xustackframes option requires a non-negative value
+ *
+ * SECTION: Options and Tunables/Consumer Options
+ */
+
+/* @@runtest-opts: -xustackframes=-1 */
+
+BEGIN
+{
+	exit(0);
+}
+
+ERROR
+{
+	exit(1);
+}
diff --git a/test/unittest/options/err.ustackframes-too-high.sh b/test/unittest/options/err.ustackframes-too-high.sh
new file mode 100755
index 00000000..fc362fff
--- /dev/null
+++ b/test/unittest/options/err.ustackframes-too-high.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Oracle Linux DTrace.
+# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+
+dtrace=$1
+
+#
+# ASSERTION: The -xustackframes option value must be within the system limit.
+#
+# SECTION: Options and Tunables/Consumer Options
+#
+maxframes=$[ `cat /proc/sys/kernel/perf_event_max_stack` + 1 ]
+
+$dtrace $dt_flags -xustackframes=$maxframes -qn '
+BEGIN, ERROR
+{
+	trace("ERROR: DTrace should fail to start.");
+	exit(0);
+}'
+
+exit $?
-- 
2.43.5