[php-src] master: perf: move AG and SCNG into native __thread storage

henderkes via Arnaud Le Blanc <[email protected]>
Newsgroups gmane.comp.php.cvs.general
Message-ID <[email protected]> 
Author: henderkes (henderkes)
Committer: Arnaud Le Blanc (arnaud-lb)
Date: 2026-07-13T14:39:58+02:00

Commit: https://github.com/php/php-src/commit/0b0284eb16dba22bcb6a56d8da9604b8ec4228cc
Raw diff: https://github.com/php/php-src/commit/0b0284eb16dba22bcb6a56d8da9604b8ec4228cc.diff

perf: move AG and SCNG into native __thread storage

Replay of GH-22231

Closes GH-22595

Changed paths:
  M  NEWS
  M  TSRM/TSRM.c
  M  TSRM/TSRM.h
  M  UPGRADING.INTERNALS
  M  Zend/zend.c
  M  Zend/zend_alloc.c
  M  Zend/zend_alloc.h
  M  Zend/zend_globals.h
  M  Zend/zend_globals_macros.h
  M  Zend/zend_language_scanner.l
  M  main/main.c


Diff:

diff --git a/NEWS b/NEWS
index 35eb80d675ca..8c29466dcc64 100644
--- a/NEWS
+++ b/NEWS
@@ -12,6 +12,7 @@ PHP                                                                        NEWS
   . Lock unmodified readonly properties for modification after clone-with.
     (NickSdot)
   . abort() instead of exit() on hard OOM. (realFlowControl)
+  . perf: ZTS: move AG and SCNG into native __thread storage. (henderkes)
 
 - Calendar:
   . Fixed bug GH-22602 (gregoriantojd() and juliantojd() integer overflow with
diff --git a/TSRM/TSRM.c b/TSRM/TSRM.c
index 4222e88755d6..a5032e456aae 100644
--- a/TSRM/TSRM.c
+++ b/TSRM/TSRM.c
@@ -41,6 +41,8 @@ typedef struct {
 	ts_allocate_ctor ctor;
 	ts_allocate_dtor dtor;
 	ptrdiff_t fast_offset;
+	/* When set, storage comes from __thread memory instead of being allocated by TSRM. */
+	void *(*tls_addr)(void);
 	int done;
 } tsrm_resource_type;
 
@@ -164,14 +166,20 @@ TSRM_API bool tsrm_startup(int expected_threads, int expected_resources, int deb
 
 static void ts_free_resources(tsrm_tls_entry *thread_resources)
 {
+	bool own_thread = thread_resources->thread_id == tsrm_thread_id();
+
 	/* Need to destroy in reverse order to respect dependencies. */
 	for (int i = thread_resources->count - 1; i >= 0; i--) {
 		if (!resource_types_table[i].done) {
+			/* A __thread block of a foreign thread is inaccessible. */
+			if (resource_types_table[i].tls_addr && !own_thread) {
+				continue;
+			}
 			if (resource_types_table[i].dtor) {
 				resource_types_table[i].dtor(thread_resources->storage[i]);
 			}
 
-			if (!resource_types_table[i].fast_offset) {
+			if (!resource_types_table[i].fast_offset && !resource_types_table[i].tls_addr) {
 				free(thread_resources->storage[i]);
 			}
 		}
@@ -180,7 +188,9 @@ static void ts_free_resources(tsrm_tls_entry *thread_resources)
 	free(thread_resources->storage);
 }
 
-/* Shutdown TSRM (call once for the entire process) */
+/* Shutdown TSRM (call once for the entire process). Tears down every thread left
+ * in the table. For resources allocated with ts_allocate_tls_id(), only the dtor
+ * of the calling thread is invoked. */
 TSRM_API void tsrm_shutdown(void)
 {/*{{{*/
 	if (is_thread_shutdown) {
@@ -258,7 +268,10 @@ static void tsrm_update_active_threads(void)
 
 				p->storage = (void *) realloc(p->storage, sizeof(void *)*id_count);
 				for (j=p->count; j<id_count; j++) {
-					if (resource_types_table[j].fast_offset) {
+					if (resource_types_table[j].tls_addr) {
+						TSRM_ASSERT(p->thread_id == tsrm_thread_id());
+						p->storage[j] = resource_types_table[j].tls_addr();
+					} else if (resource_types_table[j].fast_offset) {
 						p->storage[j] = (void *) (((char*)p) + resource_types_table[j].fast_offset);
 					} else {
 						p->storage[j] = (void *) malloc(resource_types_table[j].size);
@@ -303,6 +316,7 @@ TSRM_API ts_rsrc_id ts_allocate_id(ts_rsrc_id *rsrc_id, size_t size, ts_allocate
 	resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].ctor = ctor;
 	resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].dtor = dtor;
 	resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].fast_offset = 0;
+	resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].tls_addr = NULL;
 	resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].done = 0;
 
 	tsrm_update_active_threads();
@@ -381,6 +395,7 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset,
 	resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].ctor = ctor;
 	resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].dtor = dtor;
 	resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].fast_offset = *offset;
+	resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].tls_addr = NULL;
 	resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].done = 0;
 
 	tsrm_update_active_threads();
@@ -390,6 +405,41 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset,
 	return *rsrc_id;
 }
 
+/* allocates a resource id whose per-thread storage is a native __thread block */
+TSRM_API ts_rsrc_id ts_allocate_tls_id(ts_rsrc_id *rsrc_id, void *(*tls_addr)(void), size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor)
+{
+	TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Obtaining a new TLS resource id, %d bytes", size));
+
+	tsrm_mutex_lock(tsmm_mutex);
+
+	*rsrc_id = TSRM_SHUFFLE_RSRC_ID(id_count++);
+
+	if (resource_types_table_size < id_count) {
+		tsrm_resource_type *_tmp;
+		_tmp = (tsrm_resource_type *) realloc(resource_types_table, sizeof(tsrm_resource_type)*id_count);
+		if (!_tmp) {
+			TSRM_ERROR((TSRM_ERROR_LEVEL_ERROR, "Unable to allocate storage for resource"));
+			*rsrc_id = 0;
+			tsrm_mutex_unlock(tsmm_mutex);
+			return 0;
+		}
+		resource_types_table = _tmp;
+		resource_types_table_size = id_count;
+	}
+	resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].size = size;
+	resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].ctor = ctor;
+	resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].dtor = dtor;
+	resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].fast_offset = 0;
+	resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].tls_addr = tls_addr;
+	resource_types_table[TSRM_UNSHUFFLE_RSRC_ID(*rsrc_id)].done = 0;
+
+	tsrm_update_active_threads();
+	tsrm_mutex_unlock(tsmm_mutex);
+
+	TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Successfully allocated new TLS resource id %d", *rsrc_id));
+	return *rsrc_id;
+}
+
 static void set_thread_local_storage_resource_to(tsrm_tls_entry *thread_resource)
 {
 	tsrm_tls_set(thread_resource);
@@ -422,7 +472,9 @@ static void allocate_new_resource(tsrm_tls_entry **thread_resources_ptr, THREAD_
 		if (resource_types_table[i].done) {
 			(*thread_resources_ptr)->storage[i] = NULL;
 		} else {
-			if (resource_types_table[i].fast_offset) {
+			if (resource_types_table[i].tls_addr) {
+				(*thread_resources_ptr)->storage[i] = resource_types_table[i].tls_addr();
+			} else if (resource_types_table[i].fast_offset) {
 				(*thread_resources_ptr)->storage[i] = (void *) (((char*)(*thread_resources_ptr)) + resource_types_table[i].fast_offset);
 			} else {
 				(*thread_resources_ptr)->storage[i] = (void *) malloc(resource_types_table[i].size);
@@ -510,7 +562,9 @@ TSRM_API void *ts_resource_ex(ts_rsrc_id id, THREAD_T *th_id)
 		/* In case that extensions don't use the pointer passed from the dtor, but incorrectly
 		 * use the global pointer, we need to setup the global pointer temporarily here. */
 		set_thread_local_storage_resource_to(thread_resources);
-		/* Free up the old resource from the old thread instance */
+		/* Dead thread with a recycled id: its __thread blocks are gone, and this
+		 * thread's blocks were never constructed, so keep tls dtors from running. */
+		thread_resources->thread_id = 0;
 		ts_free_resources(thread_resources);
 		free((char *) thread_resources - tsrm_reserved_front);
 		/* Allocate a new resource at the same point in the linked list, and relink the next pointer */
@@ -584,7 +638,7 @@ void ts_free_id(ts_rsrc_id id)
 						if (resource_types_table[rsrc_id].dtor) {
 							resource_types_table[rsrc_id].dtor(p->storage[rsrc_id]);
 						}
-						if (!resource_types_table[rsrc_id].fast_offset) {
+						if (!resource_types_table[rsrc_id].fast_offset && !resource_types_table[rsrc_id].tls_addr) {
 							free(p->storage[rsrc_id]);
 						}
 					}
diff --git a/TSRM/TSRM.h b/TSRM/TSRM.h
index 2e8cbddfcda7..752ea1803f9c 100644
--- a/TSRM/TSRM.h
+++ b/TSRM/TSRM.h
@@ -100,6 +100,10 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, siz
 TSRM_API void tsrm_reserve_fast_front(size_t size);
 TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset, ptrdiff_t fixed_offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor);
 
+/* Resource whose per-thread storage is a native __thread block.
+ * Must be called at startup before any other thread exists. */
+TSRM_API ts_rsrc_id ts_allocate_tls_id(ts_rsrc_id *rsrc_id, void *(*tls_addr)(void), size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor);
+
 /* fetches the requested resource for the current thread */
 TSRM_API void *ts_resource_ex(ts_rsrc_id id, THREAD_T *th_id);
 #define ts_resource(id)			ts_resource_ex(id, NULL)
diff --git a/UPGRADING.INTERNALS b/UPGRADING.INTERNALS
index 7775c9316040..462e567f3f36 100644
--- a/UPGRADING.INTERNALS
+++ b/UPGRADING.INTERNALS
@@ -235,3 +235,11 @@ PHP 8.6 INTERNALS UPGRADE NOTES
 ========================
 5. SAPI changes
 ========================
+
+- SAPIs should explicitly release a thread's resources by calling
+  ts_free_thread() before terminating it. tsrm_shutdown() can only release the
+  resources of the calling thread, for resources allocated with
+  ts_allocate_tls_id().
+
+- AG and SCNG are now allocated with ts_allocate_tls_id() and live in native
+  __thread storage on ZTS builds.
diff --git a/Zend/zend.c b/Zend/zend.c
index 9411b92a2018..07692db85196 100644
--- a/Zend/zend.c
+++ b/Zend/zend.c
@@ -54,6 +54,9 @@ ZEND_API int compiler_globals_id;
 ZEND_API int executor_globals_id;
 ZEND_API size_t compiler_globals_offset;
 ZEND_API size_t executor_globals_offset;
+/* ts_allocate_tls_id takes a callback so each thread resolves its own block.
+ * A plain &language_scanner_globals would capture only the registering thread's address. */
+static void *language_scanner_globals_tls_addr(void) { return &language_scanner_globals; }
 static HashTable *global_function_table = NULL;
 static HashTable *global_class_table = NULL;
 static HashTable *global_constants_table = NULL;
@@ -1021,10 +1024,9 @@ void zend_startup(zend_utility_functions *utility_functions) /* {{{ */
 #ifdef ZTS
 	ts_allocate_fast_id_at(&compiler_globals_id, &compiler_globals_offset, ZEND_CG_OFFSET, sizeof(zend_compiler_globals), (ts_allocate_ctor) compiler_globals_ctor, (ts_allocate_dtor) compiler_globals_dtor);
 	ts_allocate_fast_id_at(&executor_globals_id, &executor_globals_offset, ZEND_EG_OFFSET, sizeof(zend_executor_globals), (ts_allocate_ctor) executor_globals_ctor, (ts_allocate_dtor) executor_globals_dtor);
-	ts_allocate_fast_id_at(&language_scanner_globals_id, &language_scanner_globals_offset, ZEND_SCNG_OFFSET, sizeof(zend_php_scanner_globals), (ts_allocate_ctor) php_scanner_globals_ctor, NULL);
+	ts_allocate_tls_id(&language_scanner_globals_id, language_scanner_globals_tls_addr, sizeof(zend_php_scanner_globals), (ts_allocate_ctor) php_scanner_globals_ctor, NULL);
 	ZEND_ASSERT(compiler_globals_offset == ZEND_CG_OFFSET);
 	ZEND_ASSERT(executor_globals_offset == ZEND_EG_OFFSET);
-	ZEND_ASSERT(language_scanner_globals_offset == ZEND_SCNG_OFFSET);
 	ts_allocate_fast_id(&ini_scanner_globals_id, &ini_scanner_globals_offset, sizeof(zend_ini_scanner_globals), (ts_allocate_ctor) ini_scanner_globals_ctor, NULL);
 	compiler_globals = ts_resource(compiler_globals_id);
 	executor_globals = ts_resource(executor_globals_id);
diff --git a/Zend/zend_alloc.c b/Zend/zend_alloc.c
index 5ff2860d8223..fc7bc1f4d9d4 100644
--- a/Zend/zend_alloc.c
+++ b/Zend/zend_alloc.c
@@ -2613,13 +2613,12 @@ typedef struct _zend_alloc_globals {
 
 #ifdef ZTS
 static int alloc_globals_id;
-static size_t alloc_globals_offset;
-# define ZEND_AG_OFFSET (ZEND_SCNG_OFFSET - (ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_alloc_globals)))
-# define AG(v) ZEND_TSRMG_FAST(ZEND_AG_OFFSET, zend_alloc_globals *, v)
+static TSRM_TLS TSRM_TLS_MODEL_ATTR zend_alloc_globals alloc_globals;
+static void *alloc_globals_tls_addr(void) { return &alloc_globals; }
 #else
-# define AG(v) (alloc_globals.v)
 static zend_alloc_globals alloc_globals;
 #endif
+#define AG(v) (alloc_globals.v)
 
 ZEND_API bool is_zend_mm(void)
 {
@@ -3336,8 +3335,7 @@ ZEND_API void start_memory_manager(void)
 #  endif
 #endif
 #ifdef ZTS
-	ts_allocate_fast_id_at(&alloc_globals_id, &alloc_globals_offset, ZEND_AG_OFFSET, sizeof(zend_alloc_globals), (ts_allocate_ctor) alloc_globals_ctor, (ts_allocate_dtor) alloc_globals_dtor);
-	ZEND_ASSERT(alloc_globals_offset == ZEND_AG_OFFSET);
+	ts_allocate_tls_id(&alloc_globals_id, alloc_globals_tls_addr, sizeof(zend_alloc_globals), (ts_allocate_ctor) alloc_globals_ctor, (ts_allocate_dtor) alloc_globals_dtor);
 #else
 	alloc_globals_ctor(&alloc_globals);
 #endif
@@ -3583,9 +3581,3 @@ ZEND_API char * __zend_strdup(const char *s)
 	zend_out_of_memory();
 }
 
-#ifdef ZTS
-size_t zend_mm_globals_size(void)
-{
-	return sizeof(zend_alloc_globals);
-}
-#endif
diff --git a/Zend/zend_alloc.h b/Zend/zend_alloc.h
index ff51c4fe8652..7e3ea031dc72 100644
--- a/Zend/zend_alloc.h
+++ b/Zend/zend_alloc.h
@@ -403,10 +403,6 @@ static void apc_init_heap(void)
 
 */
 
-#ifdef ZTS
-size_t zend_mm_globals_size(void);
-#endif
-
 END_EXTERN_C()
 
 #endif
diff --git a/Zend/zend_globals.h b/Zend/zend_globals.h
index 61499c0cc23d..f78567cfaa74 100644
--- a/Zend/zend_globals.h
+++ b/Zend/zend_globals.h
@@ -330,11 +330,9 @@ struct _zend_executor_globals {
 };
 
 #ifdef ZTS
-/* Compile-time offsets of the hot globals, in a reserved region just before the
- * cache pointer. ZEND_AG_OFFSET is furthest, in zend_alloc.c. */
+/* Compile-time offsets of the hot globals, in a reserved region just before *_tsrm_ls_cache. */
 # define ZEND_CG_OFFSET   (-(ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_compiler_globals)))
 # define ZEND_EG_OFFSET   (ZEND_CG_OFFSET - (ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals)))
-# define ZEND_SCNG_OFFSET (ZEND_EG_OFFSET - (ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_php_scanner_globals)))
 #endif
 
 #define EG_FLAGS_INITIAL				(0)
diff --git a/Zend/zend_globals_macros.h b/Zend/zend_globals_macros.h
index 2d2948e50a86..adb3913ece71 100644
--- a/Zend/zend_globals_macros.h
+++ b/Zend/zend_globals_macros.h
@@ -48,12 +48,20 @@ extern ZEND_API zend_executor_globals executor_globals;
 
 /* Language Scanner */
 #ifdef ZTS
-# define LANG_SCNG(v) ZEND_TSRMG_FAST(ZEND_SCNG_OFFSET, zend_php_scanner_globals *, v)
 extern ZEND_API ts_rsrc_id language_scanner_globals_id;
-extern ZEND_API size_t language_scanner_globals_offset;
+# if defined(ZEND_WIN32) && !defined(LIBZEND_EXPORTS)
+# define LANG_SCNG(v) TSRMG(language_scanner_globals_id, zend_php_scanner_globals *, v)
+# else
+#  ifdef ZEND_WIN32
+extern TSRM_TLS zend_php_scanner_globals language_scanner_globals;
+#  else
+extern ZEND_API TSRM_TLS TSRM_TLS_MODEL_ATTR zend_php_scanner_globals language_scanner_globals;
+#  endif
+#  define LANG_SCNG(v) (language_scanner_globals.v)
+# endif
 #else
-# define LANG_SCNG(v) (language_scanner_globals.v)
 extern ZEND_API zend_php_scanner_globals language_scanner_globals;
+# define LANG_SCNG(v) (language_scanner_globals.v)
 #endif
 
 
diff --git a/Zend/zend_language_scanner.l b/Zend/zend_language_scanner.l
index 07f2d44cb5c6..1d7e41a2a7d3 100644
--- a/Zend/zend_language_scanner.l
+++ b/Zend/zend_language_scanner.l
@@ -84,7 +84,11 @@
 #define SCNG	LANG_SCNG
 #ifdef ZTS
 ZEND_API ts_rsrc_id language_scanner_globals_id;
-ZEND_API size_t language_scanner_globals_offset;
+# ifdef ZEND_WIN32
+TSRM_TLS zend_php_scanner_globals language_scanner_globals;
+# else
+ZEND_API TSRM_TLS TSRM_TLS_MODEL_ATTR zend_php_scanner_globals language_scanner_globals;
+# endif
 #else
 ZEND_API zend_php_scanner_globals language_scanner_globals;
 #endif
diff --git a/main/main.c b/main/main.c
index 48e4a757513b..9e77c99a45df 100644
--- a/main/main.c
+++ b/main/main.c
@@ -2819,13 +2819,11 @@ PHPAPI void php_reserve_tsrm_memory(void)
 	tsrm_reserve(
 		TSRM_ALIGNED_SIZE(sizeof(zend_compiler_globals)) +
 		TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals)) +
-		TSRM_ALIGNED_SIZE(sizeof(zend_php_scanner_globals)) +
 		TSRM_ALIGNED_SIZE(sizeof(zend_ini_scanner_globals)) +
 		TSRM_ALIGNED_SIZE(sizeof(virtual_cwd_globals)) +
 #ifdef ZEND_SIGNALS
 		TSRM_ALIGNED_SIZE(sizeof(zend_signal_globals_t)) +
 #endif
-		TSRM_ALIGNED_SIZE(zend_mm_globals_size()) +
 		TSRM_ALIGNED_SIZE(zend_gc_globals_size()) +
 		TSRM_ALIGNED_SIZE(sizeof(php_core_globals)) +
 		TSRM_ALIGNED_SIZE(sizeof(sapi_globals_struct)) +
@@ -2845,9 +2843,7 @@ PHPAPI bool php_tsrm_startup_ex(int expected_threads)
 	/* Must cover the total size of every ZEND_*_OFFSET global, or the furthest underflows the block. */
 	tsrm_reserve_fast_front(
 		TSRM_ALIGNED_SIZE(sizeof(zend_compiler_globals)) +
-		TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals)) +
-		TSRM_ALIGNED_SIZE(sizeof(zend_php_scanner_globals)) +
-		TSRM_ALIGNED_SIZE(zend_mm_globals_size())); // AG size, exposed through function call
+		TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals)));
 	(void)ts_resource(0);
 	return ret;
 }
lmpx.com only provides a reader for public news (NNTP) servers. It is not affiliated with the servers or forums shown here and is not responsible for the content of articles, which is written by their respective authors.