[DOC-CVS] [doc-base] master: Rewrite file-entities to debug entity collisions (#313)
[email protected] (alfsb via GitHub) Thu, 16 Jul 2026 15:20:33 +0000
| Newsgroups | php.doc.cvs |
|---|---|
| Message-ID | <[email protected]> |
Author: alfsb (alfsb)
Committer: GitHub (web-flow)
Pusher: alfsb
Date: 2026-07-16T12:20:30-03:00
Commit: https://github.com/php/doc-base/commit/6b0bfe23bf7c5f639d34e09353ca9c442534d8b6
Raw diff: https://github.com/php/doc-base/commit/6b0bfe23bf7c5f639d34e09353ca9c442534d8b6.diff
Rewrite file-entities to debug entity collisions (#313)
Changed paths:
M scripts/file-entities.php
Diff:
diff --git a/scripts/file-entities.php b/scripts/file-entities.php
index b6b277d86b..62f96f454d 100644
--- a/scripts/file-entities.php
+++ b/scripts/file-entities.php
@@ -17,37 +17,41 @@
# Description
-This script creates various "file entities", that is, DTD entities that
-point to files and file listings, named and composed of:
+This script creates various "file entities" files, that is, DTD entities
+that include files directly, and some "dir entities", that includes all
+XML files from a directory. The historical naming schema is:
-- dir.dir.file : pulls in a dir/dir/file.xml
-- dir.dif.entities.dir : pulls in XML files from dir/dir/dir/*.xml
+- dir.dir.file : includes one file from dir/dir/file.xml
+- dir.dir.entities.dir : includes all files from dir/dir/dir/*.xml
-In the original file-entities.php.in, the files are created at:
-
-- doc-base/entities/file-entities.ent
-- doc-en/reference/entities.*.xml
-
-In new idempotent mode, files are created at:
+The files are created at:
- doc-base/temp/file-entites.ent
- doc-base/temp/file-entites/dir.dir.ent
The file entity for directories (file listings) are keep as individual
-files instead to avoid these libxml errors, in some OS/versions:
+files, to avoid these libxml errors, in some OS/versions:
- Detected an entity reference loop [1]
- Maximum entity amplification factor exceeded [2]
See LIBXML_LIMITS_HACK below. This workaround creates about a thousand
-files per running, that slowsdows even more the manual building on HDD
-systems.
+files per running, that slows down even more the building of the manual
+on HDD systems.
+
+There is also a mysterious replacement of underlines for dashes on entity
+names. In future, would be better to remove this, so manual writing gets
+less surprising.
[1] https://github.com/php/doc-base/pull/183
[2] https://github.com/php/doc-en/pull/4330
*/
+const BACKPORT_MIXED_REPLACE = true;
+const ENTITY_NAME_REPLACE = true;
+const LIBXML_LIMITS_HACK = true;
+
// Setup
ini_set( 'display_errors' , 1 );
@@ -56,275 +60,318 @@
set_time_limit( 0 );
ob_implicit_flush();
-const LIBXML_LIMITS_HACK = true;
-
// Usage
-$root = realpain( __DIR__ . "/../.." );
$lang = "";
+$langs = [ "en" ];
+$langBase = realpain( __DIR__ . "/../.." );
$chmonly = false;
-$debug = false;
array_shift( $argv );
foreach( $argv as $arg )
{
- if ( $arg == "--chmonly" )
- {
- $chmonly = true;
- continue;
- }
- if ( $arg == "--debug" )
- {
- $debug = true;
- continue;
- }
- $lang = $arg;
+ $lang = rtrim( $arg , "\\/" );
+ $langs[] = $lang;
}
-// Main
+// Generation
-echo "Running file-entities.php... ";
+print "Running file-entities.php... ";
+$allFiles = [];
$entities = [];
-$mixedCase = [];
-generate_file_entities( $root , "en" );
-generate_list_entities( $root , "en" );
+foreach( $langs as $lang )
+ scan_files( $langBase , $lang , $allFiles );
+check_case_conflict( $allFiles );
-if ( $lang != "" )
- generate_file_entities( $root , $lang );
+generate_entities( $allFiles , $entities );
+writeEntities( $entities );
-pushEntity( "global.function-index", path: realpain( __DIR__ . "/.." ) . "/funcindex.xml" );
+$total = count( $entities );
+print "done: $total entities.\n";
-if ( ! $chmonly )
- foreach( $entities as $ent )
- if ( str_starts_with( $ent->name , "chmonly." ) )
- $ent->path = '';
+exit( 0 );
-$outfile = realpain( __DIR__ . "/../temp/file-entities.ent" , touch: true );
+// old scheme
+// file en
+// list en
+// file? lang
-$file = fopen( $outfile , "w" );
-if ( ! $file )
+class Entity
{
- echo "Failed to open $outfile\n.";
- exit( 1 );
+ public function __construct(
+ public string $name,
+ public string $text,
+ public string $file,
+ ) {}
}
-fputs( $file , "<!-- DON'T TOUCH - AUTOGENERATED BY file-entities.php -->\n\n" );
-
-ksort( $entities );
-
-foreach ( $entities as $ent )
- writeEntity( $file , $ent );
-
-fclose( $file );
-
-$total = count( $entities );
-echo "done: $total entities.\n";
-exit( 0 );
-
-
+function scan_files( string $langBase , string $lang , array & $allFIles )
+{
+ $todo = [ "" ];
+ while ( count( $todo ) > 0 )
+ {
+ $dir = array_pop( $todo );
+ $scan = "$langBase/$lang/$dir";
+ $paths = scandir( $scan );
+ foreach( $paths as $path )
+ {
+ if ( $path == "" || $path[0] == '.' )
+ continue;
+
+ $part = trim( "$dir/$path" , '/' );
+ $full = "$langBase/$lang/$dir/$path";
+ if ( is_dir( $full ) )
+ {
+ $todo[] = $part;
+ continue;
+ }
+ if ( ! str_ends_with( $part , ".xml" ) )
+ continue;
+
+ $real = realpain( $full );
+ $allFIles[ $part ] = $real;
+ }
+ }
+}
-class Entity
+function check_case_conflict( array $allFIles )
{
- function __construct( public string $name , public string $text , public string $path ) {}
+ $mixedCase = [];
+ foreach( $allFIles as $name => $file )
+ {
+ $lname = strtolower( $name );
+ if ( isset( $mixedCase[ $lname ] ) && $mixedCase[ $lname ] != $name )
+ {
+ print <<<END
+ \n\n
+ BRICKED/BROKEN BUILD on case insensitive file systems!
+
+ Detected file entities names, distinct only by upper/lower case:
+ - {$mixedCase[ $lname ]}
+ - $name
+
+ This will PERMANENTLY BRICK manual build on Windows machines!
+
+ If you are seeing this message building doc-en, avoid committing any changes
+ on repository, and if it's already committed, revert and send a heads up on
+ mail lists, on how to fix the issue (refer to this message).
+
+ If you are seeing this message building a translation, this means that the
+ translation has files or directories that differ from doc-en only by
+ upper or lower case letters. Find these differences and fix them at the git
+ level ('git mv"). After, delete the files and 'git restore' them, to see if
+ the 'git mv' worked.
+
+ This message only may be visible on non-Windows machines. Mixed cases inside
+ a repository, or between repositories, WILL cause difficult to debug build
+ failures on Windows, without any other information. After a local checkout
+ is bricked, there is no easy fix, other than DELETING the local checkout and
+ doing a fresh checkout.
+
+ See: https://github.com/php/doc-en/pull/4330#issuecomment-2557306828\n\n
+ END;
+ exit( 1 );
+ }
+ $mixedCase[ $lname ] = $name;
+ }
}
-function pushEntity( string $name , string $text = '' , string $path = '' )
+function generate_entities( array $allFiles , array & $entities )
{
- global $entities;
- global $mixedCase;
+ // Ugly, but necessary
+ // TODO move this file from doc-bese to doc-en, with a do-not-translate PI
+
+ $name = 'global.function-index';
+ $file = realpain( __DIR__ . "/../funcindex.xml" );
+ $text = "<!ENTITY $name SYSTEM '$file'>";
+ pushEntity( $entities , $name , $text );
- $name = str_replace( '_' , '-' , $name );
- $path = str_replace( '\\' , '/' , $path );
- $ent = new Entity( $name , $text , $path );
- $entities[ $name ] = $ent;
+ // Inclusion of a single file is easy. The entity name is the
+ // relative path without the .xml extension (sadly), and the text
+ // is complete DTD entity with a SYSTEM pointing to the real path
+ // of the included file.
- if ( ( $text == "" && $path == "" ) || ( $text != "" && $path != "" ) )
+ foreach( $allFiles as $path => $file )
{
- echo "Something went wrong on file-entities.php.\n";
- exit( 1 );
+ $name = pathToEntityName( $path );
+ $text = "<!ENTITY $name SYSTEM '$file'>";
+ pushEntity( $entities , $name , $text );
}
- $lname = strtolower( $name );
- if ( isset( $mixedCase[ $lname ] ) && $mixedCase[ $lname ] != $name )
- {
- echo <<<END
- \n\n
- BROKEN BUILD on case insensitive file systems!
+ // Inclusion of reference/ directories is a little more involved.
+ // The entity name is calculated from the relative path, but with
+ // an 'entities' component added in penultimae position. The
+ // contents are concatened DTD entities references, as above.
- Detected file entities names, distinct only by case:
- - {$mixedCase[ $lname ]}
- - $name
+ // LIBXML_LIMITS_HACK - Unfortunatlly, we nedd to put these entities
+ // that expand in another DTD entities as separate files, to bypass
+ // some hardcoded limits of libxml2. This is slow, more so on HDDs.
- This may PERMANENTLY BRICK manual build on Windows machines!
+ // BACKPORT_MIXED_REPLACE - Anoying enought, the previous script
+ // normalized the entity name, but not the file name of the extra file
+ // file. So indirect file entities ends having a surprising convention:
+ //
+ // <!ENTITY name-dir SYSTEM 'name_dir.ent'>
+ //
+ // Mind the distinction between _ and - above. In the future, let's
+ // remove this, to make debugging easier.
- If you are seeing this message building doc-en, avoid committing any changes
- on repository, and if it's already committed, revert and send a heads up on
- mail lists, on how to fix the issue (refer to this message).
+ $groupFilename = []; // LIBXML_LIMITS_HACK
+ $groupContents = [];
- If you are seeing this message building a translation, this means that the
- translation may have files or directories that differ from doc-en only by
- upper or lower case letters. Find these differences and fix them at the git
- level ('git mv"). After, delete the files and 'git restore' them, to see if
- the 'git mv' worked.
+ foreach( $allFiles as $path => $null )
+ {
+ // Only generate directory inclusions for reference/
- This message only may be visible on non-Windows machines. Mixed cases inside
- a repository, or between repositories, may only cause difficult to debug build
- failures on Windows, without any other information. There is no easy fix for
- this than a complete new checkout of the affected repository.
+ if ( ! str_starts_with ( $path , 'reference' ) )
+ continue;
- See: https://github.com/php/doc-en/pull/4330#issuecomment-2557306828\n\n
- END;
- exit( 1 );
- }
- $mixedCase[ $lname ] = $name;
-}
+ // Entity name
+ //
+ // Discard the file part, 'entities' in the
+ // second-to-last position.
+ //
+ // dir/dir/dir/file.xml -> dir.dir.entities.dir
-function generate_file_entities( string $root , string $lang )
-{
- $path = "$root/$lang";
- $test = realpain( $path );
- if ( $test === false || is_dir( $path ) == false )
- {
- echo "Language directory not found: $path\n.";
- exit( 1 );
- }
- $path = $test;
+ $parts = explode( '/' , $path );
+ array_pop( $parts );
+ $last = array_pop( $parts );
+ $parts[] = 'entities';
+ $parts[] = $last;
+ $entName = implode( '.' , $parts );
+ $entName = str_replace( '_' , '-' , $entName ); // BACKPORT_MIXED_REPLACE
- file_entities_recurse( $path , array() );
-}
+ // Entity fila
+ //
+ // dir/dir/dir/file.xml -> dir.dir.dir.ent
-function file_entities_recurse( string $langroot , array $dirs )
-{
- $dir = rtrim( "$langroot/" . implode( '/' , $dirs ) , "/" );
- $files = scandir( $dir );
- $subdirs = [];
+ $parts = explode( '/' , $path );
+ array_pop( $parts );
+ array_push( $parts , 'ent');
+ $entFile = implode( '.' , $parts );
- foreach( $files as $file )
- {
- if ( $file == "" )
- continue;
- if ( $file[0] == "." )
- continue;
- if ( $file == "entities" && count( $dirs ) == 0 )
- continue;
+ $groupFilename[ $entName ] = $entFile;
- $path = "$dir/$file";
+ // Contents
- if ( is_dir ( $path ) )
- {
- $subdirs[] = $file;
- continue;
- }
- if ( str_ends_with( $file , ".xml" ) )
- {
- $name = implode( '.' , $dirs ) . "." . basename( $file , ".xml" );
- $name = trim( $name , "." );
- pushEntity( $name , path: $path );
- }
+ $name = pathToEntityName( $path );
+ $entRef = "&{$name};";
+
+ $groupContents[ $entName ][ $name ] = $entRef;
}
- foreach( $subdirs as $subdir )
+ // Merge
+
+ foreach( $groupContents as $name => $list )
{
- $recurse = $dirs;
- $recurse[] = $subdir;
- file_entities_recurse( $langroot , $recurse );
+ ksort( $list );
+ $text = implode ( "\n" , $list );
+ $file = $groupFilename[ $name ];
+ pushEntity( $entities , $name , $text , $file );
}
}
-function generate_list_entities( string $root , string $lang )
+function pathToEntityName( string $name , string $removeSuffix = "" ) : string
+{
+ if ( str_ends_with( $name , ".xml" ) )
+ $name = substr( $name , 0 , -4 );
+ else
+ throw new Exception( "Expected extension .xml" );
+
+ $name = str_replace( '\\' , '/' , $name );
+ $name = str_replace( '_' , '-' , $name ); // ENTITY_NAME_REPLACE
+ $name = str_replace( '/' , '.' , $name );
+ $name = trim( $name , '.' );
+ return $name;
+
+ // ENTITY_NAME_REPLACE, or a TODO to a far future
+ // - Replace all name replaced entities from doc en
+ // - Add the removed entities on doc-en/entities/remove.ent
+ // - Remove all codepaths related to ENTITY_NAME_REPLACE constant
+}
+
+function pushEntity( array & $entities , string $name , string $text , string $file = "" )
{
- $path = "$root/$lang";
- $test = realpain( $path );
- if ( $test === false || is_dir( $path ) == false )
+ if ( $name == "" || $text == "" )
{
- echo "Language directory not found: $path\n.";
+ print "Something went very wrong on file-entities.php.\n";
exit( 1 );
}
- $path = $test;
- $dirs = [ "reference" ];
- list_entities_recurse( $path , $dirs );
+ $entity = new Entity( $name , $text , $file );
+ $entities[ $name ] = $entity;
}
-function list_entities_recurse( string $root , array $dirs )
+function writeEntities( array $entities )
{
- $list = [];
-
- $dir = rtrim( "$root/" . implode( '/' , $dirs ) , "/" );
- $files = scandir( $dir );
- $subdirs = [];
+ // Output a single temp/file-entities.ent file for single file inclusion.
- foreach( $files as $file )
- {
- if ( $file == "" )
- continue;
- if ( $file[0] == "." )
- continue;
+ // Output separate files for file list inclusions, at
+ // temp/file-entities/dir.dir.dir.ent
+ // LIBXML_LIMITS_HACK
- $path = "$dir/$file";
+ ksort( $entities );
- if ( is_dir ( $path ) )
- {
- $subdirs[] = $file;
- continue;
- }
+ $outFile = realpain( __DIR__ . "/../temp/file-entities.ent" , touch: true );
+ $lstFile = realpain( __DIR__ . "/../temp/file-entities.txt" , touch: true );
+ $sepPath = realpain( __DIR__ . "/../temp/file-entities" , mkdir: true );
- if ( str_ends_with( $file , ".xml" ) )
- {
- $name = implode( '.' , $dirs ) . "." . basename( $file , ".xml" );
- $name = trim( $name , "." );
- $name = str_replace( '_' , '-' , $name );
- $list[ $name ] = "&{$name};";
- }
+ $singleFile = fopen( $outFile , "w" );
+ if ( ! $singleFile )
+ {
+ print "Failed to open $outFile\n.";
+ exit( 1 );
}
- ksort( $list );
+ fputs( $singleFile , "<!-- DON'T TOUCH - AUTOGENERATED BY file-entities.php -->\n\n" );
- $copy = $dirs;
- $last = array_pop( $copy );
- $copy[] = "entities";
- $copy[] = $last;
+ // Life could be simpler, but the building of PHP Manual is already
+ // triping some hardcoded limits of bundled libxml2.
- $name = implode( "." , $copy );
- $text = implode( "\n" , $list );
+ // Off loading DTD entities that expand to more DTD entities,
+ // as external files, somehow avoid these limits.
- if ( $text != "" )
+ if ( LIBXML_LIMITS_HACK )
{
- if ( LIBXML_LIMITS_HACK )
+ foreach ( $entities as $entity )
{
- static $entityDir = "";
- if ( $entityDir == "" )
- $entityDir = realpain( __DIR__ . "/../temp/file-entities" , mkdir: true );
-
- $path = $entityDir . "/" . implode( '.' , $dirs ) . ".ent";
- file_put_contents( $path , $text );
- pushEntity( $name , path: $path );
+ $name = $entity->name;
+ $text = $entity->text;
+ $file = $entity->file;
+ $extraFile = "{$sepPath}/{$file}";
+
+ if ( $text[0] == '&' )
+ writeEntityIndirectSlow( $singleFile , $extraFile , $name , $text );
+ else
+ fputs( $singleFile , "$text\n" );
}
- else
- pushEntity( $name , text: $text );
}
-
- foreach( $subdirs as $subdir )
+ else
{
- $recurse = $dirs;
- $recurse[] = $subdir;
- list_entities_recurse( $root , $recurse );
+ foreach ( $entities as $name => $text )
+ fputs( $singleFile , "$text\n" );
}
+
+ fclose( $singleFile );
+
+ // After everything is said and done, also output a listing file, so
+ // it is possible to analyse collisions between 'text' and 'file'
+ // entities.
+
+ $contents = implode( "\n" , array_keys( $entities ) );
+ file_put_contents( $lstFile , $contents );
}
-function writeEntity( $file , Entity $ent )
+function writeEntityIndirectSlow( $singleFile , string $extraFile , string $name , string $text )
{
- $name = $ent->name;
- $text = $ent->text;
- $path = $ent->path;
+ // The entity will point to to a new, individual filename
- if ( $path == "" )
- $line = "<!ENTITY $name '$text'>\n";
- else
- $line = "<!ENTITY $name SYSTEM '$path'>\n";
+ fputs( $singleFile , "<!ENTITY $name SYSTEM '$extraFile'>\n" );
+
+ // And the new individual file will hold the final text
- fwrite( $file , $line );
+ file_put_contents( $extraFile , $text );
}
function realpain( string $path , bool $touch = false , bool $mkdir = false ) : string
@@ -332,7 +379,7 @@ function realpain( string $path , bool $touch = false , bool $mkdir = false ) :
// pain is real
// care for external XML tools (realpath() everywhere)
- // care for Windows builds (foward slashes everywhere)
+ // care for Windows builds (forward slashes everywhere)
// avoid `cd` and chdir() like the plague
$path = str_replace( "\\" , '/' , $path );