From 55bfcc2ea255e5b22b152ead2248b21012c58f9a Mon Sep 17 00:00:00 2001 From: tianfenghan Date: Tue, 15 Sep 2026 12:09:27 +0800 Subject: [PATCH] feat(build): enhance incremental compilation and response file handling - Add explicit response file path support to compiler backends - Implement TYPEPHP_COMPILER_EXECUTABLE constant for compiler fingerprinting - Add exception handling for native source project compilation - Create dedicated extension header inclusion logic to reduce build invalidations - Implement small task lane reservation in parallel compilation - Add DOM extension requirement check for native project support - Optimize symbol collection with single depth-first AST walk - Add request array default metadata finalization - Implement content-based rather than timestamp-based object caching - Add link cache metadata file management - Update incremental compilation to preserve timestamps for identical content - Add comprehensive test coverage for response file and caching features --- phpunit/src/Backend/BackendTest.php | 20 +++ .../src/Build/IncrementalDeclarationTest.php | 105 ++++++++++- phpunit/src/ParallelCompileTest.php | 57 ++++++ project.yml | 1 + src/Backend/CompilerBackend.php | 16 +- src/Backend/GccLikeBackend.php | 6 +- src/Backend/Msvc.php | 6 +- src/Build/IncrementalCompilationTrait.php | 21 +++ src/Build/NativeBuilder.php | 24 ++- src/Build/NativeCommandOptionsTrait.php | 2 + src/Build/NativeSourceProjectConfig.php | 13 ++ src/Build/SourcePipelineTrait.php | 14 +- src/Preprocessor.php | 168 +++++++++++------- src/Translator.php | 165 ++++++++++++----- src/compiler.php | 15 +- 15 files changed, 514 insertions(+), 119 deletions(-) diff --git a/phpunit/src/Backend/BackendTest.php b/phpunit/src/Backend/BackendTest.php index f0dba82d..d55c67d5 100644 --- a/phpunit/src/Backend/BackendTest.php +++ b/phpunit/src/Backend/BackendTest.php @@ -330,6 +330,26 @@ class BackendTest extends TestCase $this->assertSame('"' . $objectWithoutSpace . '"', $lines[1]); } + public function testResponseFileCanBePlacedInTheBuildDirectory(): void + { + $platform = new Linux(); + $compiler = new Gcc($platform, 'g++'); + $dir = $this->createTemporaryDirectory('backend response build directory'); + $target = $dir . '/output/my app'; + $rspFile = $dir . '/build/my app.rsp'; + $object = $dir . '/object.o'; + + $cmd = $compiler->buildLinkCommand( + [$object], + $target, + ['response_file' => $rspFile], + ); + + $this->assertStringContainsString(escapeshellarg('@' . $rspFile), $cmd); + $this->assertFileExists($rspFile); + $this->assertFileDoesNotExist($target . '.rsp'); + } + /** * 测试 Clang 编译器基本信息 */ diff --git a/phpunit/src/Build/IncrementalDeclarationTest.php b/phpunit/src/Build/IncrementalDeclarationTest.php index 62d2f5aa..a63c53c4 100644 --- a/phpunit/src/Build/IncrementalDeclarationTest.php +++ b/phpunit/src/Build/IncrementalDeclarationTest.php @@ -79,6 +79,9 @@ PHP); $this->buildDirectory . '/include/php_incremental_runtime_decl.h', ); $consumerCode = (string) file_get_contents($consumerCpp); + $extensionCode = (string) file_get_contents( + $this->buildDirectory . '/extension-incremental.cc', + ); self::assertStringContainsString('php_incremental__answer(', $providerDeclarations); self::assertStringContainsString('_const_var_Incremental__LIMIT', $providerDeclarations); self::assertStringContainsString('_global_var_shared', $providerDeclarations); @@ -94,6 +97,22 @@ PHP); '#include <' . basename($consumerHeader) . '>', $consumerCode, ); + self::assertStringContainsString( + '#include ', + $extensionCode, + ); + self::assertStringContainsString( + '#include <' . basename($compiler->getArgInfoHeaderFile($this->provider)) . '>', + $extensionCode, + ); + self::assertStringNotContainsString( + '#include <' . basename($providerHeader) . '>', + $extensionCode, + ); + self::assertStringNotContainsString( + '#include <' . basename($consumerHeader) . '>', + $extensionCode, + ); $symbols = $this->property($compiler, 'symbolDeclInFile'); self::assertSame($this->provider, $symbols['function:incremental\\answer']); @@ -118,28 +137,89 @@ PHP); ); } - public function testChangedDeclarationRegeneratesItsTransitiveConsumersOnly(): void + public function testGeneratorFingerprintChangeKeepsIdenticalCppTimestamps(): void + { + $first = $this->convertProject(); + $generatedFiles = [ + $this->invoke($first, 'getCppFile', $this->provider), + $this->invoke($first, 'getCppFile', $this->consumer), + $this->invoke($first, 'getCppFile', $this->independent), + ]; + $oldTimestamp = 1_600_000_000; + foreach ($generatedFiles as $generatedFile) { + touch($generatedFile, $oldTimestamp); + } + + $stateFile = $this->buildDirectory . '/cache/incremental/incremental/build-state.json'; + $state = json_decode((string) file_get_contents($stateFile), true, flags: JSON_THROW_ON_ERROR); + $state['generatorFingerprint'] = str_repeat('0', 64); + file_put_contents( + $stateFile, + json_encode($state, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . PHP_EOL, + ); + clearstatcache(); + + $this->convertProject(); + + clearstatcache(); + foreach ($generatedFiles as $generatedFile) { + self::assertSame($oldTimestamp, filemtime($generatedFile)); + } + } + + public function testRegenerationKeepsTimestampsForIdenticalGeneratedContents(): void { $first = $this->convertProject(); $providerCpp = $this->invoke($first, 'getCppFile', $this->provider); + $providerHeader = $first->getDeclarationHeaderFile($this->provider); $consumerCpp = $this->invoke($first, 'getCppFile', $this->consumer); $independentCpp = $this->invoke($first, 'getCppFile', $this->independent); $oldTimestamp = 1_600_000_000; - foreach ([$providerCpp, $consumerCpp, $independentCpp] as $cppFile) { - touch($cppFile, $oldTimestamp); + foreach ([$providerCpp, $providerHeader, $consumerCpp, $independentCpp] as $generatedFile) { + touch($generatedFile, $oldTimestamp); } - file_put_contents($this->provider, "\n// changed provider\n", FILE_APPEND); + $providerSource = (string) file_get_contents($this->provider); + file_put_contents($this->provider, str_replace('const LIMIT = 42;', 'const LIMIT = 43;', $providerSource)); clearstatcache(); $this->convertProject(); clearstatcache(); - self::assertGreaterThan($oldTimestamp, filemtime($providerCpp)); - self::assertGreaterThan($oldTimestamp, filemtime($consumerCpp)); + self::assertGreaterThan($oldTimestamp, filemtime($providerHeader)); + self::assertSame($oldTimestamp, filemtime($providerCpp)); + // The dependency graph still regenerates the consumer, but its own C++ + // bytes only include the provider header by its stable name. Preserve + // the timestamp; the changed header content invalidates its object key. + self::assertSame($oldTimestamp, filemtime($consumerCpp)); self::assertSame($oldTimestamp, filemtime($independentCpp)); } - public function testGeneratedObjectCacheRequiresAnObjectNotOlderThanItsCppSource(): void + public function testCachedClassKeepsRuntimeArrayPropertyDefaultAllocator(): void + { + file_put_contents($this->independent, <<<'PHP' +convertProject(); + $this->convertProject(); + + $extensionCode = (string) file_get_contents( + $this->buildDirectory . '/extension-incremental.cc', + ); + $initializer = 'typephp_ensure_request_array_defaults_IncrementalDefaults()'; + self::assertSame(2, substr_count($extensionCode, $initializer)); + self::assertStringContainsString( + 'php_class_entry_IncrementalDefaults->create_object =', + $extensionCode, + ); + } + + public function testGeneratedObjectCacheUsesContentRatherThanSourceTimestamp(): void { $compiler = $this->convertProject(); $consumerCpp = $this->invoke($compiler, 'getCppFile', $this->consumer); @@ -165,11 +245,16 @@ PHP); touch($consumerCpp, $objectTimestamp + 1); clearstatcache(); + self::assertTrue( + $this->invoke($compiler, 'hasGeneratedObjectFileCache', $consumerCpp, $consumerObject), + ); + + file_put_contents($consumerCpp, "\n// changed generated input\n", FILE_APPEND); + clearstatcache(); self::assertFalse( $this->invoke($compiler, 'hasGeneratedObjectFileCache', $consumerCpp, $consumerObject), ); - touch($consumerCpp, $sourceTimestamp); unlink($consumerObject); clearstatcache(); self::assertFalse( @@ -200,6 +285,10 @@ PHP); $objects = [$firstObject, $secondObject]; $this->invoke($compiler, 'writeLinkCache', $objects, $target); + self::assertFileExists( + $this->buildDirectory . '/incremental-bin.typephp-link-cache', + ); + self::assertFileDoesNotExist($target . '.typephp-link-cache'); self::assertTrue($this->invoke($compiler, 'hasLinkCache', $objects, $target)); touch($secondObject, $timestamp + 1); diff --git a/phpunit/src/ParallelCompileTest.php b/phpunit/src/ParallelCompileTest.php index 97c13626..7902280c 100644 --- a/phpunit/src/ParallelCompileTest.php +++ b/phpunit/src/ParallelCompileTest.php @@ -89,6 +89,43 @@ final class ParallelCompileTest extends TestCase self::assertStringContainsString('compiler-error', implode("\n", $completion['output'])); } + public function testProcessPoolCanReserveOneLaneForSmallTasks(): void + { + $smallMarker = $this->directory . '/small.ready'; + $largeObject = $this->directory . '/large.o'; + $middleObject = $this->directory . '/middle.o'; + $smallObject = $this->directory . '/small.o'; + + $builder = new NativeBuilder($this->createMock(CompilerBackend::class)); + $result = $builder->dispatchProcessParallel( + [ + $this->task( + 'large.cc', + $largeObject, + $this->waitForFileCommand($smallMarker, $largeObject), + ), + $this->task( + 'middle.cc', + $middleObject, + $this->waitForFileCommand($largeObject, $middleObject), + ), + $this->task( + 'small.cc', + $smallObject, + $this->createFilesCommand([$smallMarker, $smallObject]), + ), + ], + 2, + reserveSmallTaskLane: true, + ); + + self::assertSame([], $result['failures']); + self::assertEqualsCanonicalizing( + [$largeObject, $middleObject, $smallObject], + $result['objects'], + ); + } + /** @return array{source: string, object: string, command: string} */ private function task(string $source, string $object, string $command): array { @@ -106,6 +143,26 @@ final class ParallelCompileTest extends TestCase return $this->phpCommand($code); } + /** @param list $files */ + private function createFilesCommand(array $files): string + { + $code = ''; + foreach ($files as $file) { + $code .= 'file_put_contents(' . var_export($file, true) . ", 'ready');"; + } + return $this->phpCommand($code); + } + + private function waitForFileCommand(string $dependency, string $object): string + { + $code = '$deadline = microtime(true) + 2;' + . 'while (file_exists(' . var_export($dependency, true) . ') === false && microtime(true) < $deadline) {' + . ' usleep(10000); }' + . 'if (file_exists(' . var_export($dependency, true) . ') === false) { exit(9); }' + . 'file_put_contents(' . var_export($object, true) . ", 'object');"; + return $this->phpCommand($code); + } + private function phpCommand(string $code): string { return escapeshellarg(PHP_BINARY) . ' -n -r ' . escapeshellarg($code); diff --git a/project.yml b/project.yml index aeb48562..501a2aeb 100644 --- a/project.yml +++ b/project.yml @@ -2,6 +2,7 @@ name: tpc build-mode: bin version: 0.9.0 cxx-std: c++17 +optimize: 2 cxx-flags: - -Wall diff --git a/src/Backend/CompilerBackend.php b/src/Backend/CompilerBackend.php index 87b751ad..43bdfd6e 100644 --- a/src/Backend/CompilerBackend.php +++ b/src/Backend/CompilerBackend.php @@ -162,12 +162,22 @@ abstract class CompilerBackend * Write the object file list to a Response File to avoid exceeding the OS command-line length limit (8191 characters on Windows). * * @param array $objectFiles List of object file paths. - * @param string $targetFile Final output file path (the Response File is written to the same directory). + * @param string $targetFile Final output file path. + * @param string|null $responseFile Explicit intermediate response-file path. * @return string Linker argument, e.g. @build/project.rsp */ - protected function createResponseFile(array $objectFiles, string $targetFile): string + protected function createResponseFile( + array $objectFiles, + string $targetFile, + ?string $responseFile = null, + ): string { - $rspFile = dirname($targetFile) . DIRECTORY_SEPARATOR . basename($targetFile) . '.rsp'; + $rspFile = $responseFile + ?? dirname($targetFile) . DIRECTORY_SEPARATOR . basename($targetFile) . '.rsp'; + $directory = dirname($rspFile); + if (!is_dir($directory) && !mkdir($directory, 0777, true) && !is_dir($directory)) { + throw new \RuntimeException('Cannot create response-file directory: ' . $directory); + } $this->lastResponseFile = $rspFile; $lines = []; foreach ($objectFiles as $file) { diff --git a/src/Backend/GccLikeBackend.php b/src/Backend/GccLikeBackend.php index 7fdde5d5..a6c42ae7 100644 --- a/src/Backend/GccLikeBackend.php +++ b/src/Backend/GccLikeBackend.php @@ -261,7 +261,11 @@ abstract class GccLikeBackend extends CompilerBackend public function buildLinkCommand(array $objectFiles, string $outputFile, array $options = []): string { $cmd = $this->getLinkerCommand(); - $cmd .= ' ' . $this->createResponseFile($objectFiles, $outputFile); + $cmd .= ' ' . $this->createResponseFile( + $objectFiles, + $outputFile, + $options['response_file'] ?? null, + ); $cmd .= ' ' . $this->getLinkerOutputFlag() . ' ' . escapeshellarg($outputFile); if (!empty($options['library_paths'])) { diff --git a/src/Backend/Msvc.php b/src/Backend/Msvc.php index 6504dda5..b0eeaf85 100644 --- a/src/Backend/Msvc.php +++ b/src/Backend/Msvc.php @@ -176,7 +176,11 @@ class Msvc extends CompilerBackend public function buildLinkCommand(array $objectFiles, string $outputFile, array $options = []): string { $cmd = $this->getLinkerCommand(); - $cmd .= ' ' . $this->createResponseFile($objectFiles, $outputFile); + $cmd .= ' ' . $this->createResponseFile( + $objectFiles, + $outputFile, + $options['response_file'] ?? null, + ); $cmd .= ' /OUT:' . escapeshellarg($outputFile); if (!empty($options['debug'])) { diff --git a/src/Build/IncrementalCompilationTrait.php b/src/Build/IncrementalCompilationTrait.php index 555e0419..5e6b58d6 100644 --- a/src/Build/IncrementalCompilationTrait.php +++ b/src/Build/IncrementalCompilationTrait.php @@ -337,6 +337,27 @@ trait IncrementalCompilationTrait 'literalStrings' => !$this->noLiteralStrings, 'debug' => $this->debug, ])); + + // A compiled tpc executable is an immutable snapshot of the generator. + // Walking and hashing every compiler PHP source on each consumer build + // is both unnecessary and disproportionately expensive through the AOT + // Zend bridge. Fingerprint the executable snapshot with one native hash + // operation instead. The interpreted development entry keeps the source + // walk below so edits invalidate generated-code caches immediately. + if (!defined('TYPEPHP_PHP_SCRIPT_ENTRY') + && defined('TYPEPHP_COMPILER_EXECUTABLE')) { + $executable = constant('TYPEPHP_COMPILER_EXECUTABLE'); + if (is_string($executable) && is_file($executable)) { + hash_update($context, str_replace('\\', '/', $executable) . "\0"); + if (!hash_update_file($context, $executable)) { + throw new \RuntimeException( + 'Cannot fingerprint TypePHP compiler executable: ' . $executable, + ); + } + return hash_final($context); + } + } + $sourceDirectory = dirname(__DIR__); $iterator = new \RecursiveIteratorIterator( new \RecursiveDirectoryIterator($sourceDirectory, \FilesystemIterator::SKIP_DOTS), diff --git a/src/Build/NativeBuilder.php b/src/Build/NativeBuilder.php index 3dfb8d28..1bceea38 100644 --- a/src/Build/NativeBuilder.php +++ b/src/Build/NativeBuilder.php @@ -68,12 +68,15 @@ final readonly class NativeBuilder * * @param list $tasks * @param null|Closure(string, string, string, list, int, bool, int): void $completed + * @param bool $reserveSmallTaskLane Whether the size-sorted queue reserves + * one worker for tasks taken from its small-file end. * @return array{objects: list, failures: list} */ public function dispatchProcessParallel( array $tasks, int $jobs, ?Closure $completed = null, + bool $reserveSmallTaskLane = false, ): array { $jobs = max(1, $jobs); $queue = $tasks; @@ -81,12 +84,24 @@ final readonly class NativeBuilder $objects = []; $failures = []; $completedCount = 0; + $largeTaskCount = 0; + $largeLaneLimit = $reserveSmallTaskLane ? max(1, $jobs - 1) : $jobs; $nullDevice = PHP_OS_FAMILY === 'Windows' ? 'NUL' : '/dev/null'; try { while ($queue !== [] || $running !== []) { while (count($running) < $jobs && $queue !== []) { - $task = array_shift($queue); + // The caller supplies tasks largest-first. Keep most workers + // on that end to minimize the parallel tail, while one fast + // lane drains the smallest end and keeps progress observable + // during very expensive translation units. + if ($reserveSmallTaskLane && $largeTaskCount >= $largeLaneLimit) { + $task = array_pop($queue); + $lane = 'small'; + } else { + $task = array_shift($queue); + $lane = 'large'; + } $logFile = tempnam(sys_get_temp_dir(), 'typephp-compile-'); if ($logFile === false) { $failures[] = $task['source']; @@ -132,7 +147,11 @@ final readonly class NativeBuilder 'task' => $task, 'process' => $process, 'log' => $logFile, + 'lane' => $lane, ]; + if ($lane === 'large') { + $largeTaskCount++; + } } if ($running === []) { @@ -159,6 +178,9 @@ final readonly class NativeBuilder ? [] : (preg_split('/\R/', rtrim($contents)) ?: []); $task = $entry['task']; + if ($entry['lane'] === 'large') { + $largeTaskCount--; + } $success = $exitCode === 0 && is_file($task['object']); if ($success) { $objects[] = $task['object']; diff --git a/src/Build/NativeCommandOptionsTrait.php b/src/Build/NativeCommandOptionsTrait.php index 7793654f..6b86175b 100644 --- a/src/Build/NativeCommandOptionsTrait.php +++ b/src/Build/NativeCommandOptionsTrait.php @@ -187,6 +187,8 @@ trait NativeCommandOptionsTrait 'sanitize' => $this->sanitize, 'lto' => $this->enableLto, 'target_platform' => $targetPlatform, + 'response_file' => $this->getBuildDir() . DIRECTORY_SEPARATOR + . basename($this->getTargetFileName()) . '.rsp', ]; $rpaths = $this->isNanoMode() diff --git a/src/Build/NativeSourceProjectConfig.php b/src/Build/NativeSourceProjectConfig.php index 44aed5ba..c9766e77 100644 --- a/src/Build/NativeSourceProjectConfig.php +++ b/src/Build/NativeSourceProjectConfig.php @@ -32,6 +32,7 @@ final readonly class NativeSourceProjectConfig if (strtolower(pathinfo($path, PATHINFO_EXTENSION)) !== 'xml' || !is_file($path)) { return false; } + self::requireDomExtension(); $document = new DOMDocument(); if (!@$document->load($path, LIBXML_NONET | LIBXML_NOBLANKS)) { return false; @@ -47,6 +48,7 @@ final readonly class NativeSourceProjectConfig throw new RuntimeException("Native project does not exist: {$path}"); } + self::requireDomExtension(); $document = new DOMDocument(); if (!@$document->load($file, LIBXML_NONET | LIBXML_NOBLANKS)) { throw new RuntimeException("Unable to parse native project XML: {$file}"); @@ -145,6 +147,17 @@ final readonly class NativeSourceProjectConfig return $value === '' ? null : $value; } + private static function requireDomExtension(): void + { + if (!extension_loaded('dom') + || !class_exists(DOMDocument::class) + || !class_exists(DOMElement::class)) { + throw new RuntimeException( + 'Native project.xml support requires the PHP DOM extension (ext-dom)' + ); + } + } + private static function absolutePath(string $base, string $path): string { if ($path[0] === '/' || $path[0] === '\\' diff --git a/src/Build/SourcePipelineTrait.php b/src/Build/SourcePipelineTrait.php index 3a1de608..d55b4041 100644 --- a/src/Build/SourcePipelineTrait.php +++ b/src/Build/SourcePipelineTrait.php @@ -355,6 +355,11 @@ trait SourcePipelineTrait // expressions before translating any function body so cache IDs // are assigned exclusively in the convert phase. $this->finalizeDeclarationExpressions($files); + // Whole-program extension generation must not depend on conversion + // side effects from dirty files. Clean incremental files are not + // converted, but their non-empty property defaults still require a + // custom allocation path in the regenerated module entry. + $this->finalizeRequestArrayDefaultMetadata(); $this->initializeDeclarationHeaderFiles($files); $this->restoreCleanIncrementalMetadata($files); @@ -373,12 +378,17 @@ trait SourcePipelineTrait $sourceFiles[] = $cppFile; } $this->climate->darkGray( - '[incremental] reuse: ' . $this->getRelativePath($path), + '[cached] ' . $this->getRelativePath($path), ); continue; } $statisticsBefore = $this->compilationStatistics->all(); - $cppFile = $this->convertFile($path, true); + // A dirty dependency means code generation must run; + // it does not mean the generated bytes changed. Keep + // an existing translation unit's timestamp when the + // output is identical, matching CMake/Ninja's restat + // model and preventing needless native recompilation. + $cppFile = $this->convertFile($path); $this->recordIncrementalConversion( $path, $cppFile !== null, diff --git a/src/Preprocessor.php b/src/Preprocessor.php index af49d8eb..e987acb5 100644 --- a/src/Preprocessor.php +++ b/src/Preprocessor.php @@ -1050,83 +1050,129 @@ class Preprocessor extends CompilerBase */ protected function findSymbolUsing(NodeAbstract|array $ast): void { - $nodeFinder = new NodeFinder(); - $functionCalls = $nodeFinder->findInstanceOf($ast, Node\Expr\FuncCall::class); + // NodeFinder performs a complete recursive walk for every requested + // node type. Collect the same category groups in one depth-first walk, + // then process them in the historical order so dependency/header order + // and stable-ID allocation remain reproducible. + $groups = array_fill(0, 13, []); + $nodes = is_array($ast) ? array_reverse(array_values($ast)) : [$ast]; + while ($nodes !== []) { + $node = array_pop($nodes); + if (!$node instanceof NodeAbstract) { + continue; + } - foreach ($functionCalls as $call) { - if ($call->name instanceof Node\Name) { - // Internal functions do not participate in dependency management - $resolvedName = $call->name->getAttribute('resolvedName') - ?? $call->name->getAttribute('namespacedName') - ?? $call->name; - $funcName = strtolower($resolvedName->toString()); - if (!$this->isInternalFunction($funcName)) { - $this->symbolCallInFile[$this->file][] = $this->getFunctionDependencySymbol($funcName); + $group = match (true) { + $node instanceof Node\Expr\FuncCall => 0, + $node instanceof Node\Expr\ConstFetch => 1, + $node instanceof Node\Expr\StaticCall => 2, + $node instanceof Node\Expr\StaticPropertyFetch => 3, + $node instanceof Node\Expr\ClassConstFetch => 4, + $node instanceof Node\Expr\New_ => 5, + $node instanceof Node\Expr\Instanceof_ => 6, + $node instanceof Node\FunctionLike => 7, + $node instanceof Node\Stmt\Property => 8, + $node instanceof Node\Stmt\ClassConst => 9, + $node instanceof Node\Stmt\Catch_ => 10, + $node instanceof Node\Attribute => 11, + $node instanceof Node\Stmt\Global_ => 12, + default => null, + }; + if ($group !== null) { + $groups[$group][] = $node; + } + + $children = []; + foreach ($node->getSubNodeNames() as $name) { + $child = $node->{$name}; + if ($child instanceof NodeAbstract) { + $children[] = $child; + continue; + } + if (!is_array($child)) { + continue; + } + foreach ($child as $item) { + if ($item instanceof NodeAbstract) { + $children[] = $item; + } } } + for ($index = count($children) - 1; $index >= 0; --$index) { + $nodes[] = $children[$index]; + } } - - foreach ($nodeFinder->findInstanceOf($ast, Node\Expr\ConstFetch::class) as $fetch) { - $resolvedName = $fetch->name->getAttribute('resolvedName') - ?? $fetch->name->getAttribute('namespacedName') - ?? $fetch->name; - $constant = $resolvedName->toString(); - if (!in_array(strtolower($constant), ['true', 'false', 'null'], true)) { - $this->symbolCallInFile[$this->file][] = $this->getConstantDependencySymbol($constant); + foreach ($groups as $nodesInGroup) { + foreach ($nodesInGroup as $node) { + $this->recordSymbolUsingNode($node); } } + $this->deduplicateCurrentFileSymbolDependencies(); + } - $depClasses = []; - $depClasses = array_merge($depClasses, $nodeFinder->findInstanceOf($ast, Node\Expr\StaticCall::class)); - $depClasses = array_merge($depClasses, $nodeFinder->findInstanceOf($ast, Node\Expr\StaticPropertyFetch::class)); - $depClasses = array_merge($depClasses, $nodeFinder->findInstanceOf($ast, Node\Expr\ClassConstFetch::class)); - $depClasses = array_merge($depClasses, $nodeFinder->findInstanceOf($ast, Node\Expr\New_::class)); - foreach ($depClasses as $call) { - if ($call->class instanceof Node\Name) { - $resolvedClass = $call->class->getAttribute('resolvedName') - ?? $call->class->getAttribute('namespacedName') - ?? $call->class; - $className = $resolvedClass->toString(); - if ($className !== 'self' && $className !== 'static') { - $this->symbolCallInFile[$this->file][] = $this->getClassDependencySymbol($className); - } + protected function recordSymbolUsingNode(Node $node): void + { + if ($node instanceof Node\Expr\FuncCall && $node->name instanceof Node\Name) { + // Internal functions do not participate in dependency management + $resolvedName = $node->name->getAttribute('resolvedName') + ?? $node->name->getAttribute('namespacedName') + ?? $node->name; + $funcName = strtolower($resolvedName->toString()); + if (!$this->isInternalFunction($funcName)) { + $this->symbolCallInFile[$this->file][] = $this->getFunctionDependencySymbol($funcName); } - } - foreach ($nodeFinder->findInstanceOf($ast, Node\Expr\Instanceof_::class) as $instanceof) { - if ($instanceof->class instanceof Node\Name) { - $this->recordClassTypeDependency($instanceof->class); + } elseif ($node instanceof Node\Expr\ConstFetch) { + $resolvedName = $node->name->getAttribute('resolvedName') + ?? $node->name->getAttribute('namespacedName') + ?? $node->name; + $constantName = $resolvedName->toString(); + if (!in_array(strtolower($constantName), ['true', 'false', 'null'], true)) { + $this->symbolCallInFile[$this->file][] = $this->getConstantDependencySymbol($constantName); } - } - foreach ($nodeFinder->findInstanceOf($ast, Node\FunctionLike::class) as $functionLike) { - $this->recordClassTypeDependency($functionLike->getReturnType()); - foreach ($functionLike->getParams() as $parameter) { + } elseif (($node instanceof Node\Expr\StaticCall + || $node instanceof Node\Expr\StaticPropertyFetch + || $node instanceof Node\Expr\ClassConstFetch + || $node instanceof Node\Expr\New_) + && $node->class instanceof Node\Name + ) { + $resolvedClass = $node->class->getAttribute('resolvedName') + ?? $node->class->getAttribute('namespacedName') + ?? $node->class; + $className = $resolvedClass->toString(); + if ($className !== 'self' && $className !== 'static') { + $this->symbolCallInFile[$this->file][] = $this->getClassDependencySymbol($className); + } + } elseif ($node instanceof Node\Expr\Instanceof_ && $node->class instanceof Node\Name) { + $this->recordClassTypeDependency($node->class); + } elseif ($node instanceof Node\FunctionLike) { + $this->recordClassTypeDependency($node->getReturnType()); + foreach ($node->getParams() as $parameter) { $this->recordClassTypeDependency($parameter->type); } - } - foreach ($nodeFinder->findInstanceOf($ast, Node\Stmt\Property::class) as $property) { - $this->recordClassTypeDependency($property->type); - } - foreach ($nodeFinder->findInstanceOf($ast, Node\Stmt\ClassConst::class) as $constant) { - $this->recordClassTypeDependency($constant->type); - } - foreach ($nodeFinder->findInstanceOf($ast, Node\Stmt\Catch_::class) as $catch) { - foreach ($catch->types as $type) { + } elseif ($node instanceof Node\Stmt\Property) { + $this->recordClassTypeDependency($node->type); + } elseif ($node instanceof Node\Stmt\ClassConst) { + $this->recordClassTypeDependency($node->type); + } elseif ($node instanceof Node\Stmt\Catch_) { + foreach ($node->types as $type) { $this->recordClassTypeDependency($type); } - } - foreach ($nodeFinder->findInstanceOf($ast, Node\Attribute::class) as $attribute) { - $this->recordClassTypeDependency($attribute->name); - } - foreach ($nodeFinder->findInstanceOf($ast, Node\Stmt\Global_::class) as $global) { - foreach ($global->vars as $variable) { - if ($variable instanceof Node\Expr\Variable && is_string($variable->name)) { - $this->symbolCallInFile[$this->file][] = $this->getGlobalDependencySymbol( - $this->escapeVarName($variable->name), - ); + } elseif ($node instanceof Node\Attribute) { + $this->recordClassTypeDependency($node->name); + } elseif ($node instanceof Node\Stmt\Global_) { + foreach ($node->vars as $variable) { + if (!$variable instanceof Node\Expr\Variable || !is_string($variable->name)) { + continue; } + $this->symbolCallInFile[$this->file][] = $this->getGlobalDependencySymbol( + $this->escapeVarName($variable->name), + ); } } - // Deduplicate dependencies + } + + private function deduplicateCurrentFileSymbolDependencies(): void + { $this->symbolCallInFile[$this->file] = array_values(array_unique($this->symbolCallInFile[$this->file])); } diff --git a/src/Translator.php b/src/Translator.php index 10ea167f..b82f715c 100644 --- a/src/Translator.php +++ b/src/Translator.php @@ -1179,10 +1179,14 @@ class Translator extends Preprocessor sort($this->registerSymbols, SORT_STRING); sort($this->releaseAstConstantFns, SORT_STRING); $this->localHeaders = $this->argInfoHeaderFiles; + // genExtension() is also a public code-generation entry used directly + // by tooling and tests, outside SourcePipelineTrait::convert(). Keep the + // whole-program property-default invariant local to the consumer too. + $this->finalizeRequestArrayDefaultMetadata(); $this->genClassCeList(); $this->indentLevel++; - $code = $this->genIncludeHeaderFiles(true); + $code = $this->genExtensionIncludeHeaderFiles(); // Only the generated module entry allocates request-cache storage. // Keep out of the shared PCH dependency set used by every source. $code .= '#include ' . PHP_EOL; @@ -1909,11 +1913,16 @@ CODE; private function hashGeneratedCompileInput(\HashContext $context, string $file, array &$visited): void { $real = realpath($file); - if ($real === false || isset($visited[$real])) { + // Generated files can be replaced atomically while a parallel build is + // finishing. realpath() may therefore succeed immediately before the + // old directory entry disappears. Treat that race like a cache miss: + // an incomplete content graph cannot match metadata written from the + // complete graph. + if ($real === false || !is_file($real) || isset($visited[$real])) { return; } $visited[$real] = true; - $contents = file_get_contents($real); + $contents = @file_get_contents($real); if (!is_string($contents)) { return; } @@ -1957,11 +1966,12 @@ CODE; || trim($key) !== $this->getGeneratedObjectCacheKey($sourceFile, $objectFile)) { return false; } - $objectMtime = filemtime($objectFile); - $sourceMtime = filemtime($sourceFile); - return $objectMtime !== false - && $sourceMtime !== false - && $objectMtime >= $sourceMtime; + // The key covers the compile command, source contents, and all generated + // project headers recursively. A regenerated file may have a newer + // timestamp while retaining byte-identical contents; rejecting that + // object would turn a generator-fingerprint change into a full native + // rebuild and defeat the content-addressed cache. + return true; } private function writeGeneratedObjectCacheMetadata(string $sourceFile, string $objectFile): void @@ -2425,6 +2435,7 @@ CODE; $progress->renderInPlace($absoluteCompleted, $totalFiles, 'Compiling'); } }, + true, ); if (!$this->noProgress) { @@ -2475,15 +2486,21 @@ CODE; return false; } } - $metadata = $targetFile . '.typephp-link-cache'; + $metadata = $this->getLinkCacheMetadataFile($targetFile); $key = is_file($metadata) ? file_get_contents($metadata) : false; return is_string($key) && trim($key) === $this->getLinkCacheKey($objectFiles, $targetFile); } + private function getLinkCacheMetadataFile(string $targetFile): string + { + return $this->getBuildDir() . DIRECTORY_SEPARATOR + . basename($targetFile) . '.typephp-link-cache'; + } + /** @param list $objectFiles */ private function writeLinkCache(array $objectFiles, string $targetFile): void { - $metadata = $targetFile . '.typephp-link-cache'; + $metadata = $this->getLinkCacheMetadataFile($targetFile); if (file_put_contents($metadata, $this->getLinkCacheKey($objectFiles, $targetFile) . PHP_EOL) === false) { throw new \RuntimeException('Cannot write link cache metadata: ' . $metadata); } @@ -2992,7 +3009,73 @@ CODE; public function genIncludeHeaderFiles(bool $allDeclarations = false): string { - $globalHeaders = $this->isNanoMode() + $globalHeaders = $this->getGeneratedSourceGlobalHeaders(); + if ($this->declarationHeaderFiles === []) { + $declarationHeaders = [ + "php_{$this->targetName}_func_decl.h", + "php_{$this->targetName}_data_decl.h", + ]; + } elseif ($allDeclarations) { + $declarationHeaders = [ + $this->getRuntimeDeclarationHeaderName(), + ...array_values($this->declarationHeaderFiles), + ]; + } else { + $declarationHeaders = [ + $this->getRuntimeDeclarationHeaderName(), + ...$this->getDeclarationHeadersForFile($this->file), + ]; + } + return $this->renderIncludeHeaderFiles([ + ...$globalHeaders, + ...$declarationHeaders, + ...$this->localHeaders, + ]); + } + + /** + * The module entry aggregates Zend arginfo and registration helpers, but it + * does not call every compiled php_* function. Pulling every per-source + * declaration into this translation unit made an unrelated declaration + * change invalidate the large extension object and greatly increased C++ + * parsing work. Project-wide runtime storage is declared by runtime_decl; + * the generated arginfo headers provide the Zend-facing symbols themselves. + */ + private function genExtensionIncludeHeaderFiles(): string + { + if ($this->declarationHeaderFiles === []) { + $declarationHeaders = [ + "php_{$this->targetName}_func_decl.h", + "php_{$this->targetName}_data_decl.h", + ]; + } else { + $declarationHeaders = [$this->getRuntimeDeclarationHeaderName()]; + // Generated arginfo registration helpers call compile-time + // attribute factories directly to materialize lazy values such as + // enum cases. Include only the declaration owners of those helper + // functions instead of every PHP declaration header. + foreach ($this->symbols->functions() as $functionDef) { + if (!$functionDef->attributeFactory) { + continue; + } + $header = $this->declarationHeaderFiles[$functionDef->sourceFile] ?? null; + if ($header !== null) { + $declarationHeaders[] = $header; + } + } + } + + return $this->renderIncludeHeaderFiles([ + ...$this->getGeneratedSourceGlobalHeaders(), + ...$declarationHeaders, + ...$this->localHeaders, + ]); + } + + /** @return list */ + private function getGeneratedSourceGlobalHeaders(): array + { + return $this->isNanoMode() ? [ 'cstring', 'phpx.h', @@ -3014,23 +3097,11 @@ CODE; 'std/random.h', ] : $this->globalHeaders; - if ($this->declarationHeaderFiles === []) { - $declarationHeaders = [ - "php_{$this->targetName}_func_decl.h", - "php_{$this->targetName}_data_decl.h", - ]; - } elseif ($allDeclarations) { - $declarationHeaders = [ - $this->getRuntimeDeclarationHeaderName(), - ...array_values($this->declarationHeaderFiles), - ]; - } else { - $declarationHeaders = [ - $this->getRuntimeDeclarationHeaderName(), - ...$this->getDeclarationHeadersForFile($this->file), - ]; - } - $headers = array_merge($globalHeaders, $declarationHeaders, $this->localHeaders); + } + + /** @param list $headers */ + private function renderIncludeHeaderFiles(array $headers): string + { $headers = array_values(array_unique($headers)); $lines = []; foreach ($headers as $header) { @@ -3065,6 +3136,31 @@ CODE; return 'typephp_ensure_request_array_defaults_' . $classDef->getNamespacedName(); } + /** + * Finalize whole-program metadata required to materialize non-empty array + * property defaults. This cannot be a side effect of per-file conversion: + * an incrementally reused file does not run genClassWrapper(), while the + * regenerated extension source still needs the custom object allocator. + */ + protected function finalizeRequestArrayDefaultMetadata(): void + { + foreach ($this->symbols->classes() as $classDef) { + if ($classDef->trait !== null || $classDef->nativeObject) { + continue; + } + foreach ($classDef->properties as $property) { + if ($property->isStatic() || !$property->requiresRuntimeDefaultInit) { + continue; + } + $property->runtimeDefaultOffset = $this->getPropertyOffset( + $classDef->getNamespacedName(false), + $property->name, + ); + $classDef->requireCtor = true; + } + } + } + private function getRequestArrayDefaultTemplateName(ClassDef $classDef, PropertyDef $property): string { return 'typephp_request_array_default_' @@ -5730,19 +5826,6 @@ CODE; if ($classDef->nativeObject) { return ''; } - $defaultPropCount = 0; - foreach ($classDef->properties as $property) { - if (!$property->isStatic() && $property->requiresRuntimeDefaultInit) { - $property->runtimeDefaultOffset = $this->getPropertyOffset( - $classDef->getNamespacedName(false), - $property->name, - ); - $defaultPropCount++; - } - } - if ($defaultPropCount > 0) { - $classDef->requireCtor = true; - } $methods = $classDef->methods; foreach ($methods as $methodDef) { if ($this->functionUsesNativeObject($methodDef->functionDef)) { diff --git a/src/compiler.php b/src/compiler.php index 5fe149ff..649a6742 100644 --- a/src/compiler.php +++ b/src/compiler.php @@ -20,6 +20,13 @@ function main(int $argc, array $argv): void if (!defined('TYPEPHP_DEBUG')) { define('TYPEPHP_DEBUG', true); } + if (!defined('TYPEPHP_COMPILER_EXECUTABLE')) { + $compilerExecutable = realpath($argv[0]); + define( + 'TYPEPHP_COMPILER_EXECUTABLE', + $compilerExecutable !== false ? $compilerExecutable : $argv[0], + ); + } // The Zend PHP entrypoint loads the consumer project's Composer autoloader // in bin/bootstrap.php. The AOT compiler starts here directly and therefore @@ -44,7 +51,13 @@ function main(int $argc, array $argv): void return; } - if (shouldCompileNativeSourceProject($argv)) { + try { + $nativeSourceProject = shouldCompileNativeSourceProject($argv); + } catch (RuntimeException $exception) { + fwrite(STDERR, 'Native source build failed: ' . $exception->getMessage() . "\n"); + exit(1); + } + if ($nativeSourceProject) { compileNativeSourceProject($argv); return; }