diff --git a/phpunit/src/Build/IncrementalBuildStateTest.php b/phpunit/src/Build/IncrementalBuildStateTest.php new file mode 100644 index 00000000..7f89f39d --- /dev/null +++ b/phpunit/src/Build/IncrementalBuildStateTest.php @@ -0,0 +1,112 @@ +directory = sys_get_temp_dir() . '/typephp_build_graph_' . bin2hex(random_bytes(8)); + mkdir($this->directory, 0777, true); + $this->stateFile = $this->directory . '/build-state.json'; + } + + protected function tearDown(): void + { + if (is_file($this->stateFile)) { + unlink($this->stateFile); + } + rmdir($this->directory); + } + + public function testChangedDependencyPropagatesThroughCyclesAndConsumers(): void + { + $this->saveBaseline(); + $state = new IncrementalBuildState($this->stateFile); + + $dirty = $state->dirtyFiles( + ['A.php' => 'a', 'B.php' => 'b2', 'C.php' => 'c', 'D.php' => 'd'], + [ + 'A.php' => ['B.php'], + 'B.php' => ['A.php'], + 'C.php' => ['A.php'], + 'D.php' => [], + ], + 'generator', + static fn(string $_source, ?array $_metadata): bool => true, + ); + + self::assertSame(['A.php', 'B.php', 'C.php'], array_keys($dirty)); + } + + public function testRemovedDependencyInvalidatesItsPreviousConsumers(): void + { + $this->saveBaseline(); + $state = new IncrementalBuildState($this->stateFile); + + $dirty = $state->dirtyFiles( + ['B.php' => 'b', 'C.php' => 'c', 'D.php' => 'd'], + ['B.php' => [], 'C.php' => [], 'D.php' => []], + 'generator', + static fn(string $_source, ?array $_metadata): bool => true, + ); + + self::assertSame(['B.php', 'C.php'], array_keys($dirty)); + } + + public function testMissingArtifactAndGeneratorChangeInvalidateExpectedFiles(): void + { + $this->saveBaseline(); + $state = new IncrementalBuildState($this->stateFile); + $hashes = ['A.php' => 'a', 'B.php' => 'b', 'C.php' => 'c', 'D.php' => 'd']; + $dependencies = [ + 'A.php' => ['B.php'], + 'B.php' => ['A.php'], + 'C.php' => ['A.php'], + 'D.php' => [], + ]; + + $dirty = $state->dirtyFiles( + $hashes, + $dependencies, + 'generator', + static fn(string $source, ?array $_metadata): bool => $source !== 'A.php', + ); + self::assertSame(['A.php', 'B.php', 'C.php'], array_keys($dirty)); + + $dirty = $state->dirtyFiles( + $hashes, + $dependencies, + 'new-generator', + static fn(string $_source, ?array $_metadata): bool => true, + ); + self::assertSame(array_keys($hashes), array_keys($dirty)); + } + + private function saveBaseline(): void + { + $state = new IncrementalBuildState($this->stateFile); + $state->save('generator', [ + 'A.php' => $this->metadata('a', ['B.php']), + 'B.php' => $this->metadata('b', ['A.php']), + 'C.php' => $this->metadata('c', ['A.php']), + 'D.php' => $this->metadata('d', []), + ]); + } + + /** @param list $dependencies @return array */ + private function metadata(string $hash, array $dependencies): array + { + return [ + 'hash' => $hash, + 'dependencies' => $dependencies, + 'emitsTranslationUnit' => true, + ]; + } +} diff --git a/phpunit/src/Build/IncrementalDeclarationTest.php b/phpunit/src/Build/IncrementalDeclarationTest.php new file mode 100644 index 00000000..62d2f5aa --- /dev/null +++ b/phpunit/src/Build/IncrementalDeclarationTest.php @@ -0,0 +1,290 @@ +directory = sys_get_temp_dir() . '/typephp_incremental_' . bin2hex(random_bytes(8)); + $this->buildDirectory = $this->directory . '/build'; + mkdir($this->directory, 0777, true); + $this->provider = $this->directory . '/provider.php'; + $this->consumer = $this->directory . '/consumer.php'; + $this->independent = $this->directory . '/independent.php'; + file_put_contents($this->provider, <<<'PHP' +independent, <<<'PHP' +consumer, <<<'PHP' +removeDirectory($this->directory); + } + + public function testDeclarationsAreSplitBySourceAndDependenciesAreIncluded(): void + { + $compiler = $this->convertProject(); + $providerHeader = $compiler->getDeclarationHeaderFile($this->provider); + $consumerHeader = $compiler->getDeclarationHeaderFile($this->consumer); + $consumerCpp = $this->invoke($compiler, 'getCppFile', $this->consumer); + + self::assertFileExists($providerHeader); + self::assertFileExists($consumerHeader); + self::assertFileExists($this->buildDirectory . '/include/php_incremental_runtime_decl.h'); + self::assertFileExists($this->buildDirectory . '/include/php_incremental_all_decl.h'); + self::assertFileDoesNotExist($this->buildDirectory . '/include/php_incremental_func_decl.h'); + self::assertFileDoesNotExist($this->buildDirectory . '/include/php_incremental_data_decl.h'); + + $providerDeclarations = (string) file_get_contents($providerHeader); + $consumerDeclarations = (string) file_get_contents($consumerHeader); + $runtimeDeclarations = (string) file_get_contents( + $this->buildDirectory . '/include/php_incremental_runtime_decl.h', + ); + $consumerCode = (string) file_get_contents($consumerCpp); + self::assertStringContainsString('php_incremental__answer(', $providerDeclarations); + self::assertStringContainsString('_const_var_Incremental__LIMIT', $providerDeclarations); + self::assertStringContainsString('_global_var_shared', $providerDeclarations); + self::assertStringNotContainsString('_global_var_shared', $consumerDeclarations); + self::assertStringNotContainsString('_global_var_shared', $runtimeDeclarations); + self::assertStringNotContainsString('php_main(', $providerDeclarations); + self::assertStringContainsString('php_main(', $consumerDeclarations); + self::assertStringContainsString( + '#include <' . basename($providerHeader) . '>', + $consumerDeclarations, + ); + self::assertStringContainsString( + '#include <' . basename($consumerHeader) . '>', + $consumerCode, + ); + + $symbols = $this->property($compiler, 'symbolDeclInFile'); + self::assertSame($this->provider, $symbols['function:incremental\\answer']); + self::assertSame($this->provider, $symbols['constant:Incremental\\LIMIT']); + } + + public function testUnchangedGeneratedCppKeepsItsTimestamp(): void + { + $first = $this->convertProject(); + $consumerCpp = $this->invoke($first, 'getCppFile', $this->consumer); + $oldTimestamp = 1_600_000_000; + touch($consumerCpp, $oldTimestamp); + clearstatcache(true, $consumerCpp); + + $this->convertProject(); + + clearstatcache(true, $consumerCpp); + self::assertSame($oldTimestamp, filemtime($consumerCpp)); + self::assertStringContainsString( + 'get_str(uint32_t index)', + (string) file_get_contents($this->buildDirectory . '/include/php_incremental_runtime_decl.h'), + ); + } + + public function testChangedDeclarationRegeneratesItsTransitiveConsumersOnly(): void + { + $first = $this->convertProject(); + $providerCpp = $this->invoke($first, 'getCppFile', $this->provider); + $consumerCpp = $this->invoke($first, 'getCppFile', $this->consumer); + $independentCpp = $this->invoke($first, 'getCppFile', $this->independent); + $oldTimestamp = 1_600_000_000; + foreach ([$providerCpp, $consumerCpp, $independentCpp] as $cppFile) { + touch($cppFile, $oldTimestamp); + } + file_put_contents($this->provider, "\n// changed provider\n", FILE_APPEND); + clearstatcache(); + + $this->convertProject(); + + clearstatcache(); + self::assertGreaterThan($oldTimestamp, filemtime($providerCpp)); + self::assertGreaterThan($oldTimestamp, filemtime($consumerCpp)); + self::assertSame($oldTimestamp, filemtime($independentCpp)); + } + + public function testGeneratedObjectCacheRequiresAnObjectNotOlderThanItsCppSource(): void + { + $compiler = $this->convertProject(); + $consumerCpp = $this->invoke($compiler, 'getCppFile', $this->consumer); + $consumerObject = $compiler->getObjectFile($consumerCpp); + $sourceTimestamp = 1_600_000_000; + $objectTimestamp = $sourceTimestamp + 10; + + file_put_contents($consumerObject, 'object'); + touch($consumerCpp, $sourceTimestamp); + touch($consumerObject, $objectTimestamp); + clearstatcache(); + $this->invoke($compiler, 'writeGeneratedObjectCacheMetadata', $consumerCpp, $consumerObject); + + self::assertTrue( + $this->invoke($compiler, 'hasGeneratedObjectFileCache', $consumerCpp, $consumerObject), + ); + + touch($consumerCpp, $objectTimestamp); + clearstatcache(); + self::assertTrue( + $this->invoke($compiler, 'hasGeneratedObjectFileCache', $consumerCpp, $consumerObject), + ); + + touch($consumerCpp, $objectTimestamp + 1); + clearstatcache(); + self::assertFalse( + $this->invoke($compiler, 'hasGeneratedObjectFileCache', $consumerCpp, $consumerObject), + ); + + touch($consumerCpp, $sourceTimestamp); + unlink($consumerObject); + clearstatcache(); + self::assertFalse( + $this->invoke($compiler, 'hasGeneratedObjectFileCache', $consumerCpp, $consumerObject), + ); + + file_put_contents($consumerObject, 'object'); + touch($consumerObject, $objectTimestamp); + unlink($consumerCpp); + clearstatcache(); + self::assertFalse( + $this->invoke($compiler, 'hasGeneratedObjectFileCache', $consumerCpp, $consumerObject), + ); + } + + public function testLinkCacheRequiresEveryObjectAndAnUpToDateTarget(): void + { + $compiler = $this->newCompiler(); + $target = $this->directory . '/incremental-bin'; + $firstObject = $this->directory . '/first.o'; + $secondObject = $this->directory . '/second.o'; + $timestamp = 1_600_000_000; + foreach ([$target, $firstObject, $secondObject] as $file) { + file_put_contents($file, 'artifact'); + touch($file, $timestamp); + } + clearstatcache(); + + $objects = [$firstObject, $secondObject]; + $this->invoke($compiler, 'writeLinkCache', $objects, $target); + self::assertTrue($this->invoke($compiler, 'hasLinkCache', $objects, $target)); + + touch($secondObject, $timestamp + 1); + clearstatcache(); + self::assertFalse($this->invoke($compiler, 'hasLinkCache', $objects, $target)); + + unlink($secondObject); + clearstatcache(); + self::assertFalse($this->invoke($compiler, 'hasLinkCache', $objects, $target)); + } + + public function testForceCacheClearRemovesAstAndTargetIncrementalState(): void + { + $compiler = $this->newCompiler(); + $astDirectory = $this->buildDirectory . '/cache/ast'; + $incrementalDirectory = $this->buildDirectory . '/cache/incremental/incremental'; + mkdir($astDirectory, 0777, true); + mkdir($incrementalDirectory, 0777, true); + file_put_contents($astDirectory . '/entry.ast', 'ast'); + file_put_contents($incrementalDirectory . '/stable-ids.json', '{}'); + + $this->invoke($compiler, 'clearIncrementalBuildCache'); + + self::assertDirectoryDoesNotExist($astDirectory); + self::assertDirectoryDoesNotExist($incrementalDirectory); + } + + private function convertProject(): CompilerTest + { + global $translator; + + $compiler = $this->newCompiler(); + $translator = $compiler; + $files = [$this->provider, $this->consumer, $this->independent]; + $compiler->addFiles([$this->directory]); + foreach ($files as $file) { + $compiler->prepareFile($file); + } + $files = $compiler->getSortedFiles($files); + $this->invoke($compiler, 'initializeIncrementalCompilation', $files); + $compiler->convert($files); + return $compiler; + } + + private function newCompiler(): CompilerTest + { + $compiler = CompilerTest::create($this->directory); + // Exercise the same on-disk stable-ID registry used by real builds. + $reflection = new ReflectionClass($compiler); + $forTest = $reflection->getProperty('forTest'); + $forTest->setAccessible(true); + $forTest->setValue($compiler, false); + $compiler->setTargetName('incremental'); + $this->invoke($compiler, 'setBuildDir', $this->buildDirectory); + return $compiler; + } + + private function invoke(CompilerTest $compiler, string $method, mixed ...$arguments): mixed + { + $reflection = new ReflectionClass($compiler); + $methodReflection = $reflection->getMethod($method); + $methodReflection->setAccessible(true); + return $methodReflection->invoke($compiler, ...$arguments); + } + + private function property(CompilerTest $compiler, string $name): mixed + { + $reflection = new ReflectionClass($compiler); + $property = $reflection->getProperty($name); + $property->setAccessible(true); + return $property->getValue($compiler); + } + + private function removeDirectory(string $directory): void + { + if (!is_dir($directory)) { + return; + } + $iterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($directory, \FilesystemIterator::SKIP_DOTS), + \RecursiveIteratorIterator::CHILD_FIRST, + ); + foreach ($iterator as $entry) { + $entry->isDir() ? rmdir($entry->getPathname()) : unlink($entry->getPathname()); + } + rmdir($directory); + } +} diff --git a/phpunit/src/Build/StableIdRegistryTest.php b/phpunit/src/Build/StableIdRegistryTest.php new file mode 100644 index 00000000..a9c2227f --- /dev/null +++ b/phpunit/src/Build/StableIdRegistryTest.php @@ -0,0 +1,65 @@ +directory = sys_get_temp_dir() . '/typephp_stable_ids_' . bin2hex(random_bytes(8)); + $this->cacheFile = $this->directory . '/stable-ids.json'; + } + + protected function tearDown(): void + { + if (is_file($this->cacheFile)) { + unlink($this->cacheFile); + } + if (is_dir($this->directory)) { + rmdir($this->directory); + } + } + + public function testIdsRemainStableAcrossCompilerProcesses(): void + { + $first = new StableIdRegistry($this->cacheFile); + self::assertSame(0, $first->allocate('literal', "first\0value")); + self::assertSame(1, $first->allocate('literal', 'second')); + self::assertSame(0, $first->allocate('class', 'Example')); + $first->flush(); + + $second = new StableIdRegistry($this->cacheFile); + self::assertSame(1, $second->allocate('literal', 'second')); + self::assertSame(0, $second->allocate('literal', "first\0value")); + self::assertSame(2, $second->allocate('literal', 'third')); + self::assertSame(3, $second->capacity('literal')); + self::assertSame([ + "first\0value" => 0, + 'second' => 1, + 'third' => 2, + ], $second->entries('literal')); + } + + public function testInvalidPersistedCursorCannotReuseAnExistingId(): void + { + mkdir($this->directory, 0777, true); + file_put_contents($this->cacheFile, json_encode([ + 'schema' => 2, + 'domains' => [ + 'function' => ['b:' . base64_encode('old') => 7], + ], + 'nextIds' => ['function' => 1], + ], JSON_THROW_ON_ERROR)); + + $registry = new StableIdRegistry($this->cacheFile); + + self::assertSame(7, $registry->allocate('function', 'old')); + self::assertSame(8, $registry->allocate('function', 'new')); + } +} diff --git a/phpunit/src/CompilerBaseApiTest.php b/phpunit/src/CompilerBaseApiTest.php index 48a1450e..1f38e2d0 100644 --- a/phpunit/src/CompilerBaseApiTest.php +++ b/phpunit/src/CompilerBaseApiTest.php @@ -1833,7 +1833,7 @@ YAML); $options = $this->invokeMethod('getCompileCommandOptions'); $this->assertContains('TYPEPHP_ABI_DEFAULTS_EXPORTS=1', $options['user_defines']); - $this->assertStringEndsWith('/php_abi_defaults_func_decl.h', $options['forced_include']); + $this->assertStringEndsWith('/php_abi_defaults_all_decl.h', $options['forced_include']); if (!$this->compiler->isWindows()) { $flags = $this->getPropertyValue('compilerBackend')->buildCompileOptions($options->toArray()); $this->assertStringContainsString('-fvisibility=hidden', $flags); diff --git a/phpunit/src/PreprocessorTest.php b/phpunit/src/PreprocessorTest.php index 3cd867b0..b4eef057 100644 --- a/phpunit/src/PreprocessorTest.php +++ b/phpunit/src/PreprocessorTest.php @@ -272,6 +272,23 @@ class PreprocessorTest extends TestCase $this->assertIsArray($files); } + public function testSortFilesAllowsCircularFunctionDependencies(): void + { + $this->setProperty('symbolDeclInFile', [ + 'function:a' => '/a.php', + 'function:b' => '/b.php', + ]); + $this->setProperty('symbolCallInFile', [ + '/a.php' => ['function:b'], + '/b.php' => ['function:a'], + '/consumer.php' => ['function:a'], + ]); + + $files = $this->invokeMethod('getSortedFiles', ['/consumer.php', '/a.php', '/b.php']); + + $this->assertSame(['/b.php', '/a.php', '/consumer.php'], $files); + } + public function testPrepareFileParsesInterfaceMembersAndTypeChecks(): void { $file = __DIR__ . '/../code/preprocessor/interface_members.php'; diff --git a/project.yml b/project.yml index 3b560d4b..aeb48562 100644 --- a/project.yml +++ b/project.yml @@ -1,6 +1,6 @@ name: tpc build-mode: bin -version: 0.8.2 +version: 0.9.0 cxx-std: c++17 cxx-flags: - -Wall @@ -12,8 +12,8 @@ resource: icon: swoole-logo.ico # 版本信息 version-info: - file-version: 0.8.2.1112 - product-version: 0.8.2 + file-version: 0.9.0.1112 + product-version: 0.9.0 company-name: "上海识沃网络科技有限公司" file-description: "TypePHP Compiler" internal-name: "typephp" diff --git a/src/Analysis/CompilationStatistics.php b/src/Analysis/CompilationStatistics.php index fec5d05e..e8cbefc9 100644 --- a/src/Analysis/CompilationStatistics.php +++ b/src/Analysis/CompilationStatistics.php @@ -59,6 +59,41 @@ final class CompilationStatistics return isset($this->counters[$category][$name]); } + /** @param array $counters */ + public function merge(array $counters): void + { + foreach ($counters as $category => $values) { + if (!is_string($category) || !is_array($values)) { + continue; + } + foreach ($values as $name => $count) { + if (is_string($name) && is_int($count) && $count > 0) { + $this->counters[$category][$name] = ($this->counters[$category][$name] ?? 0) + $count; + } + } + } + } + + /** + * Return counters recorded since an earlier all() snapshot. + * + * @param array> $before + * @return array> + */ + public function delta(array $before): array + { + $delta = []; + foreach ($this->counters as $category => $values) { + foreach ($values as $name => $count) { + $difference = $count - ($before[$category][$name] ?? 0); + if ($difference > 0) { + $delta[$category][$name] = $difference; + } + } + } + return $delta; + } + /** @return array> */ public function all(): array { diff --git a/src/Build/IncrementalBuildState.php b/src/Build/IncrementalBuildState.php new file mode 100644 index 00000000..0f77b55f --- /dev/null +++ b/src/Build/IncrementalBuildState.php @@ -0,0 +1,189 @@ +> */ + private array $files = []; + private string $generatorFingerprint = ''; + + public function __construct(private readonly string $file) + { + $this->restore(); + } + + /** @return null|array */ + public function metadata(string $source): ?array + { + return $this->files[$source] ?? null; + } + + /** + * @param array $sourceHashes + * @param array> $dependencies + * @param callable(string, ?array): bool $artifactsExist + * @return array + */ + public function dirtyFiles( + array $sourceHashes, + array $dependencies, + string $generatorFingerprint, + callable $artifactsExist, + bool $force = false, + ): array { + $currentFiles = array_fill_keys(array_keys($sourceHashes), true); + if ($force || $this->generatorFingerprint !== $generatorFingerprint) { + return $currentFiles; + } + + $dirty = []; + foreach ($sourceHashes as $source => $hash) { + $metadata = $this->files[$source] ?? null; + $previousDependencies = is_array($metadata['dependencies'] ?? null) + ? array_values(array_filter($metadata['dependencies'], 'is_string')) + : []; + $currentDependencies = $dependencies[$source] ?? []; + sort($previousDependencies, SORT_STRING); + sort($currentDependencies, SORT_STRING); + if (!is_array($metadata) + || ($metadata['hash'] ?? null) !== $hash + || $previousDependencies !== $currentDependencies + || !$artifactsExist($source, $metadata)) { + $dirty[$source] = true; + } + } + + // A removed source is a dirty dependency node as well. Consumers found + // through the previous graph must be regenerated so a removed/moved + // declaration cannot remain embedded in an old translation unit. + foreach ($this->files as $source => $_metadata) { + if (!isset($currentFiles[$source])) { + $dirty[$source] = true; + } + } + + $reverse = []; + foreach ([$this->previousDependencies(), $dependencies] as $graph) { + foreach ($graph as $consumer => $requirements) { + foreach ($requirements as $requirement) { + $reverse[$requirement][$consumer] = true; + } + } + } + + $queue = array_keys($dirty); + for ($offset = 0; isset($queue[$offset]); ++$offset) { + $dependency = $queue[$offset]; + foreach ($reverse[$dependency] ?? [] as $consumer => $_) { + if (isset($dirty[$consumer])) { + continue; + } + $dirty[$consumer] = true; + $queue[] = $consumer; + } + } + + $dirty = array_intersect_key($dirty, $currentFiles); + ksort($dirty, SORT_STRING); + return $dirty; + } + + /** + * @param array> $files + */ + public function save(string $generatorFingerprint, array $files): void + { + if ($this->file === '') { + $this->generatorFingerprint = $generatorFingerprint; + $this->files = $files; + return; + } + ksort($files, SORT_STRING); + $contents = json_encode([ + 'schema' => self::SCHEMA_VERSION, + 'generatorFingerprint' => $generatorFingerprint, + 'files' => $files, + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . PHP_EOL; + $directory = dirname($this->file); + if (!is_dir($directory) && !mkdir($directory, 0777, true) && !is_dir($directory)) { + throw new \RuntimeException('Cannot create incremental build state directory: ' . $directory); + } + $temporary = tempnam($directory, '.graph-'); + if ($temporary === false) { + throw new \RuntimeException('Cannot create incremental build state temporary file'); + } + try { + if (file_put_contents($temporary, $contents, LOCK_EX) === false) { + throw new \RuntimeException('Cannot write incremental build state: ' . $this->file); + } + if (!@rename($temporary, $this->file)) { + @unlink($this->file); + if (!@rename($temporary, $this->file)) { + throw new \RuntimeException('Cannot write incremental build state: ' . $this->file); + } + } + } finally { + if (is_file($temporary)) { + @unlink($temporary); + } + } + $this->generatorFingerprint = $generatorFingerprint; + $this->files = $files; + } + + /** @return array> */ + private function previousDependencies(): array + { + $dependencies = []; + foreach ($this->files as $source => $metadata) { + $values = is_array($metadata['dependencies'] ?? null) + ? array_values(array_filter($metadata['dependencies'], 'is_string')) + : []; + $dependencies[$source] = $values; + } + return $dependencies; + } + + private function restore(): void + { + if ($this->file === '' || !is_file($this->file)) { + return; + } + try { + $contents = file_get_contents($this->file); + $payload = is_string($contents) + ? json_decode($contents, true, flags: JSON_THROW_ON_ERROR) + : null; + if (!is_array($payload) + || ($payload['schema'] ?? null) !== self::SCHEMA_VERSION + || !is_string($payload['generatorFingerprint'] ?? null) + || !is_array($payload['files'] ?? null)) { + return; + } + foreach ($payload['files'] as $source => $metadata) { + if (!is_string($source) + || !is_array($metadata) + || !is_string($metadata['hash'] ?? null) + || !is_array($metadata['dependencies'] ?? null)) { + return; + } + } + $this->generatorFingerprint = $payload['generatorFingerprint']; + $this->files = $payload['files']; + } catch (\Throwable) { + $this->files = []; + $this->generatorFingerprint = ''; + } + } +} diff --git a/src/Build/IncrementalCompilationTrait.php b/src/Build/IncrementalCompilationTrait.php new file mode 100644 index 00000000..555e0419 --- /dev/null +++ b/src/Build/IncrementalCompilationTrait.php @@ -0,0 +1,357 @@ + */ + private array $incrementalSourceHashes = []; + /** @var array> */ + private array $incrementalDependencies = []; + /** @var array */ + private array $incrementalDirtyFiles = []; + /** @var array> */ + private array $incrementalFileMetadata = []; + /** @var array */ + private array $incrementalTranslationUnits = []; + private string $incrementalGeneratorFingerprint = ''; + private bool $incrementalPlanInitialized = false; + + /** @param list $files */ + protected function initializeIncrementalCompilation(array $files): void + { + $this->incrementalPlanInitialized = false; + $this->incrementalFileMetadata = []; + $this->incrementalTranslationUnits = []; + $this->incrementalDirtyFiles = []; + + $phpFiles = []; + foreach ($files as $file) { + if (!FileScanner::isPhpFile($file)) { + continue; + } + $path = realpath($file); + if ($path !== false) { + $phpFiles[] = $path; + } + } + sort($phpFiles, SORT_STRING); + + $this->incrementalSourceHashes = []; + foreach ($phpFiles as $file) { + $hash = hash_file('sha256', $file); + if (!is_string($hash)) { + throw new \RuntimeException('Cannot hash PHP source: ' . $file); + } + $this->incrementalSourceHashes[$file] = $hash; + } + $state = $this->getIncrementalBuildState(); + $this->restoreGlobalDependencyOwnersForPlanning($phpFiles, $state); + $this->incrementalDependencies = $this->buildPhpFileDependencyGraph($phpFiles); + $this->incrementalGeneratorFingerprint = $this->getIncrementalGeneratorFingerprint(); + $this->incrementalDirtyFiles = $state->dirtyFiles( + $this->incrementalSourceHashes, + $this->incrementalDependencies, + $this->incrementalGeneratorFingerprint, + function (string $source, ?array $metadata): bool { + if (!is_array($metadata) + || !is_file($this->getDeclarationHeaderFile($source)) + || !is_file($this->getArgInfoHeaderFile($source))) { + return false; + } + $emits = $metadata['emitsTranslationUnit'] ?? null; + if (!is_bool($emits)) { + return false; + } + return !$emits || is_file($this->getCppFile($source)); + }, + $this->climate->arguments->defined('force'), + ); + $this->incrementalPlanInitialized = true; + } + + protected function shouldRegeneratePhpFile(string $file): bool + { + if (!$this->incrementalPlanInitialized) { + return true; + } + $path = realpath($file) ?: $file; + return isset($this->incrementalDirtyFiles[$path]); + } + + protected function incrementalTranslationUnitWasEmitted(string $file): bool + { + $path = realpath($file) ?: $file; + return $this->incrementalTranslationUnits[$path] + ?? (bool) ($this->getIncrementalBuildState()->metadata($path)['emitsTranslationUnit'] ?? false); + } + + protected function registerGeneratedProjectSource(string $source): void + { + $this->generatedProjectSources[$source] = true; + } + + /** + * Restore conversion outputs which are consumed by whole-program generators. + * + * @param list $files + */ + protected function restoreCleanIncrementalMetadata(array $files): void + { + if (!$this->incrementalPlanInitialized) { + return; + } + foreach ($files as $file) { + if (!FileScanner::isPhpFile($file)) { + continue; + } + $file = realpath($file) ?: $file; + if ($this->shouldRegeneratePhpFile($file)) { + continue; + } + $metadata = $this->getIncrementalBuildState()->metadata($file); + if (!is_array($metadata)) { + continue; + } + $this->incrementalFileMetadata[$file] = $metadata; + $this->incrementalTranslationUnits[$file] = (bool) ($metadata['emitsTranslationUnit'] ?? false); + $statistics = $metadata['statistics'] ?? []; + if (is_array($statistics)) { + $this->compilationStatistics->merge($statistics); + } + $globals = $metadata['globals'] ?? []; + if (is_array($globals)) { + foreach ($globals as $name => $type) { + if (is_string($name) && is_string($type)) { + $this->globalVarsInFile[$file][$name] = $type; + } + } + } + $initializers = $metadata['nativeStaticInitializers'] ?? []; + if (is_array($initializers)) { + foreach ($initializers as $name) { + if (is_string($name)) { + $this->nativeStaticInitializersInFile[$file][$name] = true; + } + } + } + $this->registerExistingArgInfoHeader($file); + } + $this->rebuildIncrementalGlobalState(); + } + + /** @param array> $statistics */ + protected function recordIncrementalConversion( + string $file, + bool $emitsTranslationUnit, + array $statistics, + ): void { + $path = realpath($file) ?: $file; + $this->incrementalTranslationUnits[$path] = $emitsTranslationUnit; + $this->incrementalFileMetadata[$path]['statistics'] = $statistics; + } + + protected function rebuildIncrementalGlobalState(): void + { + foreach ($this->globalVarDeclInFile as $name => $_file) { + unset($this->symbolDeclInFile[$this->getGlobalDependencySymbol($name)]); + } + $this->globalVarDeclInFile = []; + $this->nativeStaticInitializerDeclInFile = []; + foreach ($this->globalVarsInFile as $file => $globals) { + ksort($globals, SORT_STRING); + foreach ($globals as $name => $type) { + if (!isset($this->nativeGlobalObjects[$name])) { + $this->globalVars[$name] = $type; + } + if (!isset($this->globalVarDeclInFile[$name])) { + $this->globalVarDeclInFile[$name] = $file; + $this->symbolDeclInFile[$this->getGlobalDependencySymbol($name)] = $file; + } + $this->symbolCallInFile[$file][] = $this->getGlobalDependencySymbol($name); + } + } + foreach ($this->nativeStaticInitializersInFile as $file => $initializers) { + foreach ($initializers as $name => $_) { + $this->nativeStaticInitializers[$name] = true; + $this->nativeStaticInitializerDeclInFile[$name] = $file; + } + } + } + + /** @param list $files */ + protected function finalizeIncrementalConversionMetadata(array $files): void + { + if (!$this->incrementalPlanInitialized) { + return; + } + $this->rebuildIncrementalGlobalState(); + $phpFiles = []; + foreach ($files as $file) { + if (!FileScanner::isPhpFile($file)) { + continue; + } + $path = realpath($file); + if ($path !== false) { + $phpFiles[] = $path; + } + } + $this->incrementalDependencies = $this->buildPhpFileDependencyGraph($phpFiles); + } + + /** @param list $files */ + protected function saveIncrementalCompilationState(array $files): void + { + if (!$this->incrementalPlanInitialized) { + return; + } + $this->rebuildIncrementalGlobalState(); + $stateFiles = []; + foreach ($files as $file) { + if (!FileScanner::isPhpFile($file)) { + continue; + } + $path = realpath($file); + if ($path === false || !isset($this->incrementalSourceHashes[$path])) { + continue; + } + $previous = $this->incrementalFileMetadata[$path] + ?? $this->getIncrementalBuildState()->metadata($path) + ?? []; + $declared = []; + foreach ($this->symbolDeclInFile as $symbol => $declaringFile) { + if ($declaringFile === $path) { + $declared[] = $symbol; + } + } + sort($declared, SORT_STRING); + $used = array_values(array_unique($this->symbolCallInFile[$path] ?? [])); + sort($used, SORT_STRING); + $dependencies = $this->incrementalDependencies[$path] ?? []; + sort($dependencies, SORT_STRING); + $initializers = array_keys($this->nativeStaticInitializersInFile[$path] ?? []); + sort($initializers, SORT_STRING); + $stateFiles[$path] = [ + 'hash' => $this->incrementalSourceHashes[$path], + 'dependencies' => $dependencies, + 'symbolsDeclared' => $declared, + 'symbolsUsed' => $used, + 'emitsTranslationUnit' => $this->incrementalTranslationUnits[$path] + ?? (bool) ($previous['emitsTranslationUnit'] ?? false), + 'header' => $this->getDeclarationHeaderFile($path), + 'cpp' => $this->getCppFile($path), + 'statistics' => $this->incrementalFileMetadata[$path]['statistics'] + ?? ($previous['statistics'] ?? []), + 'globals' => $this->globalVarsInFile[$path] ?? [], + 'nativeStaticInitializers' => $initializers, + ]; + } + $this->getIncrementalBuildState()->save( + $this->incrementalGeneratorFingerprint, + $stateFiles, + ); + } + + /** @param list $files @return array> */ + private function buildPhpFileDependencyGraph(array $files): array + { + $known = array_fill_keys($files, true); + $dependencies = []; + foreach ($files as $file) { + $requirements = []; + foreach ($this->symbolCallInFile[$file] ?? [] as $symbol) { + $dependency = $this->symbolDeclInFile[$symbol] ?? null; + if (is_string($dependency) + && $dependency !== $file + && isset($known[$dependency])) { + $requirements[$dependency] = true; + } + } + $list = array_keys($requirements); + sort($list, SORT_STRING); + $dependencies[$file] = $list; + } + return $dependencies; + } + + private function getIncrementalBuildState(): IncrementalBuildState + { + $file = $this->getBuildDir() . '/cache/incremental/' . $this->targetName . '/build-state.json'; + if ($this->incrementalBuildState === null || $this->incrementalBuildStateFile !== $file) { + $this->incrementalBuildState = new IncrementalBuildState($file); + $this->incrementalBuildStateFile = $file; + } + return $this->incrementalBuildState; + } + + /** @param list $files */ + private function restoreGlobalDependencyOwnersForPlanning( + array $files, + IncrementalBuildState $state, + ): void { + $knownFiles = array_fill_keys($files, true); + foreach ($files as $file) { + $metadata = $state->metadata($file); + if (!is_array($metadata)) { + continue; + } + foreach ($metadata['symbolsDeclared'] ?? [] as $symbol) { + if (is_string($symbol) && str_starts_with($symbol, 'global:')) { + $this->symbolDeclInFile[$symbol] = $file; + } + } + } + foreach ($files as $file) { + foreach ($this->symbolCallInFile[$file] ?? [] as $symbol) { + if (!str_starts_with($symbol, 'global:') + || isset($this->symbolDeclInFile[$symbol])) { + continue; + } + // A newly introduced global has no previous owner. Assign one + // deterministically for this planning pass; conversion will + // record the actual first declaration owner in sorted order. + $this->symbolDeclInFile[$symbol] = $file; + } + } + foreach ($this->symbolDeclInFile as $symbol => $file) { + if (str_starts_with($symbol, 'global:') && !isset($knownFiles[$file])) { + unset($this->symbolDeclInFile[$symbol]); + } + } + } + + private function getIncrementalGeneratorFingerprint(): string + { + $context = hash_init('sha256'); + hash_update($context, serialize([ + 'translator' => self::VERSION, + 'php' => $this->phpVersion, + 'mode' => $this->buildMode, + 'nano' => $this->isNanoMode(), + 'nanoPolicy' => $this->isNanoPolicyMode(), + 'wasi' => $this->isWasiTarget(), + 'platform' => $this->targetPlatform, + 'literalStrings' => !$this->noLiteralStrings, + 'debug' => $this->debug, + ])); + $sourceDirectory = dirname(__DIR__); + $iterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($sourceDirectory, \FilesystemIterator::SKIP_DOTS), + ); + $sources = []; + foreach ($iterator as $entry) { + if ($entry->isFile() && $entry->getExtension() === 'php') { + $sources[] = $entry->getPathname(); + } + } + sort($sources, SORT_STRING); + foreach ($sources as $source) { + hash_update($context, str_replace('\\', '/', $source) . "\0"); + hash_update_file($context, $source); + } + return hash_final($context); + } +} diff --git a/src/Build/NativeCommandOptionsTrait.php b/src/Build/NativeCommandOptionsTrait.php index 86d2347c..7793654f 100644 --- a/src/Build/NativeCommandOptionsTrait.php +++ b/src/Build/NativeCommandOptionsTrait.php @@ -91,7 +91,7 @@ trait NativeCommandOptionsTrait if ($this->isBuildModeLib()) { $options = $options->with( 'forced_include', - $this->getIncludeDir() . '/php_' . $this->targetName . '_func_decl.h' + $this->getIncludeDir() . '/php_' . $this->targetName . '_all_decl.h' ); } diff --git a/src/Build/SourcePipelineTrait.php b/src/Build/SourcePipelineTrait.php index 68222fcc..3a1de608 100644 --- a/src/Build/SourcePipelineTrait.php +++ b/src/Build/SourcePipelineTrait.php @@ -48,6 +48,9 @@ trait SourcePipelineTrait $this->targetPlatform = $wasi ? 'wasm32-wasip2' : ''; $this->setTargetName($targetName); $this->setBuildDir($buildDir); + if ($this->climate->arguments->defined('force')) { + $this->clearIncrementalBuildCache(); + } $resolvedFiles = []; foreach ($files as $file) { @@ -86,7 +89,9 @@ trait SourcePipelineTrait $resolvedFiles = array_values($resolvedFiles); $this->composeTraitDeclarations($resolvedFiles); $this->discoverNativeGlobalObjects($resolvedFiles); - return $this->getSortedFiles($resolvedFiles); + $resolvedFiles = $this->getSortedFiles($resolvedFiles); + $this->initializeIncrementalCompilation($resolvedFiles); + return $resolvedFiles; } public function addFiles(array $files): void @@ -128,6 +133,9 @@ trait SourcePipelineTrait // Apply command-line arguments after all configuration is loaded (so they // take the highest precedence) $this->applyCommandLineArguments(); + if ($this->climate->arguments->defined('force')) { + $this->clearIncrementalBuildCache(); + } $this->validateProjectObjectFiles(); // The generated public import stub is an output artifact, not an input @@ -240,6 +248,7 @@ trait SourcePipelineTrait // per-file C++ body is generated. $this->discoverNativeGlobalObjects(array_values($files)); $files = $this->getSortedFiles($files); + $this->initializeIncrementalCompilation($files); return $files; } @@ -339,10 +348,15 @@ trait SourcePipelineTrait try { $this->composeTraitDeclarations($files); $previousPhase = $this->enterCompilerPhase(self::PHASE_CONVERT); + // Hydrate persistent literal/resource IDs before any unchanged + // translation unit or declaration header is reused. + $this->getStableIdRegistry(); // All declarations are now known. Lower declaration constant // expressions before translating any function body so cache IDs // are assigned exclusively in the convert phase. $this->finalizeDeclarationExpressions($files); + $this->initializeDeclarationHeaderFiles($files); + $this->restoreCleanIncrementalMetadata($files); $sourceFiles = []; $validSourceCount = 0; @@ -350,7 +364,26 @@ trait SourcePipelineTrait foreach ($files as $k => $file) { try { if (FileScanner::isPhpFile($file)) { - $cppFile = $this->convertFile($file); + $path = realpath($file) ?: $file; + if (!$this->shouldRegeneratePhpFile($path)) { + $validSourceCount++; + if ($this->incrementalTranslationUnitWasEmitted($path)) { + $cppFile = $this->getCppFile($path); + $this->registerGeneratedProjectSource($cppFile); + $sourceFiles[] = $cppFile; + } + $this->climate->darkGray( + '[incremental] reuse: ' . $this->getRelativePath($path), + ); + continue; + } + $statisticsBefore = $this->compilationStatistics->all(); + $cppFile = $this->convertFile($path, true); + $this->recordIncrementalConversion( + $path, + $cppFile !== null, + $this->compilationStatistics->delta($statisticsBefore), + ); } elseif (FileScanner::isNativeSourceFile($file)) { $cppFile = $file; } else { @@ -366,6 +399,7 @@ trait SourcePipelineTrait unset($files[$k]); } } + $this->finalizeIncrementalConversionMetadata($files); // A valid PHP input may intentionally emit no standalone translation // unit (for example a compile-time trait or an interface). The shared @@ -381,16 +415,17 @@ trait SourcePipelineTrait $this->genLibraryImportStub($files); } - // Generate the build-time internal headers: function declarations and - // runtime data declarations - $this->genFunctionDeclarations($this->getIncludeDir() . "/php_{$this->targetName}_func_decl.h"); - $this->genDataDeclarations($this->getIncludeDir() . "/php_{$this->targetName}_data_decl.h"); + // Function and data declarations are emitted together, one header + // per PHP source, plus a small project-runtime ABI header. + $this->genDeclarationHeaders($files); // Nano keeps the ordinary statically registered Zend class/module // metadata, then adds a direct native process entry beside it. $sourceFiles[] = $this->genExtension(); if ($this->isNanoMode()) { $sourceFiles[] = $this->genNanoEntrypoint(); } + $this->getStableIdRegistry()->flush(); + $this->saveIncrementalCompilationState($files); return $sourceFiles; } finally { diff --git a/src/Build/StableIdRegistry.php b/src/Build/StableIdRegistry.php new file mode 100644 index 00000000..d5f7ff2a --- /dev/null +++ b/src/Build/StableIdRegistry.php @@ -0,0 +1,158 @@ +> */ + private array $domains = []; + /** @var array */ + private array $nextIds = []; + private bool $dirty = false; + + public function __construct(private readonly string $file) + { + $this->restore(); + } + + public function allocate(string $domain, string $key): int + { + // Prefix prevents a base64 result made only of digits from being cast + // to an integer array key by PHP. + $encoded = 'b:' . base64_encode($key); + if (isset($this->domains[$domain][$encoded])) { + return $this->domains[$domain][$encoded]; + } + $id = $this->nextIds[$domain] ?? 0; + $this->domains[$domain][$encoded] = $id; + $this->nextIds[$domain] = $id + 1; + $this->dirty = true; + return $id; + } + + public function capacity(string $domain): int + { + return $this->nextIds[$domain] ?? 0; + } + + /** @return array */ + public function entries(string $domain): array + { + $entries = []; + foreach ($this->domains[$domain] ?? [] as $key => $id) { + $decoded = str_starts_with($key, 'b:') + ? base64_decode(substr($key, 2), true) + : false; + if ($decoded !== false) { + $entries[$decoded] = $id; + } + } + asort($entries, SORT_NUMERIC); + return $entries; + } + + public function flush(): void + { + if (!$this->dirty || $this->file === '') { + return; + } + $directory = dirname($this->file); + if (!is_dir($directory) && !mkdir($directory, 0777, true) && !is_dir($directory)) { + throw new \RuntimeException('Cannot create stable ID cache directory: ' . $directory); + } + $contents = json_encode([ + 'schema' => self::SCHEMA_VERSION, + 'domains' => $this->domains, + 'nextIds' => $this->nextIds, + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . PHP_EOL; + $temporary = tempnam($directory, '.ids-'); + if ($temporary === false) { + throw new \RuntimeException('Cannot create stable ID cache temporary file'); + } + try { + if (file_put_contents($temporary, $contents, LOCK_EX) === false) { + throw new \RuntimeException('Cannot write stable ID cache: ' . $this->file); + } + if (!@rename($temporary, $this->file)) { + @unlink($this->file); + if (!@rename($temporary, $this->file)) { + throw new \RuntimeException('Cannot replace stable ID cache: ' . $this->file); + } + } + } finally { + if (is_file($temporary)) { + @unlink($temporary); + } + } + $this->dirty = false; + } + + private function restore(): void + { + if ($this->file === '' || !is_file($this->file)) { + return; + } + try { + $contents = file_get_contents($this->file); + $payload = is_string($contents) + ? json_decode($contents, true, flags: JSON_THROW_ON_ERROR) + : null; + if (!is_array($payload) + || ($payload['schema'] ?? null) !== self::SCHEMA_VERSION + || !is_array($payload['domains'] ?? null) + || !is_array($payload['nextIds'] ?? null) + ) { + return; + } + $domains = []; + $nextIds = []; + foreach ($payload['domains'] as $domain => $entries) { + if (!is_string($domain) || !is_array($entries)) { + return; + } + $usedIds = []; + foreach ($entries as $key => $id) { + if (!is_string($key) + || !str_starts_with($key, 'b:') + || base64_decode(substr($key, 2), true) === false + || !is_int($id) + || $id < 0 + || isset($usedIds[$id])) { + return; + } + $domains[$domain][$key] = $id; + $usedIds[$id] = true; + } + } + foreach ($payload['nextIds'] as $domain => $nextId) { + if (is_string($domain) && is_int($nextId) && $nextId >= 0) { + $nextIds[$domain] = $nextId; + } + } + // Never trust a persisted cursor to be smaller than an existing + // entry. A truncated/manual cache edit must produce holes at worst, + // never duplicate an ID already embedded in an object file. + foreach ($domains as $domain => $entries) { + $minimumNextId = max($entries) + 1; + $nextIds[$domain] = max( + $nextIds[$domain] ?? 0, + $minimumNextId, + ); + } + $this->domains = $domains; + $this->nextIds = $nextIds; + } catch (\Throwable) { + $this->domains = []; + $this->nextIds = []; + } + } +} diff --git a/src/CompilerBase.php b/src/CompilerBase.php index ab40f478..3b52d3d3 100644 --- a/src/CompilerBase.php +++ b/src/CompilerBase.php @@ -11,6 +11,7 @@ namespace TypePhp; use TypePhp\Analysis\CompilationStatistics; use TypePhp\Build\AstCache; use TypePhp\Build\PhpxLocator; +use TypePhp\Build\StableIdRegistry; use League\CLImate\CLImate; use TypePhp\Backend\CompilerBackend; @@ -443,6 +444,11 @@ class CompilerBase implements PropertyAccessContext protected int $methodCallCacheIndex = 0; protected int $functionCallCacheIndex = 0; protected int $functionResolutionCacheIndex = 0; + /** @var array Per-function ordinal used to name dynamic cache sites. */ + protected array $cacheSiteOrdinals = []; + /** Separates declaration-expression sites from executable function bodies. */ + protected string $cacheSitePass = 'body'; + protected ?StableIdRegistry $stableIdRegistry = null; /** @var array> Prepared declaration ASTs keyed by real path. */ protected array $preparedFileAsts = []; protected bool $traitDeclarationsComposed = false; @@ -483,11 +489,7 @@ class CompilerBase implements PropertyAccessContext protected array $internalFunctions = []; protected array $internalConstants = []; - /** - * Stores the declaration of every function and class method. Key is the - * symbol name; value is the file in which the function or method is declared. - * @var array - */ + /** @var array Canonical symbol key => declaring PHP file. */ protected array $symbolDeclInFile = []; /** @@ -496,6 +498,10 @@ class CompilerBase implements PropertyAccessContext * @var array> */ protected array $symbolCallInFile = []; + /** @var array Canonical PHP file => generated declaration header. */ + protected array $declarationHeaderFiles = []; + /** @var array C++ translation units emitted by this compiler run. */ + protected array $generatedProjectSources = []; protected array $redoAfterDeclare = []; protected array $constData = []; protected int $optimizeLevel = 0; @@ -597,6 +603,10 @@ class CompilerBase implements PropertyAccessContext 'GLOBALS' => Type::ARRAY, ]; protected array $globalVars = []; + /** @var array> PHP source => global slot => inferred type. */ + protected array $globalVarsInFile = []; + /** @var array Global/static storage name => owning PHP file. */ + protected array $globalVarDeclInFile = []; /** @var array Global/static Native pointer slot => class name. */ protected array $nativeGlobalObjects = []; /** Immutable metadata shared by Native global pre-discovery and lowering. */ @@ -605,12 +615,17 @@ class CompilerBase implements PropertyAccessContext protected array $nativeClassDeclarations = []; /** @var array Request-reset initialization flags for Native static locals. */ protected array $nativeStaticInitializers = []; + /** @var array> PHP source => Native static initialization flags. */ + protected array $nativeStaticInitializersInFile = []; + /** @var array Native static initialization flag => owning PHP file. */ + protected array $nativeStaticInitializerDeclInFile = []; /** Box inferred integer locals in php::Var to retain Zend integer widening semantics. */ protected bool $varIntTypes = false; protected bool $decimalTypes = false; protected bool $bigintTypes = false; protected string $rootPath; protected string $buildDir; + protected string $targetName = 'app'; protected string $outputDir = ''; // Output directory specified by the -o option protected int $debugLine = 0; protected CLImate $climate; @@ -1132,12 +1147,18 @@ class CompilerBase implements PropertyAccessContext return $this->getIndent() . 'php::appendCallExtraNamedArgs(' . $var . ');' . PHP_EOL; } - public function writeFile(string $file, string $content): void + public function writeFile(string $file, string $content, bool $force = false): void { $dir = dirname($file); if (!is_dir($dir)) { mkdir($dir, 0777, true); } + if (!$force && is_file($file)) { + $existing = file_get_contents($file); + if ($existing === $content) { + return; + } + } if (!file_put_contents($file, $content)) { throw new \RuntimeException('Can not write file: ' . $file); } @@ -1291,6 +1312,7 @@ class CompilerBase implements PropertyAccessContext $this->interfacesDefineInFile = []; $this->functionDefineInFile = []; $this->stubImportLibrary = ''; + $this->cacheSiteOrdinals = []; } protected function resetNamespace(): void @@ -1315,6 +1337,31 @@ class CompilerBase implements PropertyAccessContext return $this->getNativeName($this->parseIdentifier($v->name), $this->namespace, $this->class); } + protected function getClassDependencySymbol(string $name): string + { + return 'class:' . strtolower(ltrim($name, '\\')); + } + + protected function getFunctionDependencySymbol(string $name): string + { + return 'function:' . strtolower(ltrim($name, '\\')); + } + + protected function getConstantDependencySymbol(string $name): string + { + return 'constant:' . ltrim($name, '\\'); + } + + protected function getGlobalDependencySymbol(string $name): string + { + return 'global:' . $name; + } + + protected function getNativeFunctionDependencySymbol(string $name): string + { + return 'native-function:' . strtolower($name); + } + protected function getFullClassName(): string { if ($this->classDef !== null) { @@ -1489,10 +1536,18 @@ class CompilerBase implements PropertyAccessContext return $this->persistentClassMap[$className]; } if ($this->isProcessStableClass($className)) { - $id = $this->persistentClassIndex++; + $id = $this->getStableIdRegistry()->allocate( + 'persistent-class', + strtolower(ltrim($className, '\\')), + ); + $this->persistentClassIndex = max($this->persistentClassIndex, $id + 1); $this->persistentClassMap[$className] = $id; } else { - $id = $this->classIndex++; + $id = $this->getStableIdRegistry()->allocate( + 'request-class', + strtolower(ltrim($className, '\\')), + ); + $this->classIndex = max($this->classIndex, $id + 1); $this->classMap[$className] = $id; } return $id; @@ -1508,10 +1563,18 @@ class CompilerBase implements PropertyAccessContext return $this->persistentFuncMap[$funcName]; } if ($this->isProcessStableFunction($funcName)) { - $id = $this->persistentFuncIndex++; + $id = $this->getStableIdRegistry()->allocate( + 'persistent-function', + strtolower(ltrim($funcName, '\\')), + ); + $this->persistentFuncIndex = max($this->persistentFuncIndex, $id + 1); $this->persistentFuncMap[$funcName] = $id; } else { - $id = $this->funcIndex++; + $id = $this->getStableIdRegistry()->allocate( + 'request-function', + strtolower(ltrim($funcName, '\\')), + ); + $this->funcIndex = max($this->funcIndex, $id + 1); $this->funcMap[$funcName] = $id; } return $id; @@ -1535,7 +1598,9 @@ class CompilerBase implements PropertyAccessContext if (isset($this->persistentPropMap[$key])) { return $this->persistentPropMap[$key]; } - $id = $this->persistentPropIndex++; + $stableKey = strtolower(ltrim($className, '\\')) . "\0" . $propName; + $id = $this->getStableIdRegistry()->allocate('persistent-property', $stableKey); + $this->persistentPropIndex = max($this->persistentPropIndex, $id + 1); $this->persistentPropMap[$key] = $id; return $id; } @@ -1543,21 +1608,24 @@ class CompilerBase implements PropertyAccessContext protected function getPropertyAccessCache(): string { $this->assertCompilerPhase(self::PHASE_CONVERT, 'property access cache ID allocation'); - $id = $this->propertyAccessCacheIndex++; + $id = $this->getStableCallSiteId('property-access'); + $this->propertyAccessCacheIndex = max($this->propertyAccessCacheIndex, $id + 1); return 'get_property_cache(PropertyCacheId{' . $id . '})'; } protected function getMethodCallCache(): string { $this->assertCompilerPhase(self::PHASE_CONVERT, 'method call cache ID allocation'); - $id = $this->methodCallCacheIndex++; + $id = $this->getStableCallSiteId('method-call'); + $this->methodCallCacheIndex = max($this->methodCallCacheIndex, $id + 1); return 'typephp_get_method_call_cache(MethodCallCacheId{' . $id . '})'; } protected function getFunctionCallCache(): string { $this->assertCompilerPhase(self::PHASE_CONVERT, 'function call cache ID allocation'); - $id = $this->functionCallCacheIndex++; + $id = $this->getStableCallSiteId('function-call'); + $this->functionCallCacheIndex = max($this->functionCallCacheIndex, $id + 1); return 'typephp_get_function_call_cache(FunctionCallCacheId{' . $id . '})'; } @@ -1565,10 +1633,27 @@ class CompilerBase implements PropertyAccessContext protected function getFunctionResolutionCache(): string { $this->assertCompilerPhase(self::PHASE_CONVERT, 'function resolution cache ID allocation'); - $id = $this->functionResolutionCacheIndex++; + $id = $this->getStableCallSiteId('function-resolution'); + $this->functionResolutionCacheIndex = max($this->functionResolutionCacheIndex, $id + 1); return 'typephp_get_function_resolution_cache(FunctionResolutionCacheId{' . $id . '})'; } + protected function getStableCallSiteId(string $domain): int + { + $scope = implode("\0", [ + $this->cacheSitePass, + isset($this->file) ? str_replace('\\', '/', $this->file) : '', + strtolower($this->namespace), + strtolower($this->class), + strtolower($this->function), + strtolower($this->method), + ]); + $ordinalKey = $domain . "\0" . $scope; + $ordinal = $this->cacheSiteOrdinals[$ordinalKey] ?? 0; + $this->cacheSiteOrdinals[$ordinalKey] = $ordinal + 1; + return $this->getStableIdRegistry()->allocate($domain, $scope . "\0" . $ordinal); + } + /** Return the function-local late-static-bound class entry. */ protected function getCalledCeExpr(): string { @@ -1801,7 +1886,7 @@ class CompilerBase implements PropertyAccessContext $seen[$interfaceNameLower] = true; $list[] = $interfaceName; if (!$this->isInternalInterface($interfaceName)) { - $this->symbolCallInFile[$this->file][] = $interfaceNameLower; + $this->symbolCallInFile[$this->file][] = $this->getClassDependencySymbol($interfaceNameLower); } } return $list; @@ -4497,6 +4582,8 @@ class CompilerBase implements PropertyAccessContext if (isset($this->nativeGlobalObjects[$globalVar])) { $flag = $globalVar . '__initialized'; $this->nativeStaticInitializers[$flag] = true; + $this->nativeStaticInitializersInFile[$this->file][$flag] = true; + $this->nativeStaticInitializerDeclInFile[$flag] = $this->file; $initState = $this->escapeGlobalVar($flag); $initCode = ''; } else { @@ -5204,6 +5291,7 @@ class CompilerBase implements PropertyAccessContext } $this->buildDir = $resolved; $this->astCache = null; + $this->stableIdRegistry = null; } protected function getAstCache(): AstCache @@ -5211,6 +5299,58 @@ class CompilerBase implements PropertyAccessContext return $this->astCache ??= new AstCache($this->parser, $this->buildDir, $this->phpVersion); } + protected function getStableIdRegistry(): StableIdRegistry + { + if ($this->stableIdRegistry === null) { + if ($this->forTest) { + $this->stableIdRegistry = new StableIdRegistry(''); + return $this->stableIdRegistry; + } + $target = $this->targetName !== '' ? $this->targetName : 'default'; + $directory = $this->buildDir . '/cache/incremental/' . $target; + $this->stableIdRegistry = new StableIdRegistry($directory . '/stable-ids.json'); + foreach ($this->stableIdRegistry->entries('literal') as $value => $id) { + $this->literalStrings[$value] = $id; + } + } + return $this->stableIdRegistry; + } + + protected function clearIncrementalBuildCache(): void + { + $targets = [ + $this->buildDir . '/cache/ast', + $this->buildDir . '/cache/incremental/' . ($this->targetName !== '' ? $this->targetName : 'default'), + ]; + foreach ($targets as $directory) { + if (!is_dir($directory)) { + continue; + } + $iterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($directory, \FilesystemIterator::SKIP_DOTS), + \RecursiveIteratorIterator::CHILD_FIRST, + ); + foreach ($iterator as $entry) { + if ($entry->isDir()) { + if (!rmdir($entry->getPathname())) { + throw new \RuntimeException('Cannot clear incremental cache directory: ' . $entry->getPathname()); + } + } else { + if (!unlink($entry->getPathname())) { + throw new \RuntimeException('Cannot clear incremental cache file: ' . $entry->getPathname()); + } + } + } + if (!rmdir($directory)) { + throw new \RuntimeException('Cannot clear incremental cache directory: ' . $directory); + } + } + $this->astCache = null; + $this->stableIdRegistry = null; + $this->literalStrings = []; + $this->literalStringIndex = 0; + } + protected function isStubFile(string $file): bool { return str_ends_with($file, '.stub.php'); diff --git a/src/Context/CompilationStateTrait.php b/src/Context/CompilationStateTrait.php index bd63f482..8a623547 100644 --- a/src/Context/CompilationStateTrait.php +++ b/src/Context/CompilationStateTrait.php @@ -120,7 +120,9 @@ trait CompilationStateTrait if (isset($this->stdTypeMap[$key])) { return $this->stdTypeMap[$key]; } - $typeId = count($this->stdTypeMap) + 1; + // Zero means "not a boxed std container" at runtime. Persistent IDs + // therefore start at one and are never renumbered between builds. + $typeId = $this->getStableIdRegistry()->allocate('std-type', $key) + 1; $this->stdTypeMap[$key] = $typeId; return $typeId; } @@ -158,7 +160,8 @@ trait CompilationStateTrait protected function addLiteralString(string $value): int { - $index = $this->literalStringIndex++; + $index = $this->getStableIdRegistry()->allocate('literal', $value); + $this->literalStringIndex = max($this->literalStringIndex, $index + 1); $this->literalStrings[$value] = $index; return $index; @@ -167,6 +170,21 @@ trait CompilationStateTrait protected function addGlobalVar(string $name, string $type): void { $this->globalVars[$name] = $type; + $this->recordGlobalVarUsage($name); + } + + protected function recordGlobalVarUsage(string $name): void + { + if (!isset($this->file) || $this->file === '') { + return; + } + $this->globalVarsInFile[$this->file][$name] = $this->globalVars[$name] ?? Type::VAR; + $symbol = $this->getGlobalDependencySymbol($name); + if (!isset($this->globalVarDeclInFile[$name])) { + $this->globalVarDeclInFile[$name] = $this->file; + $this->symbolDeclInFile[$symbol] = $this->file; + } + $this->symbolCallInFile[$this->file][] = $symbol; } protected function promoteGlobalOrStaticToNativeObject( @@ -222,6 +240,7 @@ trait CompilationStateTrait protected function addScopeGlobalVar(string $name, string $type): void { $this->context->globalVars[$name] = $type; + $this->recordGlobalVarUsage($name); } protected function addObject(string $name, string $class): void @@ -318,8 +337,9 @@ trait CompilationStateTrait { // The function declaration was detected during the preprocessing stage but is not yet defined, // meaning it is in the current file but appears in the wrong order. Skip it and handle it later. - if (isset($this->symbolDeclInFile[$name]) - and $this->symbolDeclInFile[$name] === $this->file + $symbol = $this->getNativeFunctionDependencySymbol($name); + if (isset($this->symbolDeclInFile[$symbol]) + and $this->symbolDeclInFile[$symbol] === $this->file and !$this->hasFunction($name)) { $this->redoAfterDeclare[$name] = true; throw new Skip(); diff --git a/src/Generator/AnonClassGenerator.php b/src/Generator/AnonClassGenerator.php index 6beceb62..c268b024 100644 --- a/src/Generator/AnonClassGenerator.php +++ b/src/Generator/AnonClassGenerator.php @@ -30,7 +30,9 @@ trait AnonClassGenerator { public function genAnonClassName(): string { - return self::ANON_CLASS . $this->anonClassIndex++; + $id = $this->getStableCallSiteId('anonymous-class'); + $this->anonClassIndex = max($this->anonClassIndex, $id + 1); + return self::ANON_CLASS . $id; } /** Flatten trait templates before an anonymous class is evaluated by ZendVM. */ diff --git a/src/Generator/DefaultArgumentGenerator.php b/src/Generator/DefaultArgumentGenerator.php index 2242f3b7..89662068 100644 --- a/src/Generator/DefaultArgumentGenerator.php +++ b/src/Generator/DefaultArgumentGenerator.php @@ -43,10 +43,13 @@ trait DefaultArgumentGenerator return "do {\n" . $plan->init . $body . $plan->clean . "} while (0);\n"; } - protected function genDefaultArgumentHelperDeclarations(): string + protected function genDefaultArgumentHelperDeclarations(?string $sourceFile = null): string { $code = ''; foreach ($this->symbols->functions() as $nativeName => $func) { + if ($sourceFile !== null && $func->sourceFile !== $sourceFile) { + continue; + } foreach ($func->argInfoList as $argumentIndex => $argInfo) { if (!$this->shouldGenerateDefaultArgumentHelper($argInfo)) { continue; diff --git a/src/NativeClass/NativeClassSupportTrait.php b/src/NativeClass/NativeClassSupportTrait.php index 73c84cfa..ac30cbed 100644 --- a/src/NativeClass/NativeClassSupportTrait.php +++ b/src/NativeClass/NativeClassSupportTrait.php @@ -1988,20 +1988,24 @@ trait NativeClassSupportTrait $ordered[] = $class; } - protected function genNativeObjectDeclarations(): string + protected function genNativeObjectDeclarations(?string $sourceFile = null): string { $this->validateNativeObjectMemberNames(); $classes = $this->getNativeObjectClassesInDeclarationOrder(); + if ($sourceFile !== null) { + $classes = array_values(array_filter( + $classes, + static fn (ClassDef $class): bool => $class->sourceFile === $sourceFile, + )); + } if ($classes === []) { return ''; } $code = '// TypePHP Native Object declarations' . PHP_EOL; - $code .= 'struct typephp_native_storage_constructor_t {};' . PHP_EOL; - foreach ($classes as $class) { - $code .= 'class ' . $this->getNativeObjectCppName($class) . ';' . PHP_EOL; + if ($sourceFile === null) { + $code .= $this->genNativeObjectForwardDeclarations(); } - $code .= PHP_EOL; foreach ($classes as $class) { $name = $this->getNativeObjectCppName($class); @@ -2102,6 +2106,19 @@ trait NativeClassSupportTrait return $code; } + protected function genNativeObjectForwardDeclarations(): string + { + $classes = $this->getNativeObjectClassesInDeclarationOrder(); + if ($classes === []) { + return ''; + } + $code = 'struct typephp_native_storage_constructor_t {};' . PHP_EOL; + foreach ($classes as $class) { + $code .= 'class ' . $this->getNativeObjectCppName($class) . ';' . PHP_EOL; + } + return $code . PHP_EOL; + } + protected function genNativeObjectRuntimeDefinition(ClassDef $class): string { $cpp = $this->getNativeObjectCppName($class); diff --git a/src/Preprocessor.php b/src/Preprocessor.php index e36d6036..af49d8eb 100644 --- a/src/Preprocessor.php +++ b/src/Preprocessor.php @@ -8,7 +8,6 @@ namespace TypePhp; -use MJS\TopSort\Implementations\StringSort; use TypePhp\Entity\ArgInfo; use TypePhp\Entity\ArrayInitPlan; use TypePhp\Entity\ClassDef; @@ -70,7 +69,6 @@ class Preprocessor extends CompilerBase '__debuginfo' => true, ]; - protected string $targetName = 'app'; /** * Validate every method that will become part of an enum. This is shared @@ -265,7 +263,6 @@ class Preprocessor extends CompilerBase public function getSortedFiles(array $list): array { - $sorter = new StringSort(); $fileDeps = []; // Build the dependency graph @@ -281,16 +278,35 @@ class Preprocessor extends CompilerBase } $deps = array_unique($deps); $fileDeps[$file] = $deps; - $sorter->add($file, $deps); } - $sortedFiles = $sorter->sort(); - - // Append files that do not participate in dependency management (non-stub files not present in the sorted list) - foreach ($list as $file) { - if (!$this->isStubFile($file) and !in_array($file, $sortedFiles)) { + // A source-level call graph may legally contain cycles (mutually + // recursive functions in different files). Use a cycle-tolerant DFS: + // acyclic dependencies still precede their consumers, while a back + // edge simply keeps the strongly-connected component in stable input + // order. Declaration collection has already completed for every file. + $known = array_fill_keys($list, true); + $visiting = []; + $visited = []; + $sortedFiles = []; + $visit = function (string $file) use (&$visit, &$visiting, &$visited, &$sortedFiles, $fileDeps, $known): void { + if (isset($visited[$file]) || isset($visiting[$file])) { + return; + } + $visiting[$file] = true; + foreach ($fileDeps[$file] ?? [] as $dependency) { + if (isset($known[$dependency])) { + $visit($dependency); + } + } + unset($visiting[$file]); + $visited[$file] = true; + if (!$this->isStubFile($file)) { $sortedFiles[] = $file; } + }; + foreach ($list as $file) { + $visit($file); } $this->climate->lightBlue('prepare completed: ' . count($sortedFiles) . ' source files in total'); @@ -471,6 +487,7 @@ class Preprocessor extends CompilerBase $this->fatalError($v, 'Unsupported statement: ' . $v->getType()); } } + $this->findSymbolUsing($stmts); } finally { $this->restoreCompilerPhase($previousPhase); } @@ -596,18 +613,24 @@ class Preprocessor extends CompilerBase if ($this->declarationExpressionsFinalized) { return; } - foreach ($files as $file) { - $path = realpath($file); - if ($path === false || !isset($this->preparedFileAsts[$path])) { - continue; + $previousCacheSitePass = $this->cacheSitePass; + $this->cacheSitePass = 'declaration'; + try { + foreach ($files as $file) { + $path = realpath($file); + if ($path === false || !isset($this->preparedFileAsts[$path])) { + continue; + } + $this->loadFile($path); + $this->resetFile(); + $this->resetFunction(); + $this->resetMethod(); + $this->resetClass(); + $this->resetNamespace(); + $this->finalizeDeclarationStatementList($this->preparedFileAsts[$path]); } - $this->loadFile($path); - $this->resetFile(); - $this->resetFunction(); - $this->resetMethod(); - $this->resetClass(); - $this->resetNamespace(); - $this->finalizeDeclarationStatementList($this->preparedFileAsts[$path]); + } finally { + $this->cacheSitePass = $previousCacheSitePass; } $this->declarationExpressionsFinalized = true; } @@ -1021,12 +1044,11 @@ class Preprocessor extends CompilerBase } /** - * Collect per-file symbol dependencies for the incremental compilation cache. + * Collect per-file symbol dependencies used by declaration headers. * - * The cache does not consume this graph yet, but this collector is retained - * intentionally so cache invalidation can later be based on symbol usage. + * @param NodeAbstract|array $ast */ - protected function findSymbolUsing(NodeAbstract $ast): void + protected function findSymbolUsing(NodeAbstract|array $ast): void { $nodeFinder = new NodeFinder(); $functionCalls = $nodeFinder->findInstanceOf($ast, Node\Expr\FuncCall::class); @@ -1034,13 +1056,26 @@ class Preprocessor extends CompilerBase foreach ($functionCalls as $call) { if ($call->name instanceof Node\Name) { // Internal functions do not participate in dependency management - $funcName = strtolower($call->name->toString()); + $resolvedName = $call->name->getAttribute('resolvedName') + ?? $call->name->getAttribute('namespacedName') + ?? $call->name; + $funcName = strtolower($resolvedName->toString()); if (!$this->isInternalFunction($funcName)) { - $this->symbolCallInFile[$this->file][] = $funcName; + $this->symbolCallInFile[$this->file][] = $this->getFunctionDependencySymbol($funcName); } } } + foreach ($nodeFinder->findInstanceOf($ast, Node\Expr\ConstFetch::class) as $fetch) { + $resolvedName = $fetch->name->getAttribute('resolvedName') + ?? $fetch->name->getAttribute('namespacedName') + ?? $fetch->name; + $constant = $resolvedName->toString(); + if (!in_array(strtolower($constant), ['true', 'false', 'null'], true)) { + $this->symbolCallInFile[$this->file][] = $this->getConstantDependencySymbol($constant); + } + } + $depClasses = []; $depClasses = array_merge($depClasses, $nodeFinder->findInstanceOf($ast, Node\Expr\StaticCall::class)); $depClasses = array_merge($depClasses, $nodeFinder->findInstanceOf($ast, Node\Expr\StaticPropertyFetch::class)); @@ -1048,15 +1083,76 @@ class Preprocessor extends CompilerBase $depClasses = array_merge($depClasses, $nodeFinder->findInstanceOf($ast, Node\Expr\New_::class)); foreach ($depClasses as $call) { if ($call->class instanceof Node\Name) { - $className = $this->parseIdentifier($call->class); + $resolvedClass = $call->class->getAttribute('resolvedName') + ?? $call->class->getAttribute('namespacedName') + ?? $call->class; + $className = $resolvedClass->toString(); if ($className !== 'self' && $className !== 'static') { - $fullClassName = $this->getNamespacedClassName($className); - $this->symbolCallInFile[$this->file][] = strtolower($fullClassName); + $this->symbolCallInFile[$this->file][] = $this->getClassDependencySymbol($className); + } + } + } + foreach ($nodeFinder->findInstanceOf($ast, Node\Expr\Instanceof_::class) as $instanceof) { + if ($instanceof->class instanceof Node\Name) { + $this->recordClassTypeDependency($instanceof->class); + } + } + foreach ($nodeFinder->findInstanceOf($ast, Node\FunctionLike::class) as $functionLike) { + $this->recordClassTypeDependency($functionLike->getReturnType()); + foreach ($functionLike->getParams() as $parameter) { + $this->recordClassTypeDependency($parameter->type); + } + } + foreach ($nodeFinder->findInstanceOf($ast, Node\Stmt\Property::class) as $property) { + $this->recordClassTypeDependency($property->type); + } + foreach ($nodeFinder->findInstanceOf($ast, Node\Stmt\ClassConst::class) as $constant) { + $this->recordClassTypeDependency($constant->type); + } + foreach ($nodeFinder->findInstanceOf($ast, Node\Stmt\Catch_::class) as $catch) { + foreach ($catch->types as $type) { + $this->recordClassTypeDependency($type); + } + } + foreach ($nodeFinder->findInstanceOf($ast, Node\Attribute::class) as $attribute) { + $this->recordClassTypeDependency($attribute->name); + } + foreach ($nodeFinder->findInstanceOf($ast, Node\Stmt\Global_::class) as $global) { + foreach ($global->vars as $variable) { + if ($variable instanceof Node\Expr\Variable && is_string($variable->name)) { + $this->symbolCallInFile[$this->file][] = $this->getGlobalDependencySymbol( + $this->escapeVarName($variable->name), + ); } } } // Deduplicate dependencies - $this->symbolCallInFile[$this->file] = array_unique($this->symbolCallInFile[$this->file]); + $this->symbolCallInFile[$this->file] = array_values(array_unique($this->symbolCallInFile[$this->file])); + } + + protected function recordClassTypeDependency(?NodeAbstract $type): void + { + if ($type instanceof Node\Name) { + if ($type->isSpecialClassName()) { + return; + } + $resolvedName = $type->getAttribute('resolvedName') + ?? $type->getAttribute('namespacedName') + ?? $type; + $this->symbolCallInFile[$this->file][] = $this->getClassDependencySymbol( + $resolvedName->toString(), + ); + return; + } + if ($type instanceof Node\NullableType) { + $this->recordClassTypeDependency($type->type); + return; + } + if ($type instanceof Node\UnionType || $type instanceof Node\IntersectionType) { + foreach ($type->types as $member) { + $this->recordClassTypeDependency($member); + } + } } protected function prepareNamespace(Node\Stmt\Namespace_ $node): void @@ -1597,6 +1693,12 @@ class Preprocessor extends CompilerBase ? $this->classDef->getNamespacedName(false) . '::' . $functionDef->name : $functionDef->getNamespacedName(); $this->addFunction($name, $functionDef); + $this->symbolDeclInFile[$this->getNativeFunctionDependencySymbol($name)] = $this->file; + if (!$functionDef->method) { + $this->symbolDeclInFile[$this->getFunctionDependencySymbol( + $functionDef->getNamespacedName(), + )] = $this->file; + } if ($this->methodDef) { $this->methodDef->functionDef = $functionDef; } @@ -1619,7 +1721,8 @@ class Preprocessor extends CompilerBase } else { $flags = Modifiers::PUBLIC; } - if (isset($this->symbolDeclInFile[$fullClassNameLower])) { + $classDependencySymbol = $this->getClassDependencySymbol($fullClassNameLower); + if (isset($this->symbolDeclInFile[$classDependencySymbol])) { $this->fatalError($class, "Duplicate class `{$fullClassName}`"); } // Dynamic properties are forbidden on readonly classes and enums. @@ -1676,7 +1779,7 @@ class Preprocessor extends CompilerBase $this->symbols->setParent($fullClassNameLower, $parentClassLower); $this->classSubClasses[$parentClassLower][] = $fullClassNameLower; if (!$this->isInternalClass($parentClassLower)) { - $this->symbolCallInFile[$this->file][] = $parentClassLower; + $this->symbolCallInFile[$this->file][] = $this->getClassDependencySymbol($parentClassLower); } $this->classDef->extends = $this->parentClass; // Whether it inherits from an internal class @@ -1709,7 +1812,7 @@ class Preprocessor extends CompilerBase $this->classDef->traitUseFunctions = $this->useFunctions; $this->classDef->traitUseConstants = $this->useConstants; } - $this->symbolDeclInFile[$fullClassNameLower] = $this->file; + $this->symbolDeclInFile[$classDependencySymbol] = $this->file; if ($class instanceof Node\Stmt\Class_) { $generatedPrinter = null; @@ -3046,15 +3149,16 @@ class Preprocessor extends CompilerBase $this->interfaceDef->extends = $parentName; } if (!$this->isInternalInterface($parentName)) { - $this->symbolCallInFile[$this->file][] = strtolower($parentName); + $this->symbolCallInFile[$this->file][] = $this->getClassDependencySymbol($parentName); } } - if (isset($this->symbolDeclInFile[$interfaceNameLower])) { + $interfaceDependencySymbol = $this->getClassDependencySymbol($interfaceNameLower); + if (isset($this->symbolDeclInFile[$interfaceDependencySymbol])) { $this->fatalError($v, "Duplicate interface `{$interfaceName}`"); } - $this->symbolDeclInFile[$interfaceNameLower] = $this->file; + $this->symbolDeclInFile[$interfaceDependencySymbol] = $this->file; $this->symbols->putInterface($this->escapeClass($interfaceName), $this->interfaceDef); $this->interfacesDefineInFile[$interfaceName] = $this->interfaceDef; @@ -3322,7 +3426,7 @@ class Preprocessor extends CompilerBase $traitName = $this->getNamespacedClassName($this->parseIdentifier($trait)); $this->classDef->usedTraits[] = $traitName; if (!$this->isInternalClass($traitName)) { - $this->symbolCallInFile[$this->file][] = strtolower($traitName); + $this->symbolCallInFile[$this->file][] = $this->getClassDependencySymbol($traitName); } } foreach ($aliases as $fullMethodName => $aliasList) { diff --git a/src/Resolver/DeclarationSymbolTrait.php b/src/Resolver/DeclarationSymbolTrait.php index 967a579b..804b9cc1 100644 --- a/src/Resolver/DeclarationSymbolTrait.php +++ b/src/Resolver/DeclarationSymbolTrait.php @@ -38,7 +38,9 @@ trait DeclarationSymbolTrait $constInfo->codegenFinalized = $this->compilerPhase === self::PHASE_CONVERT; $constInfo->namespace = $this->namespace; $constInfo->name = $name; + $constInfo->sourceFile = $this->file; $this->constants[$this->escapeConstVar($name)] = $constInfo; + $this->symbolDeclInFile[$this->getConstantDependencySymbol($name)] = $this->file; } protected function hasConstant(string $name): bool diff --git a/src/Translator.php b/src/Translator.php index 817d1798..10ea167f 100644 --- a/src/Translator.php +++ b/src/Translator.php @@ -17,6 +17,7 @@ use TypePhp\Analysis\SsaBuilder; use TypePhp\Backend\CompilerFactory; use TypePhp\Build\CompileOptions; use TypePhp\Build\FileScanner; +use TypePhp\Build\IncrementalCompilationTrait; use TypePhp\Build\NativeCommandOptionsTrait; use TypePhp\Build\NativeBuilder; use TypePhp\Build\NanoBuildBackend; @@ -78,12 +79,13 @@ class Translator extends Preprocessor private const string TRAIT_ORIGIN_ATTRIBUTE = 'typephp_trait_origin'; private const string TRAIT_METHOD_ATTRIBUTE = 'typephp_trait_method'; use DefaultArgumentGenerator; + use IncrementalCompilationTrait; use NativeCommandOptionsTrait; use SourcePipelineTrait; use ResourceCompilationTrait; use ClassConstantValueTrait; - public const string VERSION = '0.8.2'; + public const string VERSION = '0.9.0'; public const string APP_NAME = 'TypePHP Compiler (AOT)'; protected bool $hasExplicitOutput = false; @@ -405,7 +407,7 @@ class Translator extends Preprocessor ['--profile', 'Enable performance profiling (adds -lprofiler, forces recompile)'], ['-d, --debug', 'Enable debug mode (disables optimizations and adds debug symbols)'], ['-o, --output ', 'Output name or path (default: input basename)'], - ['-f, --force', 'Force recompilation and ignore the object cache'], + ['-f, --force', 'Clear incremental caches and force a full rebuild'], ['-m, --mode ', 'Build mode: bin, lib, or ext (default: bin)'], ['-r, --run', 'Run the compiled binary after a successful build'], ['-j, --job ', 'Number of parallel compilation jobs (default: 4)'], @@ -716,13 +718,13 @@ class Translator extends Preprocessor shell_exec($cmd); } - public function save(string $code, string $file): void + public function save(string $code, string $file, bool $force = false): void { - $this->writeFile($file, $code); + $this->writeFile($file, $code, $force); $this->formatCppCode($file); } - public function convertFile(string $file): ?string + public function convertFile(string $file, bool $forceWrite = false): ?string { $ownsStatisticsSession = !$this->compilationStatistics->isCollecting(); if ($ownsStatisticsSession) { @@ -751,7 +753,8 @@ class Translator extends Preprocessor if ($cppCode === '') { $this->removeEmptyTranslationUnitArtifacts($cppFile); } else { - $this->save($cppCode, $cppFile); + $this->generatedProjectSources[$cppFile] = true; + $this->save($cppCode, $cppFile, $forceWrite); } // Generate the stub file, which depends on the use statements // and other info collected during the convert phase. @@ -777,6 +780,7 @@ class Translator extends Preprocessor */ private function removeEmptyTranslationUnitArtifacts(string $cppFile): void { + unset($this->generatedProjectSources[$cppFile]); $objectFile = $this->getObjectFile($cppFile); foreach ([$cppFile, $objectFile, $this->getMiscObjectCacheMetadataFile($objectFile)] as $artifact) { if (is_file($artifact) && !unlink($artifact)) { @@ -866,6 +870,7 @@ class Translator extends Preprocessor exit(1); } $this->targetName = $name; + $this->stableIdRegistry = null; } /** @@ -952,6 +957,12 @@ class Translator extends Preprocessor public function genDataDeclarations(string $file): void { + $this->writeFile($file, $this->renderDataDeclarations()); + } + + protected function renderDataDeclarations(?string $sourceFile = null, bool $commonOnly = false): string + { + $includeCommon = $sourceFile === null; $projectNamespace = $this->getProjectNamespace(); $lines[] = '#include '; $lines[] = '#include '; @@ -961,63 +972,91 @@ class Translator extends Preprocessor // Embedded binaries populate the CLI script fields in $_SERVER at // request startup, even when the source does not reference $_SERVER. - if (!$this->isNanoMode() + if ($includeCommon + && !$this->isNanoMode() && $this->isBuildModeBin() && !$this->hasGlobalVar('_SERVER')) { - $this->addGlobalVar('_SERVER', Type::ARRAY); + // Runtime-provided storage has no declaring PHP source. + $this->globalVars['_SERVER'] = Type::ARRAY; } foreach ($this->globalVars as $name => $type) { + if ($sourceFile !== null + && ($this->globalVarDeclInFile[$name] ?? null) !== $sourceFile) { + continue; + } + if ($commonOnly && isset($this->globalVarDeclInFile[$name])) { + continue; + } $cppType = isset($this->nativeGlobalObjects[$name]) ? $this->getNativeObjectPointerType($this->nativeGlobalObjects[$name]) : Type::VAR; $lines[] = 'extern THREAD_LOCAL ' . $cppType . ' ' . $this->escapeGlobalVar($name) . ';'; } foreach ($this->nativeStaticInitializers as $name => $_) { + if ($sourceFile !== null + && ($this->nativeStaticInitializerDeclInFile[$name] ?? null) !== $sourceFile) { + continue; + } + if ($commonOnly && isset($this->nativeStaticInitializerDeclInFile[$name])) { + continue; + } $lines[] = 'extern THREAD_LOCAL bool ' . $this->escapeGlobalVar($name) . ';'; } - if ($this->literalStrings) { + if ($includeCommon && $this->literalStrings) { $lines[] = 'ZEND_ATTRIBUTE_CONST ' . Type::STR . ' &' . self::LITERAL_STRING_GETTER . '(uint32_t index) noexcept;' . PHP_EOL; } - foreach ($this->constants as $name => $constant) { - $lines[] = 'extern ' . $constant->type . ' ' . $name . ';'; + if (!$commonOnly) { + foreach ($this->constants as $name => $constant) { + if ($sourceFile !== null && ($constant->sourceFile ?? '') !== $sourceFile) { + continue; + } + $lines[] = 'extern ' . $constant->type . ' ' . $name . ';'; + } } - $pythonModuleDeclarations = $this->genPythonModuleDataDeclarations(); - if ($pythonModuleDeclarations !== '') { - $lines[] = $pythonModuleDeclarations; - } + if ($includeCommon) { + $pythonModuleDeclarations = $this->genPythonModuleDataDeclarations(); + if ($pythonModuleDeclarations !== '') { + $lines[] = $pythonModuleDeclarations; + } - $lines[] = 'enum class RequestClassId : uint32_t {};'; - $lines[] = 'enum class PersistentClassId : uint32_t {};'; - $lines[] = 'enum class RequestFuncId : uint32_t {};'; - $lines[] = 'enum class PersistentFuncId : uint32_t {};'; - $lines[] = 'enum class PersistentPropertyId : uint32_t {};'; - $lines[] = 'enum class PropertyCacheId : uint32_t {};' . PHP_EOL; - $lines[] = 'enum class MethodCallCacheId : uint32_t {};' . PHP_EOL; - $lines[] = 'enum class FunctionCallCacheId : uint32_t {};' . PHP_EOL; - $lines[] = 'enum class FunctionResolutionCacheId : uint32_t {};' . PHP_EOL; + $lines[] = 'enum class RequestClassId : uint32_t {};'; + $lines[] = 'enum class PersistentClassId : uint32_t {};'; + $lines[] = 'enum class RequestFuncId : uint32_t {};'; + $lines[] = 'enum class PersistentFuncId : uint32_t {};'; + $lines[] = 'enum class PersistentPropertyId : uint32_t {};'; + $lines[] = 'enum class PropertyCacheId : uint32_t {};' . PHP_EOL; + $lines[] = 'enum class MethodCallCacheId : uint32_t {};' . PHP_EOL; + $lines[] = 'enum class FunctionCallCacheId : uint32_t {};' . PHP_EOL; + $lines[] = 'enum class FunctionResolutionCacheId : uint32_t {};' . PHP_EOL; - $lines[] = 'zend_class_entry *get_class(RequestClassId class_id, const php::Str &class_name);'; - $lines[] = 'zend_function *get_func(RequestFuncId func_id, const php::Str &func_name);'; - $lines[] = 'zend_function *get_method(RequestFuncId func_id, const php::Str &method_name, RequestClassId class_id, const php::Str &class_name);'; - $lines[] = 'zend_class_entry *get_persistent_class(PersistentClassId class_id, const php::Str &class_name);'; - $lines[] = 'zend_function *get_persistent_func(PersistentFuncId func_id, const php::Str &func_name);'; - $lines[] = 'zend_function *get_persistent_method(PersistentFuncId func_id, const php::Str &method_name, PersistentClassId class_id, const php::Str &class_name);'; - $lines[] = 'uint32_t get_persistent_prop(PersistentPropertyId prop_id, const php::Str &prop_name, const php::Str &class_name);' . PHP_EOL; - $lines[] = 'php::PropertyCacheSlot &get_property_cache(PropertyCacheId cache_id) noexcept;' . PHP_EOL; - $lines[] = 'php::MethodCallCacheSlot &typephp_get_method_call_cache(MethodCallCacheId cache_id) noexcept;' . PHP_EOL; - $lines[] = 'php::FunctionCallCacheSlot &typephp_get_function_call_cache(FunctionCallCacheId cache_id) noexcept;' . PHP_EOL; - $lines[] = 'uint8_t &typephp_get_function_resolution_cache(FunctionResolutionCacheId cache_id) noexcept;' . PHP_EOL; + $lines[] = 'zend_class_entry *get_class(RequestClassId class_id, const php::Str &class_name);'; + $lines[] = 'zend_function *get_func(RequestFuncId func_id, const php::Str &func_name);'; + $lines[] = 'zend_function *get_method(RequestFuncId func_id, const php::Str &method_name, RequestClassId class_id, const php::Str &class_name);'; + $lines[] = 'zend_class_entry *get_persistent_class(PersistentClassId class_id, const php::Str &class_name);'; + $lines[] = 'zend_function *get_persistent_func(PersistentFuncId func_id, const php::Str &func_name);'; + $lines[] = 'zend_function *get_persistent_method(PersistentFuncId func_id, const php::Str &method_name, PersistentClassId class_id, const php::Str &class_name);'; + $lines[] = 'uint32_t get_persistent_prop(PersistentPropertyId prop_id, const php::Str &prop_name, const php::Str &class_name);' . PHP_EOL; + $lines[] = 'php::PropertyCacheSlot &get_property_cache(PropertyCacheId cache_id) noexcept;' . PHP_EOL; + $lines[] = 'php::MethodCallCacheSlot &typephp_get_method_call_cache(MethodCallCacheId cache_id) noexcept;' . PHP_EOL; + $lines[] = 'php::FunctionCallCacheSlot &typephp_get_function_call_cache(FunctionCallCacheId cache_id) noexcept;' . PHP_EOL; + $lines[] = 'uint8_t &typephp_get_function_resolution_cache(FunctionResolutionCacheId cache_id) noexcept;' . PHP_EOL; + } - foreach ($this->getClassLikesWithConstants() as $classDef) { - foreach ($classDef->constants as $constant) { - if ($constant->type === Type::ARRAY) { - $constName = self::PREFIX . $this->getNativeName($constant->name, $classDef->namespace, $classDef->name); - $lines[] = 'extern ' . Type::VAR . ' ' . $constName . ';' . PHP_EOL; + if (!$commonOnly) { + foreach ($this->getClassLikesWithConstants() as $classDef) { + if ($sourceFile !== null && $classDef->sourceFile !== $sourceFile) { + continue; + } + foreach ($classDef->constants as $constant) { + if ($constant->type === Type::ARRAY) { + $constName = self::PREFIX . $this->getNativeName($constant->name, $classDef->namespace, $classDef->name); + $lines[] = 'extern ' . Type::VAR . ' ' . $constName . ';' . PHP_EOL; + } } } } @@ -1025,8 +1064,7 @@ class Translator extends Preprocessor $lines[] = '} // namespace ' . $projectNamespace; $lines[] = 'using namespace ' . $projectNamespace . ';'; - $code = implode(PHP_EOL, $lines) . PHP_EOL . PHP_EOL; - $this->writeFile($file, $code); + return implode(PHP_EOL, $lines) . PHP_EOL . PHP_EOL; } public function genExtension(): string @@ -1065,12 +1103,15 @@ class Translator extends Preprocessor $entryCall = $entry->returnType === Type::VOID ? $call . ';' . PHP_EOL . ' return 0;' : 'return static_cast(' . $call . ');'; - $code = '#include targetName . '_func_decl.h>' . PHP_EOL . PHP_EOL; + $entryHeader = $this->declarationHeaderFiles[$entry->sourceFile] + ?? 'php_' . $this->targetName . '_func_decl.h'; + $code = '#include <' . $entryHeader . '>' . PHP_EOL . PHP_EOL; $code .= 'extern "C" int typephp_nano_project_main() {' . PHP_EOL; $code .= ' ' . $entryCall . PHP_EOL; $code .= '}' . PHP_EOL; $this->writeFile($file, $code); + $this->generatedProjectSources[$file] = true; return $file; } @@ -1134,11 +1175,14 @@ class Translator extends Preprocessor } } $file = $this->getBuildDir() . '/extension-' . $this->targetName . '.cc'; + sort($this->argInfoHeaderFiles, SORT_STRING); + sort($this->registerSymbols, SORT_STRING); + sort($this->releaseAstConstantFns, SORT_STRING); $this->localHeaders = $this->argInfoHeaderFiles; $this->genClassCeList(); $this->indentLevel++; - $code = $this->genIncludeHeaderFiles(); + $code = $this->genIncludeHeaderFiles(true); // Only the generated module entry allocates request-cache storage. // Keep out of the shared PCH dependency set used by every source. $code .= '#include ' . PHP_EOL; @@ -1193,17 +1237,17 @@ class Translator extends Preprocessor // a project does not use that cache kind. $code .= 'struct php_request_cache_storage final {' . PHP_EOL; $code .= $this->getIndent() . 'zend_class_entry *' . self::CLASS_MAP . '[' - . max(1, count($this->classMap)) . ']{};' . PHP_EOL; + . max(1, $this->getStableIdRegistry()->capacity('request-class')) . ']{};' . PHP_EOL; $code .= $this->getIndent() . 'zend_function *' . self::FUNC_MAP . '[' - . max(1, count($this->funcMap)) . ']{};' . PHP_EOL; + . max(1, $this->getStableIdRegistry()->capacity('request-function')) . ']{};' . PHP_EOL; $code .= $this->getIndent() . 'php::PropertyCacheSlot property_cache_map[' - . max(1, $this->propertyAccessCacheIndex) . ']{};' . PHP_EOL; + . max(1, $this->getStableIdRegistry()->capacity('property-access')) . ']{};' . PHP_EOL; $code .= $this->getIndent() . 'php::MethodCallCacheSlot method_call_cache_map[' - . max(1, $this->methodCallCacheIndex) . ']{};' . PHP_EOL; + . max(1, $this->getStableIdRegistry()->capacity('method-call')) . ']{};' . PHP_EOL; $code .= $this->getIndent() . 'php::FunctionCallCacheSlot function_call_cache_map[' - . max(1, $this->functionCallCacheIndex) . ']{};' . PHP_EOL; + . max(1, $this->getStableIdRegistry()->capacity('function-call')) . ']{};' . PHP_EOL; $code .= $this->getIndent() . 'uint8_t function_resolution_cache_map[' - . max(1, $this->functionResolutionCacheIndex) . ']{};' . PHP_EOL; + . max(1, $this->getStableIdRegistry()->capacity('function-resolution')) . ']{};' . PHP_EOL; $code .= '};' . PHP_EOL; $code .= 'static THREAD_LOCAL php_request_cache_storage *php_request_cache = nullptr;' . PHP_EOL; @@ -1211,17 +1255,17 @@ class Translator extends Preprocessor // Internal/compiled symbols have module lifetime. They are initialized // lazily after PHP startup, so disable_functions/disable_classes have // already finalized the runtime tables. ZTS publishes them atomically. - $code .= 'static php::PersistentCacheSlot ' . self::PREFIX . self::PERSISTENT_CLASS_MAP . '[' . max(1, count($this->persistentClassMap)) . ']{};' . PHP_EOL; + $code .= 'static php::PersistentCacheSlot ' . self::PREFIX . self::PERSISTENT_CLASS_MAP . '[' . max(1, $this->getStableIdRegistry()->capacity('persistent-class')) . ']{};' . PHP_EOL; $code .= "// func \n"; - $code .= 'static php::PersistentCacheSlot ' . self::PREFIX . self::PERSISTENT_FUNC_MAP . '[' . max(1, count($this->persistentFuncMap)) . ']{};' . PHP_EOL; + $code .= 'static php::PersistentCacheSlot ' . self::PREFIX . self::PERSISTENT_FUNC_MAP . '[' . max(1, $this->getStableIdRegistry()->capacity('persistent-function')) . ']{};' . PHP_EOL; $code .= $this->genPythonModuleStorage(); $code .= "// property \n"; // No dynamic propMap: the property offset cache only covers declared // properties of compiled/built-in classes (see getPropertyId). - $code .= 'static php::PersistentCacheSlot ' . self::PREFIX . self::PERSISTENT_PROP_MAP . '[' . max(1, count($this->persistentPropMap)) . ']{};' . PHP_EOL; + $code .= 'static php::PersistentCacheSlot ' . self::PREFIX . self::PERSISTENT_PROP_MAP . '[' . max(1, $this->getStableIdRegistry()->capacity('persistent-property')) . ']{};' . PHP_EOL; $code .= "// functions \n"; $code .= <<<'CODE' @@ -1754,6 +1798,7 @@ CODE; $this->writeFile($file, $code); $this->formatCppCode($file); + $this->generatedProjectSources[$file] = true; $this->localHeaders = []; return $file; } @@ -1801,7 +1846,7 @@ CODE; } $objectMtime = filemtime($objectFile); - if ($objectMtime <= filemtime($cppFile)) { + if ($objectMtime < filemtime($cppFile)) { return false; } @@ -1844,6 +1889,89 @@ CODE; return hash('sha256', $this->buildCompileFileCommand($sourceFile, $objectFile) . "\0" . serialize($abi)); } + protected function getGeneratedObjectCacheKey(string $sourceFile, string $objectFile): string + { + $context = hash_init('sha256'); + hash_update($context, $this->getMiscObjectCacheKey($sourceFile, $objectFile)); + $visited = []; + $this->hashGeneratedCompileInput($context, $sourceFile, $visited); + if ($this->isProjectRuntimeEntryFile($sourceFile)) { + $this->hashGeneratedCompileInput( + $context, + $this->getIncludeDir() . '/' . $this->getAllDeclarationHeaderName(), + $visited, + ); + } + return hash_final($context); + } + + /** @param array $visited */ + private function hashGeneratedCompileInput(\HashContext $context, string $file, array &$visited): void + { + $real = realpath($file); + if ($real === false || isset($visited[$real])) { + return; + } + $visited[$real] = true; + $contents = file_get_contents($real); + if (!is_string($contents)) { + return; + } + hash_update($context, str_replace('\\', '/', $real) . "\0" . $contents . "\0"); + $matchCount = preg_match_all('/^\s*#\s*include\s*[<\"]([^>\"]+)[>\"]/m', $contents, $matches); + if ($matchCount === false || $matchCount === 0) { + return; + } + $includeRoot = realpath($this->getIncludeDir()); + if ($includeRoot === false) { + return; + } + foreach ($matches[1] as $header) { + $candidate = $includeRoot . DIRECTORY_SEPARATOR . str_replace('/', DIRECTORY_SEPARATOR, $header); + $resolved = realpath($candidate); + if ($resolved === false + || ($resolved !== $includeRoot + && !str_starts_with($resolved, $includeRoot . DIRECTORY_SEPARATOR))) { + continue; + } + $this->hashGeneratedCompileInput($context, $resolved, $visited); + } + } + + private function isGeneratedProjectSource(string $sourceFile): bool + { + return isset($this->generatedProjectSources[$sourceFile]); + } + + private function hasGeneratedObjectFileCache(string $sourceFile, string $objectFile): bool + { + if ($this->climate->arguments->defined('force') + || $this->enableProfiler + || !is_file($sourceFile) + || !is_file($objectFile)) { + return false; + } + $metadata = $this->getMiscObjectCacheMetadataFile($objectFile); + $key = is_file($metadata) ? file_get_contents($metadata) : false; + if (!is_string($key) + || trim($key) !== $this->getGeneratedObjectCacheKey($sourceFile, $objectFile)) { + return false; + } + $objectMtime = filemtime($objectFile); + $sourceMtime = filemtime($sourceFile); + return $objectMtime !== false + && $sourceMtime !== false + && $objectMtime >= $sourceMtime; + } + + private function writeGeneratedObjectCacheMetadata(string $sourceFile, string $objectFile): void + { + $metadata = $this->getMiscObjectCacheMetadataFile($objectFile); + if (file_put_contents($metadata, $this->getGeneratedObjectCacheKey($sourceFile, $objectFile) . PHP_EOL) === false) { + throw new \RuntimeException('Cannot write generated object cache metadata: ' . $metadata); + } + } + protected function writeMiscObjectCacheMetadata(string $sourceFile, string $objectFile): void { $metadataFile = $this->getMiscObjectCacheMetadataFile($objectFile); @@ -1936,7 +2064,8 @@ CODE; * language: ?string, * options: CompileOptions, * cacheable_misc: bool, - * nano_runtime: bool + * nano_runtime: bool, + * generated_project: bool * } */ private function prepareCompileFileTask( @@ -1947,6 +2076,8 @@ CODE; $isCacheableMiscFile = $this->isPhpxMiscFile($cppFile) && !$this->isProjectRuntimeEntryFile($cppFile); $isNanoRuntimeSource = isset($this->nanoRuntimeSources[$cppFile]); + $isGeneratedProjectSource = $this->isGeneratedProjectSource($cppFile) + || $this->isProjectRuntimeEntryFile($cppFile); if ($isCacheableMiscFile && $this->hasMiscObjectFileCache($cppFile)) { if (!$parallel) { $this->climate->darkGray('[cache] skip: ' . $cppFile); @@ -1956,8 +2087,11 @@ CODE; if ($isNanoRuntimeSource && $this->hasNanoObjectFileCache($cppFile, $objectFile)) { return null; } + if ($isGeneratedProjectSource && $this->hasGeneratedObjectFileCache($cppFile, $objectFile)) { + return null; + } - if ($isCacheableMiscFile) { + if ($isCacheableMiscFile || $isGeneratedProjectSource) { $this->invalidateMiscObjectCache($objectFile); } @@ -1967,10 +2101,11 @@ CODE; 'options' => $this->getSourceCompileCommandOptions($cppFile, $language), 'cacheable_misc' => $isCacheableMiscFile, 'nano_runtime' => $isNanoRuntimeSource, + 'generated_project' => $isGeneratedProjectSource, ]; } - /** @param array{cacheable_misc: bool, nano_runtime: bool} $task */ + /** @param array{cacheable_misc: bool, nano_runtime: bool, generated_project: bool} $task */ private function finalizeCompileFileTask(string $cppFile, string $objectFile, array $task): void { if ($task['cacheable_misc']) { @@ -1979,6 +2114,9 @@ CODE; if ($task['nano_runtime']) { $this->writeMiscObjectCacheMetadata($cppFile, $objectFile); } + if ($task['generated_project']) { + $this->writeGeneratedObjectCacheMetadata($cppFile, $objectFile); + } } private function hasNanoObjectFileCache(string $sourceFile, string $objectFile): bool @@ -1996,7 +2134,7 @@ CODE; $sourceMtime = filemtime($sourceFile); return $objectMtime !== false && $sourceMtime !== false - && $objectMtime > $sourceMtime + && $objectMtime >= $sourceMtime && $objectMtime >= $this->nanoRuntimeHeaderMtime; } @@ -2006,11 +2144,17 @@ CODE; return $this->getProjectRuntimeEntryCompileCommandOptions(); } - return match ($language) { + $options = match ($language) { null => $this->getCompileCommandOptions(), 'c' => $this->getCCompileCommandOptions(), default => $this->getNativeCompileCommandOptions($language), }; + if ($this->isGeneratedProjectSource($sourceFile) && isset($options['forced_include'])) { + $values = $options->toArray(); + unset($values['forced_include']); + return new CompileOptions($values); + } + return $options; } protected function buildCompileFileCommand(string $sourceFile, string $objectFile): string @@ -2291,7 +2435,10 @@ CODE; throw new \Exception('Compilation failed for: ' . implode(', ', $result['failures'])); } $this->climate->green("Successfully compiled {$totalFiles} files"); - return [...$cachedObjects, ...$result['objects']]; + // Parallel completion and cache-hit order are intentionally unrelated + // to dependency order. Return objects in the stable source-list order + // so the link command (and therefore its cache key) is reproducible. + return array_map($this->getObjectFile(...), $sourceFiles); } public function output(string $message, string $style = 'out'): void @@ -2304,6 +2451,44 @@ CODE; return $this->getNativeBuilder()->linkCommand($objectFiles, $targetFile, $this->getLinkCommandOptions()); } + /** @param list $objectFiles */ + private function getLinkCacheKey(array $objectFiles, string $targetFile): string + { + return hash('sha256', $this->buildLinkCommand($objectFiles, $targetFile)); + } + + /** @param list $objectFiles */ + private function hasLinkCache(array $objectFiles, string $targetFile): bool + { + if ($this->climate->arguments->defined('force') + || $this->enableProfiler + || !is_file($targetFile)) { + return false; + } + $targetMtime = filemtime($targetFile); + if ($targetMtime === false) { + return false; + } + foreach ($objectFiles as $objectFile) { + $objectMtime = is_file($objectFile) ? filemtime($objectFile) : false; + if ($objectMtime === false || $targetMtime < $objectMtime) { + return false; + } + } + $metadata = $targetFile . '.typephp-link-cache'; + $key = is_file($metadata) ? file_get_contents($metadata) : false; + return is_string($key) && trim($key) === $this->getLinkCacheKey($objectFiles, $targetFile); + } + + /** @param list $objectFiles */ + private function writeLinkCache(array $objectFiles, string $targetFile): void + { + $metadata = $targetFile . '.typephp-link-cache'; + if (file_put_contents($metadata, $this->getLinkCacheKey($objectFiles, $targetFile) . PHP_EOL) === false) { + throw new \RuntimeException('Cannot write link cache metadata: ' . $metadata); + } + } + public function build(array $objectFiles): string { $targetFile = $this->getTargetFileName(); @@ -2316,6 +2501,12 @@ CODE; } } + if ($this->hasLinkCache($objectFiles, $targetFile)) { + $this->climate->darkGray('[incremental] link cache: ' . $targetFile); + $this->climate->green('Build successful: ' . $targetFile); + return $targetFile; + } + $buildError = null; $result = $this->getNativeBuilder()->link($objectFiles, $targetFile, $this->getLinkCommandOptions()); $this->climate->comment($result['command']); @@ -2335,6 +2526,7 @@ CODE; if ($this->isNanoMode()) { $this->auditNanoArtifact($objectFiles, $targetFile); } + $this->writeLinkCache($objectFiles, $targetFile); $this->climate->green('Build successful: ' . $targetFile); @@ -2441,20 +2633,34 @@ CODE; } public function genFunctionDeclarations(string $file): void + { + $this->writeFile($file, $this->renderFunctionDeclarations()); + } + + protected function renderFunctionDeclarations(?string $sourceFile = null): string { $code = '#pragma once' . PHP_EOL . PHP_EOL; $code .= '#include ' . PHP_EOL; $code .= '#include ' . PHP_EOL; $code .= '#include ' . PHP_EOL; + if ($sourceFile !== null && $this->declarationHeaderFiles !== []) { + $code .= '#include <' . $this->getRuntimeDeclarationHeaderName() . '>' . PHP_EOL; + foreach ($this->getDeclarationHeadersForFile($sourceFile, false) as $header) { + $code .= '#include <' . $header . '>' . PHP_EOL; + } + } $code .= PHP_EOL; - $code .= $this->genNativeObjectDeclarations(); + $code .= $this->genNativeObjectDeclarations($sourceFile); if ($this->isBuildModeLib()) { $code .= $this->genLibraryApiMacro($this->targetName); } $importLibraries = []; foreach ($this->symbols->functions() as $function) { + if ($sourceFile !== null && $function->sourceFile !== $sourceFile) { + continue; + } if ($this->isImportedFunction($function)) { $importLibraries[$function->importLibrary] = true; } @@ -2463,9 +2669,12 @@ CODE; $code .= $this->genLibraryImportMacro($library); } - $code .= $this->genDefaultArgumentHelperDeclarations(); + $code .= $this->genDefaultArgumentHelperDeclarations($sourceFile); foreach ($this->symbols->functions() as $name => $func) { + if ($sourceFile !== null && $func->sourceFile !== $sourceFile) { + continue; + } if ($func->abstractMethod) { continue; } @@ -2507,7 +2716,122 @@ CODE; } } - $this->writeFile($file, $code); + return $code; + } + + public function getDeclarationHeaderFile(string $file, bool $relative = false): string + { + $realFile = realpath($file) ?: $file; + $path = $this->getRelativePath(str_replace(['.stub.php', '.php'], '', $realFile)); + $name = $this->escapeFileName(str_replace(['/', '\\'], '_', $path)); + $name .= '_' . substr(hash('sha256', str_replace('\\', '/', $realFile)), 0, 10); + $absolute = $this->getIncludeDir() . '/php_' . $this->targetName . '_' . $name . '_decl.h'; + return $relative ? basename($absolute) : $absolute; + } + + private function getRuntimeDeclarationHeaderName(): string + { + return 'php_' . $this->targetName . '_runtime_decl.h'; + } + + private function getAllDeclarationHeaderName(): string + { + return 'php_' . $this->targetName . '_all_decl.h'; + } + + /** @param list $files */ + private function initializeDeclarationHeaderFiles(array $files): void + { + $this->declarationHeaderFiles = []; + foreach ($files as $file) { + if (FileScanner::isPhpFile($file)) { + $this->declarationHeaderFiles[$file] = $this->getDeclarationHeaderFile($file, true); + } + } + } + + /** @return list */ + private function getDeclarationHeadersForFile(string $file, bool $includeOwn = true): array + { + $headers = []; + if ($includeOwn && isset($this->declarationHeaderFiles[$file])) { + $headers[] = $this->declarationHeaderFiles[$file]; + } + foreach ($this->symbolCallInFile[$file] ?? [] as $symbol) { + $dependencyFile = $this->symbolDeclInFile[$symbol] ?? null; + if ($dependencyFile === null || $dependencyFile === $file) { + continue; + } + if (isset($this->declarationHeaderFiles[$dependencyFile])) { + $headers[] = $this->declarationHeaderFiles[$dependencyFile]; + } + } + $headers = array_values(array_unique($headers)); + sort($headers, SORT_STRING); + return $headers; + } + + /** @param list $files */ + private function genDeclarationHeaders(array $files): void + { + foreach ([ + $this->getIncludeDir() . '/php_' . $this->targetName . '_func_decl.h', + $this->getIncludeDir() . '/php_' . $this->targetName . '_data_decl.h', + ] as $legacyHeader) { + if (is_file($legacyHeader)) { + @unlink($legacyHeader); + } + } + + $manifest = $this->getBuildDir() . '/cache/incremental/' . $this->targetName + . '/declaration-headers.json'; + $previous = []; + if (is_file($manifest)) { + $decoded = json_decode((string) file_get_contents($manifest), true); + if (is_array($decoded)) { + $previous = array_values(array_filter($decoded, 'is_string')); + } + } + $current = array_values($this->declarationHeaderFiles); + foreach (array_diff($previous, $current) as $staleHeader) { + $path = $this->getIncludeDir() . '/' . basename($staleHeader); + if (is_file($path)) { + @unlink($path); + } + } + + $runtimeHeader = $this->getIncludeDir() . '/' . $this->getRuntimeDeclarationHeaderName(); + $this->writeFile($runtimeHeader, '#pragma once' . PHP_EOL . PHP_EOL + . $this->renderDataDeclarations(null, true) + . $this->genNativeObjectForwardDeclarations()); + foreach ($files as $file) { + if (!isset($this->declarationHeaderFiles[$file])) { + continue; + } + if (!$this->shouldRegeneratePhpFile($file)) { + continue; + } + $code = $this->renderFunctionDeclarations($file); + $code .= $this->renderDataDeclarations($file); + $this->writeFile( + $this->getIncludeDir() . '/' . $this->declarationHeaderFiles[$file], + $code, + $this->shouldRegeneratePhpFile($file), + ); + } + $allDeclarations = '#pragma once' . PHP_EOL . PHP_EOL; + $allDeclarations .= '#include <' . $this->getRuntimeDeclarationHeaderName() . '>' . PHP_EOL; + foreach ($current as $header) { + $allDeclarations .= '#include <' . $header . '>' . PHP_EOL; + } + $this->writeFile( + $this->getIncludeDir() . '/' . $this->getAllDeclarationHeaderName(), + $allDeclarations, + ); + $this->writeFile( + $manifest, + json_encode($current, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . PHP_EOL, + ); } protected function getLibraryApiMacroName(): string @@ -2666,7 +2990,7 @@ CODE; return $absPath; } - public function genIncludeHeaderFiles(): string + public function genIncludeHeaderFiles(bool $allDeclarations = false): string { $globalHeaders = $this->isNanoMode() ? [ @@ -2690,10 +3014,24 @@ CODE; 'std/random.h', ] : $this->globalHeaders; - $headers = array_merge($globalHeaders, [ - "php_{$this->targetName}_func_decl.h", - "php_{$this->targetName}_data_decl.h", - ], $this->localHeaders); + if ($this->declarationHeaderFiles === []) { + $declarationHeaders = [ + "php_{$this->targetName}_func_decl.h", + "php_{$this->targetName}_data_decl.h", + ]; + } elseif ($allDeclarations) { + $declarationHeaders = [ + $this->getRuntimeDeclarationHeaderName(), + ...array_values($this->declarationHeaderFiles), + ]; + } else { + $declarationHeaders = [ + $this->getRuntimeDeclarationHeaderName(), + ...$this->getDeclarationHeadersForFile($this->file), + ]; + } + $headers = array_merge($globalHeaders, $declarationHeaders, $this->localHeaders); + $headers = array_values(array_unique($headers)); $lines = []; foreach ($headers as $header) { $lines[] = '#include <' . $header . '>'; @@ -3530,6 +3868,7 @@ CODE; $stmts = $traverser->traverse($ast); + $this->cacheSitePass = 'body'; $this->resetFile(); $this->resetNamespace(); $this->resetClass(); @@ -3749,7 +4088,20 @@ CODE; $this->climate->info('generate arginfo file: ' . $this->getRelativePath($file)); generateStubFile($file, $this->getIncludeDir() . '/' . $headerFile, true, $this->getPhpVersion()); + $this->registerArgInfoHeader($headerFile); + } + + protected function registerExistingArgInfoHeader(string $file): void + { + $this->registerArgInfoHeader($this->getArgInfoHeaderFile($file, true)); + } + + private function registerArgInfoHeader(string $headerFile): void + { $headerCode = file_get_contents($this->getBuildDir() . '/include/' . $headerFile); + if (!is_string($headerCode)) { + throw new \RuntimeException('Cannot read generated arginfo header: ' . $headerFile); + } if (preg_match('/\\bstatic\\s+void\\s+(typephp_release_ast_constants_[A-Za-z0-9_]+)\\s*\\(void\\)/', $headerCode, $releaseMatch)) { $this->releaseAstConstantFns[] = $releaseMatch[1]; } @@ -3762,7 +4114,9 @@ CODE; $this->registerSymbols[] = $registerSymbolFn; } } - $this->argInfoHeaderFiles[] = $headerFile; + if (!in_array($headerFile, $this->argInfoHeaderFiles, true)) { + $this->argInfoHeaderFiles[] = $headerFile; + } } public function composeTraitAst(Node\Stmt\ClassLike $stmt, Node\Name $className): void