From 4aba3023cc606ba62005b11323a3d18ce4d83d67 Mon Sep 17 00:00:00 2001 From: tianfenghan Date: Tue, 15 Sep 2026 15:27:34 +0800 Subject: [PATCH] feat(compiler): implement AST caching and prepared project caching for improved performance - Add loadPristineAst method to CompilerBase for independent AST loading - Introduce PreparedProjectCacheTrait with conservative whole-project caching mechanism - Implement translation unit splitting for large C++ files with splitLargeTranslationUnit - Add preprocessing warning count tracking in CompilerDiagnosticTrait - Enhance gen_stub.php to use cached AST when available instead of reparsing - Improve incremental compilation checks with splitTranslationUnits validation - Add comprehensive test coverage for AST caching, prepared project cache, and translation unit splitting - Optimize precompiled header manager with dependency digest caching and improved invalidation - Implement file digest memoization for faster native compilation input processing - Add support for persistent AST cache reuse across compiler processes --- phpunit/src/Build/AstCacheTest.php | 8 + .../Build/PrecompiledHeaderManagerTest.php | 24 +++ .../src/Build/PreparedProjectCacheTest.php | 92 +++++++++++ .../src/Build/TranslationUnitSplitTest.php | 110 ++++++++++++ src/Build/IncrementalCompilationTrait.php | 9 + src/Build/PrecompiledHeaderManager.php | 39 ++++- src/Build/PreparedProjectCacheTrait.php | 156 ++++++++++++++++++ src/Build/SourcePipelineTrait.php | 25 +++ src/Build/TranslationUnitSplitTrait.php | 80 +++++++++ src/CompilerBase.php | 11 ++ src/Diagnostics/CompilerDiagnosticTrait.php | 2 + src/Translator.php | 61 +++++-- src/gen_stub.php | 9 +- 13 files changed, 610 insertions(+), 16 deletions(-) create mode 100644 phpunit/src/Build/PreparedProjectCacheTest.php create mode 100644 phpunit/src/Build/TranslationUnitSplitTest.php create mode 100644 src/Build/PreparedProjectCacheTrait.php create mode 100644 src/Build/TranslationUnitSplitTrait.php diff --git a/phpunit/src/Build/AstCacheTest.php b/phpunit/src/Build/AstCacheTest.php index 4a4cd0f8..ad4e3332 100644 --- a/phpunit/src/Build/AstCacheTest.php +++ b/phpunit/src/Build/AstCacheTest.php @@ -63,6 +63,7 @@ final class AstCacheTest extends TestCase self::assertFalse($convertAst[0]->getAttribute('typephpPhaseMutation', false)); self::assertDirectoryExists($this->buildDirectory . '/cache/ast'); self::assertCount(1, glob($this->buildDirectory . '/cache/ast/*.ast')); + } public function testPersistentCacheIsReusedByAnotherCompilerProcess(): void @@ -167,6 +168,13 @@ final class AstCacheTest extends TestCase self::assertNotNull($generated); self::assertFileExists($generated); self::assertCount(1, glob($this->buildDirectory . '/cache/ast/*.ast')); + // Stub visitors replace names; they require an independent pristine AST. + $stub = \TypePhp\StubGenerator\FileInfo::parseStubFile( + $source, $compiler->getPhpVersion(), $this->sourceFile, + ); + self::assertNotNull($stub); + self::assertSame(1, $parser->parseCount); + self::assertCount(1, glob($this->buildDirectory . '/cache/ast/*.ast')); } private function createCountingParser(): CountingParser diff --git a/phpunit/src/Build/PrecompiledHeaderManagerTest.php b/phpunit/src/Build/PrecompiledHeaderManagerTest.php index 7af56606..21441da2 100644 --- a/phpunit/src/Build/PrecompiledHeaderManagerTest.php +++ b/phpunit/src/Build/PrecompiledHeaderManagerTest.php @@ -95,6 +95,30 @@ final class PrecompiledHeaderManagerTest extends TestCase $this->assertNotSame($first['artifact'], $refreshed['artifact']); } + public function testSameSizeRewritePreservingMtimeInvalidatesDigest(): void + { + $dependencyDirectory = $this->cacheDirectory . '/dependencies'; + mkdir($dependencyDirectory); + $dependency = $dependencyDirectory . '/runtime.h'; + file_put_contents($dependency, '#define VALUE 1'); + $mtime = time() - 60; + touch($dependency, $mtime); + $backend = $this->createMock(CompilerBackend::class); + $backend->method('supportsPrecompiledHeaders')->willReturn(true); + $backend->method('getCompilerCommand')->willReturn('true'); + $backend->method('getPrecompiledHeaderArtifact') + ->willReturnCallback(static fn(string $header): string => $header . '.gch'); + $backend->method('buildNativeCompileCommand') + ->willReturnCallback(static fn(string $source, string $object): string => 'touch ' . escapeshellarg($object)); + $manager = new PrecompiledHeaderManager($backend, new NativeBuilder($backend)); + $first = $manager->prepare(['runtime.h'], [$dependencyDirectory], $this->cacheDirectory, new CompileOptions([])); + file_put_contents($dependency, '#define VALUE 2'); + touch($dependency, $mtime); + $second = $manager->prepare(['runtime.h'], [$dependencyDirectory], $this->cacheDirectory, new CompileOptions([])); + self::assertNotSame($first['artifact'], $second['artifact']); + self::assertFalse($second['cached']); + } + private function createCacheEntry(int $number, int $mtime): string { $directory = $this->cacheDirectory . '/' . sprintf('%024x', $number); diff --git a/phpunit/src/Build/PreparedProjectCacheTest.php b/phpunit/src/Build/PreparedProjectCacheTest.php new file mode 100644 index 00000000..5d4188fc --- /dev/null +++ b/phpunit/src/Build/PreparedProjectCacheTest.php @@ -0,0 +1,92 @@ +directory = sys_get_temp_dir() . '/typephp_prepared_' . bin2hex(random_bytes(8)); + mkdir($this->directory); + file_put_contents($this->directory . '/source.php', <<<'PHP' +directory, \FilesystemIterator::SKIP_DOTS), \RecursiveIteratorIterator::CHILD_FIRST); + foreach ($iterator as $entry) { + $entry->isDir() ? rmdir($entry->getPathname()) : unlink($entry->getPathname()); + } + rmdir($this->directory); + } + + private function compiler(): CompilerTest + { + global $translator; + $compiler = CompilerTest::create(dirname(__DIR__, 3)); + $compiler->setTargetName('prepared'); + $compiler->setBuildMode(CompilerBase::BUILD_MODE_EXT); + $this->invoke($compiler, 'setBuildDir', $this->directory . '/build'); + $translator = $compiler; + return $compiler; + } + + private function invoke(CompilerTest $compiler, string $method, mixed ...$arguments): mixed + { + return (new \ReflectionMethod($compiler, $method))->invoke($compiler, ...$arguments); + } + + public function testPreparedSymbolsTraitsAndDefaultsRoundTrip(): void + { + $file = $this->directory . '/source.php'; + $first = $this->compiler(); + $key = $this->invoke($first, 'preparedProjectKey', [$file]); + $first->prepareFile($file); + $this->invoke($first, 'composeTraitDeclarations', [$file]); + $this->invoke($first, 'discoverNativeGlobalObjects', [$file]); + $this->invoke($first, 'storePreparedProject', $key); + $first->convert([$file]); + $extension = file_get_contents($this->directory . '/build/extension-prepared.cc'); + $second = $this->compiler(); + self::assertTrue($this->invoke($second, 'restorePreparedProject', $key)); + self::assertTrue($this->invoke($second, 'hasClass', 'Prepared\\Child')); + $second->convert([$file]); + self::assertSame($extension, file_get_contents($this->directory . '/build/extension-prepared.cc')); + self::assertFalse($this->invoke($second, 'restorePreparedProject', $key)); + } + + public function testSourceChangesInvalidateEvenWithTheSameMtime(): void + { + $file = $this->directory . '/source.php'; + $compiler = $this->compiler(); + $key = $this->invoke($compiler, 'preparedProjectKey', [$file]); + $mtime = filemtime($file); + file_put_contents($file, str_replace('42', '43', file_get_contents($file))); + touch($file, $mtime); + self::assertNotSame($key, $this->invoke($compiler, 'preparedProjectKey', [$file])); + } + + public function testCorruptSnapshotFallsBackWithoutInstallingSymbols(): void + { + $compiler = $this->compiler(); + $file = $this->invoke($compiler, 'preparedProjectCacheFile'); + mkdir(dirname($file), 0777, true); + $key = str_repeat('a', 64); + file_put_contents($file, $key . "\ncorrupt"); + self::assertFalse($this->invoke($compiler, 'restorePreparedProject', $key)); + self::assertFalse($this->invoke($compiler, 'hasClass', 'Prepared\\Child')); + } +} diff --git a/phpunit/src/Build/TranslationUnitSplitTest.php b/phpunit/src/Build/TranslationUnitSplitTest.php new file mode 100644 index 00000000..5ccc5a4a --- /dev/null +++ b/phpunit/src/Build/TranslationUnitSplitTest.php @@ -0,0 +1,110 @@ +directory = sys_get_temp_dir() . '/typephp_split_' . bin2hex(random_bytes(8)); + mkdir($this->directory); + $this->source = $this->directory . '/source.php'; + file_put_contents($this->source, 'compiler = CompilerTest::create($this->directory); + $translator = $this->compiler; + $this->compiler->setTargetName('split'); + $this->compiler->prepareFile($this->source); + } + + protected function tearDown(): void + { + $iterator = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator( + $this->directory, \FilesystemIterator::SKIP_DOTS), \RecursiveIteratorIterator::CHILD_FIRST); + foreach ($iterator as $entry) { + $entry->isDir() ? rmdir($entry->getPathname()) : unlink($entry->getPathname()); + } + rmdir($this->directory); + } + + private function invoke(string $method, mixed ...$arguments): mixed + { + return (new \ReflectionMethod($this->compiler, $method))->invoke($this->compiler, ...$arguments); + } + + private function set(string $property, mixed $value): void + { + $reflection = new \ReflectionClass($this->compiler); + while (!$reflection->hasProperty($property)) { + $reflection = $reflection->getParentClass(); + } + $reflection->getProperty($property)->setValue($this->compiler, $value); + } + + public function testCompleteEntitiesKeepTheirLocalDataAndStalePartsAreRemoved(): void + { + $primary = $this->invoke('getCppFile', $this->source); + $body = "void large_method() {\n" . str_repeat("// body\n", 2000) . "use(blob);\n}\n"; + $small = "void small_helper() {}\n"; + $code = str_repeat("// padding\n", 200000) . $body . $small; + $this->set('splitTranslationUnitsEnabled', true); + $this->set('generatedMethodBodies', [$body, $small]); + $this->set('constData', ['blob' => '1,2,3', 'unused_blob' => '4,5,6']); + $remaining = $this->invoke('splitLargeTranslationUnit', $code, $primary, false); + $parts = $this->invoke('getSplitTranslationUnits', $this->source); + self::assertCount(1, $parts); + self::assertStringNotContainsString($body, $remaining); + self::assertStringContainsString($small, $remaining); + $part = file_get_contents($parts[0]); + self::assertStringContainsString($body, $part); + self::assertStringContainsString('static const unsigned char blob[]', $part); + self::assertStringNotContainsString('unused_blob', $part); + self::assertStringNotContainsString('_arginfo.h', $part); + $this->invoke('splitLargeTranslationUnit', $small, $primary, false); + self::assertSame([], $this->invoke('getSplitTranslationUnits', $this->source)); + self::assertFileDoesNotExist($parts[0]); + } + + public function testStandaloneConversionIsNotSilentlySplit(): void + { + $primary = $this->invoke('getCppFile', $this->source); + $body = str_repeat('// body', 2000); + $code = str_repeat('// prefix', 250000) . $body; + $this->set('generatedMethodBodies', [$body]); + self::assertSame($code, $this->invoke('splitLargeTranslationUnit', $code, $primary, false)); + self::assertSame([], $this->invoke('getSplitTranslationUnits', $this->source)); + } + + public function testSwitchingBackToStandaloneConversionRemovesOldParts(): void + { + $primary = $this->invoke('getCppFile', $this->source); + $body = "void large_method() {\n" . str_repeat("// body\n", 2000) . "}\n"; + $code = str_repeat("// padding\n", 200000) . $body; + $this->set('splitTranslationUnitsEnabled', true); + $this->set('generatedMethodBodies', [$body]); + $this->invoke('splitLargeTranslationUnit', $code, $primary, false); + $parts = $this->invoke('getSplitTranslationUnits', $this->source); + self::assertCount(1, $parts); + $this->set('splitTranslationUnitsEnabled', false); + self::assertSame($code, $this->invoke('splitLargeTranslationUnit', $code, $primary, false)); + self::assertSame([], $this->invoke('getSplitTranslationUnits', $this->source)); + self::assertFileDoesNotExist($parts[0]); + } + + public function testManifestCannotAuthorizeAnUnrelatedPath(): void + { + $primary = $this->invoke('getCppFile', $this->source); + if (!is_dir(dirname($primary))) { + mkdir(dirname($primary), 0777, true); + } + file_put_contents($primary . '.parts.json', json_encode([$this->source, $primary . '/../other.cc'])); + self::assertSame([], $this->invoke('getSplitTranslationUnits', $this->source)); + } +} diff --git a/src/Build/IncrementalCompilationTrait.php b/src/Build/IncrementalCompilationTrait.php index 8a7e1ca2..36b1d589 100644 --- a/src/Build/IncrementalCompilationTrait.php +++ b/src/Build/IncrementalCompilationTrait.php @@ -63,6 +63,14 @@ trait IncrementalCompilationTrait return false; } $emits = $metadata['emitsTranslationUnit'] ?? null; + if (($metadata['splitTranslationUnits'] ?? []) !== $this->getSplitTranslationUnits($source)) { + return false; + } + foreach ($this->getSplitTranslationUnits($source) as $part) { + if (!is_file($part)) { + return false; + } + } if (!is_bool($emits)) { return false; } @@ -243,6 +251,7 @@ trait IncrementalCompilationTrait ?? (bool) ($previous['emitsTranslationUnit'] ?? false), 'header' => $this->getDeclarationHeaderFile($path), 'cpp' => $this->getCppFile($path), + 'splitTranslationUnits' => $this->getSplitTranslationUnits($path), 'statistics' => $this->incrementalFileMetadata[$path]['statistics'] ?? ($previous['statistics'] ?? []), 'globals' => $this->globalVarsInFile[$path] ?? [], diff --git a/src/Build/PrecompiledHeaderManager.php b/src/Build/PrecompiledHeaderManager.php index de403bc9..d415ac51 100644 --- a/src/Build/PrecompiledHeaderManager.php +++ b/src/Build/PrecompiledHeaderManager.php @@ -30,7 +30,7 @@ final readonly class PrecompiledHeaderManager throw new \LogicException($this->backend->getName() . ' does not support precompiled headers'); } - $fingerprint = $this->buildFingerprint($headers, $dependencyDirectories, $options); + $fingerprint = $this->buildFingerprint($headers, $dependencyDirectories, $options, $cacheDirectory); $directory = rtrim($cacheDirectory, '/\\') . DIRECTORY_SEPARATOR . $fingerprint; if (!is_dir($directory) && !mkdir($directory, 0777, true) && !is_dir($directory)) { throw new \RuntimeException('Cannot create precompiled header cache directory: ' . $directory); @@ -116,7 +116,7 @@ final readonly class PrecompiledHeaderManager } /** @param list $headers @param list $dependencyDirectories */ - private function buildFingerprint(array $headers, array $dependencyDirectories, CompileOptions $options): string + private function buildFingerprint(array $headers, array $dependencyDirectories, CompileOptions $options, string $cacheDirectory): string { $compilerVersion = []; exec(escapeshellcmd($this->backend->getCompilerCommand()) . ' --version 2>&1', $compilerVersion); @@ -148,7 +148,14 @@ final readonly class PrecompiledHeaderManager } } sort($files, SORT_STRING); + $digestFile = rtrim($cacheDirectory, '/\\') . '/dependency-digests.json'; + $saved = is_file($digestFile) ? json_decode((string) file_get_contents($digestFile), true) : []; + $digests = []; + // Windows ctime can mean creation time rather than metadata-change + // time, so it cannot safely detect in-place mtime-preserving rewrites. + $strict = PHP_OS_FAMILY === 'Windows' || getenv('TYPEPHP_STRICT_CACHE') === '1'; foreach ($files as $file) { + clearstatcache(true, $file); $metadata = stat($file); if ($metadata === false) { throw new \RuntimeException('Cannot stat precompiled-header dependency: ' . $file); @@ -158,10 +165,34 @@ final readonly class PrecompiledHeaderManager // the metadata as well as the content so such SDK refreshes select // a new cache entry before native compilation begins. hash_update($context, $file . "\0" . $metadata['size'] . "\0" . $metadata['mtime'] . "\0"); - if (!hash_update_file($context, $file)) { + $signature = [$metadata['dev'], $metadata['ino'], $metadata['size'], $metadata['mtime'], $metadata['ctime']]; + $entry = is_array($saved) ? ($saved[$file] ?? null) : null; + // Like mature timestamp-based build tools, reuse digests for stable + // SDK files. ctime detects rewrites preserving mtime; always read + // recent files to close PHP stat()'s same-second timestamp window. + $digest = !$strict && max($metadata['mtime'], $metadata['ctime']) < time() - 2 + && is_array($entry) && ($entry['signature'] ?? null) === $signature + && is_string($entry['digest'] ?? null) + && preg_match('/^[a-f0-9]{64}$/D', $entry['digest']) === 1 + ? $entry['digest'] : hash_file('sha256', $file); + if (!is_string($digest)) { throw new \RuntimeException('Cannot fingerprint precompiled-header dependency: ' . $file); } - hash_update($context, "\0"); + $digests[$file] = ['signature' => $signature, 'digest' => $digest]; + hash_update($context, $digest . "\0"); + } + + if ($digests !== $saved && is_dir($cacheDirectory)) { + $temporary = tempnam($cacheDirectory, '.pch-digests-'); + if ($temporary !== false) { + try { + if (file_put_contents($temporary, json_encode($digests, JSON_THROW_ON_ERROR)) !== false) { + @rename($temporary, $digestFile); + } + } finally { + @unlink($temporary); + } + } } return substr(hash_final($context), 0, 24); diff --git a/src/Build/PreparedProjectCacheTrait.php b/src/Build/PreparedProjectCacheTrait.php new file mode 100644 index 00000000..0f31de65 --- /dev/null +++ b/src/Build/PreparedProjectCacheTrait.php @@ -0,0 +1,156 @@ +{$field}; + } + return hash('sha256', serialize([ + 2, $this->getIncrementalGeneratorFingerprint(), $sources, PHP_VERSION, + $extensions, get_defined_constants(), $this->linkLibs, + $this->varIntTypes, $this->decimalTypes, $this->bigintTypes, $initialState, + ])); + } + + private function preparedProjectCacheFile(): string + { + return $this->buildDir . '/cache/prepared/' . hash('sha256', $this->targetName) . '.bin'; + } + + private function restorePreparedProject(string $key): bool + { + if ($this->climate->arguments->defined('force') || $this->preparedFileAsts !== [] + || $this->symbols->functions() !== [] || $this->symbols->classes() !== [] + || $this->symbols->interfaces() !== []) { + return false; + } + $file = $this->preparedProjectCacheFile(); + if (!is_file($file)) { + return false; + } + $stream = @fopen($file, 'rb'); + if ($stream === false) { + return false; + } + try { + // Reject changed inputs before deserializing the large AST graph. + if (fgets($stream) !== $key . "\n") { + return false; + } + $contents = stream_get_contents($stream); + $state = is_string($contents) ? @unserialize($contents) : false; + if (!is_array($state) || array_keys($state) !== self::PREPARED_PROJECT_FIELDS + || !($state['symbols'] instanceof \TypePhp\Symbol\SymbolRepository) + || ($state['nativeGlobalTypeResolver'] !== null + && !($state['nativeGlobalTypeResolver'] instanceof \TypePhp\NativeClass\NativeGlobalTypeResolver))) { + return false; + } + foreach (self::PREPARED_PROJECT_FIELDS as $field) { + if (get_debug_type($state[$field]) !== get_debug_type($this->{$field}) + && $field !== 'nativeGlobalTypeResolver') { + return false; + } + } + foreach (self::PREPARED_PROJECT_FIELDS as $field) { + $this->{$field} = $state[$field]; + } + $this->resetFile(); + $this->resetNamespace(); + $this->resetClass(); + $this->resetMethod(); + $this->resetFunction(); + $this->climate->darkGray('[cached] project preprocessing'); + return true; + } catch (\Throwable) { + return false; + } finally { + fclose($stream); + } + } + + private function storePreparedProject(string $key): void + { + // Imported stubs can live outside the scanned input set. Do not cache + // them until they are included in the snapshot's dependency manifest. + if ($this->externalImportStubFiles !== []) { + return; + } + $temporary = ''; + $bodies = []; + try { + // Conversion always loads a fresh pristine AST. The prepared tree + // is needed only for declaration expressions, not ordinary bodies. + // Keep trait templates intact: a missing output may require their + // bodies to be composed into a dirty consuming class on a cache hit. + $stack = []; + foreach ($this->preparedFileAsts as $statements) { + array_push($stack, ...$statements); + } + while ($stack !== []) { + $node = array_pop($stack); + if (!($node instanceof \PhpParser\Node) || $node instanceof \PhpParser\Node\Stmt\Trait_) { + continue; + } + if (($node instanceof \PhpParser\Node\Stmt\Function_ || $node instanceof \PhpParser\Node\Stmt\ClassMethod) + && $node->stmts !== null) { + $bodies[] = [$node, $node->stmts]; + $node->stmts = []; + } + foreach ($node->getSubNodeNames() as $name) { + $child = $node->{$name}; + if (is_array($child)) { + array_push($stack, ...$child); + } elseif ($child instanceof \PhpParser\Node) { + $stack[] = $child; + } + } + } + $state = []; + foreach (self::PREPARED_PROJECT_FIELDS as $field) { + $state[$field] = $this->{$field}; + } + $file = $this->preparedProjectCacheFile(); + if (!is_dir(dirname($file)) && !mkdir(dirname($file), 0777, true) && !is_dir(dirname($file))) { + return; + } + $candidate = tempnam(dirname($file), '.prepared-'); + if (is_string($candidate)) { + $temporary = $candidate; + $this->writeFile($temporary, $key . "\n" . serialize($state)); + @rename($temporary, $file); + } + } catch (\Throwable) { + // Optional cache failures must not fail a valid compilation. + } finally { + foreach ($bodies as [$node, $statements]) { + $node->stmts = $statements; + } + if (is_string($temporary) && is_file($temporary)) { + @unlink($temporary); + } + } + } +} diff --git a/src/Build/SourcePipelineTrait.php b/src/Build/SourcePipelineTrait.php index 06d8b5b8..cdfe53d9 100644 --- a/src/Build/SourcePipelineTrait.php +++ b/src/Build/SourcePipelineTrait.php @@ -19,6 +19,7 @@ use TypePhp\Platform\Windows; trait SourcePipelineTrait { + use PreparedProjectCacheTrait; /** * Prepare PHP inputs for the Composer php-nano source-composition build. * @@ -224,6 +225,14 @@ trait SourcePipelineTrait } $files = $this->filterIgnoredFiles($files); + $preparedKey = $this->preparedProjectKey($files); + if ($this->restorePreparedProject($preparedKey)) { + $files = $this->getSortedFiles($files); + $this->initializeIncrementalCompilation($files); + return $files; + } + $warningsBefore = $this->preprocessingWarningCount; + $inputCount = count($files); $this->discoverNativeClassDeclarations($files); // Analyze and preprocess the PHP files foreach ($files as $k => $file) { @@ -247,6 +256,9 @@ trait SourcePipelineTrait // pointer ABI now, after declarations are known and before the first // per-file C++ body is generated. $this->discoverNativeGlobalObjects(array_values($files)); + if (count($files) === $inputCount && $this->preprocessingWarningCount === $warningsBefore) { + $this->storePreparedProject($preparedKey); + } $files = $this->getSortedFiles($files); $this->initializeIncrementalCompilation($files); return $files; @@ -344,6 +356,8 @@ trait SourcePipelineTrait public function convert(array $files): array { $this->compilationStatistics->begin(); + $previousSplitSetting = $this->splitTranslationUnitsEnabled; + $this->splitTranslationUnitsEnabled = true; $previousPhase = null; try { $this->composeTraitDeclarations($files); @@ -376,6 +390,10 @@ trait SourcePipelineTrait $cppFile = $this->getCppFile($path); $this->registerGeneratedProjectSource($cppFile); $sourceFiles[] = $cppFile; + foreach ($this->getSplitTranslationUnits($path) as $part) { + $this->registerGeneratedProjectSource($part); + $sourceFiles[] = $part; + } } $this->climate->darkGray( '[cached] ' . $this->getRelativePath($path), @@ -402,6 +420,12 @@ trait SourcePipelineTrait $validSourceCount++; if ($cppFile !== null) { $sourceFiles[] = $cppFile; + if (FileScanner::isPhpFile($file)) { + foreach ($this->getSplitTranslationUnits($file) as $part) { + $this->registerGeneratedProjectSource($part); + $sourceFiles[] = $part; + } + } } } catch (Unsupported $e) { echo ' unsupported syntax: ' . $e->getMessage() . "\n"; @@ -446,6 +470,7 @@ trait SourcePipelineTrait return $sourceFiles; } finally { + $this->splitTranslationUnitsEnabled = $previousSplitSetting; if ($previousPhase !== null) { $this->restoreCompilerPhase($previousPhase); } diff --git a/src/Build/TranslationUnitSplitTrait.php b/src/Build/TranslationUnitSplitTrait.php new file mode 100644 index 00000000..68718d5d --- /dev/null +++ b/src/Build/TranslationUnitSplitTrait.php @@ -0,0 +1,80 @@ + */ + private array $generatedMethodBodies = []; + + protected function getSplitTranslationUnits(string $source): array + { + $primary = $this->getCppFile($source); + $manifest = $primary . '.parts.json'; + $parts = is_file($manifest) ? json_decode((string) file_get_contents($manifest), true) : []; + if (!is_array($parts)) { + return []; + } + // A manifest is disposable cache data, not authority to read/delete + // arbitrary paths supplied by a damaged or edited cache file. + return array_values(array_filter($parts, static fn($part): bool => is_string($part) + && preg_match('/^' . preg_quote($primary, '/') . '\\.part-[0-9]+\\.cc$/D', $part) === 1)); + } + + private function splitLargeTranslationUnit(string $code, string $primary, bool $force): string + { + $oldParts = $this->getSplitTranslationUnits($this->file); + $parts = []; + if ($this->splitTranslationUnitsEnabled && strlen($code) >= 2 * 1024 * 1024) { + $groups = []; + $group = ''; + foreach ($this->generatedMethodBodies as $body) { + // Keep small helpers together in the primary TU, retaining + // useful inlining; move only large, complete method entities. + if (strlen($body) < 8192) { + continue; + } + $position = strpos($code, $body); + if ($position === false) { + continue; + } + $code = substr_replace($code, '', $position, strlen($body)); + if ($group !== '' && strlen($group) + strlen($body) > 384 * 1024) { + $groups[] = $group; + $group = ''; + } + $group .= $body . "\n"; + } + if ($group !== '') { + $groups[] = $group; + } + $includes = $this->genIncludeHeaderFiles(); + foreach ($groups as $index => $body) { + $data = ''; + foreach ($this->constData as $name => $value) { + if (str_contains($body, $name)) { + $data .= 'static const unsigned char ' . $name . '[] = {' . $value . "};\n"; + } + } + $part = $primary . '.part-' . $index . '.cc'; + $this->save($includes . $data . "\n" . $body, $part, $force); + $this->registerGeneratedProjectSource($part); + $parts[] = $part; + } + } + if ($this->splitTranslationUnitsEnabled || $oldParts !== []) { + $this->writeFile($primary . '.parts.json', json_encode($parts, JSON_THROW_ON_ERROR)); + } + foreach (array_diff($oldParts, $parts) as $obsolete) { + foreach ([$obsolete, $this->getObjectFile($obsolete), + $this->getMiscObjectCacheMetadataFile($this->getObjectFile($obsolete))] as $artifact) { + if (is_file($artifact)) { + unlink($artifact); + } + } + } + return $code; + } +} diff --git a/src/CompilerBase.php b/src/CompilerBase.php index 3b52d3d3..0e636ec1 100644 --- a/src/CompilerBase.php +++ b/src/CompilerBase.php @@ -5299,6 +5299,16 @@ class CompilerBase implements PropertyAccessContext return $this->astCache ??= new AstCache($this->parser, $this->buildDir, $this->phpVersion); } + /** Return an independent pristine tree for each consumer's own visitors. */ + public function loadPristineAst(string $file, string $source, string $phpVersion): array + { + if ($phpVersion === $this->phpVersion) { + return $this->getAstCache()->load($file, $source); + } + $parser = (new ParserFactory())->createForVersion(PhpVersion::fromString($phpVersion)); + return (new AstCache($parser, $this->buildDir, $phpVersion))->load($file, $source); + } + protected function getStableIdRegistry(): StableIdRegistry { if ($this->stableIdRegistry === null) { @@ -5320,6 +5330,7 @@ class CompilerBase implements PropertyAccessContext { $targets = [ $this->buildDir . '/cache/ast', + $this->buildDir . '/cache/prepared', $this->buildDir . '/cache/incremental/' . ($this->targetName !== '' ? $this->targetName : 'default'), ]; foreach ($targets as $directory) { diff --git a/src/Diagnostics/CompilerDiagnosticTrait.php b/src/Diagnostics/CompilerDiagnosticTrait.php index b554995b..400d2f7d 100644 --- a/src/Diagnostics/CompilerDiagnosticTrait.php +++ b/src/Diagnostics/CompilerDiagnosticTrait.php @@ -13,6 +13,7 @@ use PhpParser\Node\Expr\Variable; trait CompilerDiagnosticTrait { + protected int $preprocessingWarningCount = 0; /** * Report a compiler fatal error. */ @@ -28,6 +29,7 @@ trait CompilerDiagnosticTrait protected function warning(Node $node, string $msg): void { + $this->preprocessingWarningCount++; $this->getDiagnosticReporter()->warning($node, $this->file, $msg); } diff --git a/src/Translator.php b/src/Translator.php index a33f2f17..46122864 100644 --- a/src/Translator.php +++ b/src/Translator.php @@ -25,6 +25,7 @@ use TypePhp\Build\NativeDependencyAuditor; use TypePhp\Build\NanoSourceComposer; use TypePhp\Build\PrecompiledHeaderManager; use TypePhp\Build\SourcePipelineTrait; +use TypePhp\Build\TranslationUnitSplitTrait; use TypePhp\Build\SourceCompileQueue; use TypePhp\Build\WasmInterfaceGenerator; use TypePhp\Config\ProjectYamlLoader; @@ -82,6 +83,7 @@ class Translator extends Preprocessor use IncrementalCompilationTrait; use NativeCommandOptionsTrait; use SourcePipelineTrait; + use TranslationUnitSplitTrait; use ResourceCompilationTrait; use ClassConstantValueTrait; @@ -759,8 +761,9 @@ class Translator extends Preprocessor while (true) { try { $cppCode = $this->doConvert($phpCode); - $this->recordEmittedTypes($cppCode); $cppFile = $this->getCppFile($file); + $this->recordEmittedTypes($cppCode); + $cppCode = $this->splitLargeTranslationUnit($cppCode, $cppFile, $forceWrite); if ($cppCode === '') { $this->removeEmptyTranslationUnitArtifacts($cppFile); } else { @@ -1882,6 +1885,10 @@ CODE; return hash_final($context); } + private bool $memoizeGeneratedCompileInputs = false; + /** @var array */ + private array $generatedCompileInputCache = []; + /** @param array $visited */ private function hashGeneratedCompileInput(\HashContext $context, string $file, array &$visited): void { @@ -1895,20 +1902,24 @@ CODE; return; } $visited[$real] = true; - $contents = @file_get_contents($real); - if (!is_string($contents)) { - return; - } - hash_update($context, str_replace('\\', '/', $real) . "\0" . $contents . "\0"); - $matchCount = preg_match_all('/^\s*#\s*include\s*[<\"]([^>\"]+)[>\"]/m', $contents, $matches); - if ($matchCount === false || $matchCount === 0) { - return; + $input = $this->memoizeGeneratedCompileInputs ? ($this->generatedCompileInputCache[$real] ?? null) : null; + if ($input === null) { + $contents = @file_get_contents($real); + if (!is_string($contents)) { + return; + } + preg_match_all('/^\s*#\s*include\s*[<\"]([^>\"]+)[>\"]/m', $contents, $matches); + $input = ['digest' => hash('sha256', $contents), 'headers' => $matches[1] ?? []]; + if ($this->memoizeGeneratedCompileInputs) { + $this->generatedCompileInputCache[$real] = $input; + } } + hash_update($context, 'digest-v2' . "\0" . str_replace('\\', '/', $real) . "\0" . $input['digest'] . "\0"); $includeRoot = realpath($this->getIncludeDir()); if ($includeRoot === false) { return; } - foreach ($matches[1] as $header) { + foreach ($input['headers'] as $header) { $candidate = $includeRoot . DIRECTORY_SEPARATOR . str_replace('/', DIRECTORY_SEPARATOR, $header); $resolved = realpath($candidate); if ($resolved === false @@ -2278,6 +2289,18 @@ CODE; } protected function compileSourceFile(array $sourceFiles): array + { + $this->memoizeGeneratedCompileInputs = true; + $this->generatedCompileInputCache = []; + try { + return $this->compileSourceFileUncached($sourceFiles); + } finally { + $this->memoizeGeneratedCompileInputs = false; + $this->generatedCompileInputCache = []; + } + } + + private function compileSourceFileUncached(array $sourceFiles): array { $objectFiles = []; $totalFiles = count($sourceFiles); @@ -2323,6 +2346,20 @@ CODE; } protected function compileWithProcessPool(array $sourceFiles, int $job): array + { + // Generation/formatting is finished. Share immutable file digests only + // for this native build, including completion callbacks, then discard. + $this->memoizeGeneratedCompileInputs = true; + $this->generatedCompileInputCache = []; + try { + return $this->compileWithProcessPoolUncached($sourceFiles, $job); + } finally { + $this->memoizeGeneratedCompileInputs = false; + $this->generatedCompileInputCache = []; + } + } + + private function compileWithProcessPoolUncached(array $sourceFiles, int $job): array { $totalFiles = count($sourceFiles); $this->climate->lightBlue("Starting parallel compilation with {$job} jobs for {$totalFiles} files"); @@ -4078,6 +4115,7 @@ CODE; protected function doConvert(string $phpCode): string { + $this->generatedMethodBodies = []; $this->climate->info('convert: ' . $this->getRelativePath($this->file)); $ast = $this->getAstCache()->load($this->file, $phpCode); @@ -8072,6 +8110,9 @@ CODE; // only run in the implementation phase. $this->checkParentMethodCanBeOverridden($v, $name); $methodCodes[$name] = $this->parseFunction($v); + if ($this->splitTranslationUnitsEnabled && !$this->classDef->nativeObject && $methodCodes[$name] !== '') { + $this->generatedMethodBodies[] = $methodCodes[$name]; + } } elseif ($this->classDef->trait === null && !is_string($v->getAttribute(self::TRAIT_ORIGIN_ATTRIBUTE)) && $this->classDef->hasAbstractMethod($name) diff --git a/src/gen_stub.php b/src/gen_stub.php index 1c4a5f50..2fb24636 100755 --- a/src/gen_stub.php +++ b/src/gen_stub.php @@ -5028,7 +5028,6 @@ class FileInfo { } public static function parseStubFile(string $code, string $phpVersion = '8.5', string $sourceFile = ''): FileInfo { - $parser = (new PhpParser\ParserFactory())->createForVersion(PhpParser\PhpVersion::fromString($phpVersion)); $nodeTraverser = new PhpParser\NodeTraverser; $nodeTraverser->addVisitor(new PhpParser\NodeVisitor\NameResolver( null, @@ -5046,7 +5045,13 @@ class FileInfo { } }; - $stmts = $parser->parse($code); + if ($sourceFile !== '' && is_file($sourceFile)) { + $stmts = getTranslator()->loadPristineAst($sourceFile, $code, $phpVersion); + } else { + // Preserve the standalone API for source strings without a file. + $parser = (new PhpParser\ParserFactory())->createForVersion(PhpParser\PhpVersion::fromString($phpVersion)); + $stmts = $parser->parse($code); + } $stmts = $nodeTraverser->traverse($stmts); $fileTags = DocCommentTag::parseDocComments(self::getFileDocComments($stmts));