From 58fe7b5d3cfd629c34c615cd4aec3433e8a6e784 Mon Sep 17 00:00:00 2001 From: tianfenghan Date: Mon, 14 Sep 2026 19:30:00 +0800 Subject: [PATCH] feat(build): add AST cache for persistent php-parser output optimization - Implemented AstCache class with schema versioning and file-based storage - Added cache validation using source hash, PHP version, and parser version - Integrated atomic file operations to ensure cache consistency - Updated CompilerBase to use lazy-initialized AST cache instance - Replaced direct parser calls with cache-aware load operations in preprocessor - Modified translator to utilize cached AST instead of reparsing - Added comprehensive test coverage including cache hit, invalid --- phpunit/src/Build/AstCacheTest.php | 193 +++++++++++++++++++++++++++++ src/Build/AstCache.php | 180 +++++++++++++++++++++++++++ src/CompilerBase.php | 9 ++ src/Preprocessor.php | 8 +- src/Translator.php | 2 +- 5 files changed, 387 insertions(+), 5 deletions(-) create mode 100644 phpunit/src/Build/AstCacheTest.php create mode 100644 src/Build/AstCache.php diff --git a/phpunit/src/Build/AstCacheTest.php b/phpunit/src/Build/AstCacheTest.php new file mode 100644 index 00000000..4a4cd0f8 --- /dev/null +++ b/phpunit/src/Build/AstCacheTest.php @@ -0,0 +1,193 @@ +parseCount; + return $this->delegate->parse($code, $errorHandler); + } + + public function getTokens(): array + { + return $this->delegate->getTokens(); + } +} + +final class AstCacheTest extends TestCase +{ + private string $directory; + private string $sourceFile; + private string $buildDirectory; + + protected function setUp(): void + { + $this->directory = sys_get_temp_dir() . '/typephp_ast_cache_' . bin2hex(random_bytes(8)); + $this->buildDirectory = $this->directory . '/custom-build'; + mkdir($this->buildDirectory, 0777, true); + $this->sourceFile = $this->directory . '/program.php'; + } + + protected function tearDown(): void + { + $this->removeDirectory($this->directory); + } + + public function testCacheHitSkipsParserAndReturnsAnIndependentAst(): void + { + $source = "sourceFile, $source); + $parser = $this->createCountingParser(); + $cache = new AstCache($parser, $this->buildDirectory, '8.4.0', 'test-parser'); + + $prepareAst = $cache->load($this->sourceFile, $source); + $prepareAst[0]->setAttribute('typephpPhaseMutation', true); + $convertAst = $cache->load($this->sourceFile, $source); + + self::assertSame(1, $parser->parseCount); + self::assertFalse($convertAst[0]->getAttribute('typephpPhaseMutation', false)); + self::assertDirectoryExists($this->buildDirectory . '/cache/ast'); + self::assertCount(1, glob($this->buildDirectory . '/cache/ast/*.ast')); + } + + public function testPersistentCacheIsReusedByAnotherCompilerProcess(): void + { + $source = "sourceFile, $source); + $firstParser = $this->createCountingParser(); + (new AstCache($firstParser, $this->buildDirectory, '8.4.0', 'test-parser')) + ->load($this->sourceFile, $source); + + $secondParser = $this->createCountingParser(); + $ast = (new AstCache($secondParser, $this->buildDirectory, '8.4.0', 'test-parser')) + ->load($this->sourceFile, $source); + + self::assertSame(1, $firstParser->parseCount); + self::assertSame(0, $secondParser->parseCount); + self::assertNotEmpty($ast); + } + + public function testNewerOrChangedSourceInvalidatesCache(): void + { + $source = "sourceFile, $source); + $parser = $this->createCountingParser(); + $cache = new AstCache($parser, $this->buildDirectory, '8.4.0', 'test-parser'); + $cache->load($this->sourceFile, $source); + $cacheFile = glob($this->buildDirectory . '/cache/ast/*.ast')[0]; + + $changed = "sourceFile, $changed); + touch($this->sourceFile, filemtime($cacheFile) + 2); + $cache->load($this->sourceFile, $changed); + + self::assertSame(2, $parser->parseCount); + } + + public function testContentHashAndMetadataRejectStaleOrCorruptCache(): void + { + $source = "sourceFile, $source); + $parser = $this->createCountingParser(); + $cache = new AstCache($parser, $this->buildDirectory, '8.4.0', 'test-parser'); + $cache->load($this->sourceFile, $source); + $cacheFile = glob($this->buildDirectory . '/cache/ast/*.ast')[0]; + $cacheMtime = filemtime($cacheFile); + + $sameSizeChange = "sourceFile, $sameSizeChange); + touch($this->sourceFile, $cacheMtime); + $cache->load($this->sourceFile, $sameSizeChange); + self::assertSame(2, $parser->parseCount); + + file_put_contents($cacheFile, 'broken-cache'); + touch($cacheFile, time() + 2); + $cache->load($this->sourceFile, $sameSizeChange); + self::assertSame(3, $parser->parseCount); + } + + public function testParserVersionChangeInvalidatesCache(): void + { + $source = "sourceFile, $source); + $firstParser = $this->createCountingParser(); + (new AstCache($firstParser, $this->buildDirectory, '8.4.0', 'parser-v1')) + ->load($this->sourceFile, $source); + + $secondParser = $this->createCountingParser(); + (new AstCache($secondParser, $this->buildDirectory, '8.4.0', 'parser-v2')) + ->load($this->sourceFile, $source); + + self::assertSame(1, $firstParser->parseCount); + self::assertSame(1, $secondParser->parseCount); + } + + public function testPrepareAndConvertShareOneParseAndRespectBuildDirectory(): void + { + global $translator; + + $source = "sourceFile, $source); + $compiler = CompilerTest::create($this->directory); + $translator = $compiler; + $reflection = new \ReflectionClass($compiler); + $setBuildDir = $reflection->getMethod('setBuildDir'); + $setBuildDir->setAccessible(true); + $setBuildDir->invoke($compiler, $this->buildDirectory); + + $parser = $this->createCountingParser(); + $parserProperty = $reflection->getProperty('parser'); + $parserProperty->setAccessible(true); + $parserProperty->setValue($compiler, $parser); + $astCacheProperty = $reflection->getProperty('astCache'); + $astCacheProperty->setAccessible(true); + $astCacheProperty->setValue($compiler, null); + + $compiler->addFiles([$this->sourceFile]); + $compiler->prepareFile($this->sourceFile); + $generated = $compiler->convertFile($this->sourceFile); + + self::assertSame(1, $parser->parseCount); + self::assertNotNull($generated); + self::assertFileExists($generated); + self::assertCount(1, glob($this->buildDirectory . '/cache/ast/*.ast')); + } + + private function createCountingParser(): CountingParser + { + return new CountingParser((new ParserFactory())->createForVersion( + \PhpParser\PhpVersion::fromString('8.4'), + )); + } + + private function removeDirectory(string $directory): void + { + if (!is_dir($directory)) { + return; + } + $iterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($directory, \FilesystemIterator::SKIP_DOTS), + \RecursiveIteratorIterator::CHILD_FIRST, + ); + foreach ($iterator as $entry) { + $entry->isDir() ? rmdir($entry->getPathname()) : unlink($entry->getPathname()); + } + rmdir($directory); + } +} diff --git a/src/Build/AstCache.php b/src/Build/AstCache.php new file mode 100644 index 00000000..637d9b0d --- /dev/null +++ b/src/Build/AstCache.php @@ -0,0 +1,180 @@ +cacheDirectory = rtrim($buildDirectory, '/\\') + . DIRECTORY_SEPARATOR . 'cache' + . DIRECTORY_SEPARATOR . 'ast'; + $this->parserVersion = $parserVersion ?? self::detectParserVersion(); + } + + /** @return list */ + public function load(string $file, string $source): array + { + $cacheFile = $this->getCacheFile($file); + $sourceHash = hash('sha256', $source); + $cached = $this->restore($file, $cacheFile, $source, $sourceHash); + if ($cached !== null) { + return $cached; + } + + $ast = $this->parser->parse($source); + if ($ast === null) { + throw new \LogicException('php-parser returned no AST for: ' . $file); + } + $ast = $this->requireStatementList($ast); + $this->store($cacheFile, $source, $sourceHash, $ast); + return $ast; + } + + /** @return list|null */ + private function restore(string $file, string $cacheFile, string $source, string $sourceHash): ?array + { + if (!is_file($cacheFile)) { + return null; + } + + clearstatcache(true, $file); + clearstatcache(true, $cacheFile); + $sourceMtime = filemtime($file); + $cacheMtime = filemtime($cacheFile); + if ($sourceMtime === false || $cacheMtime === false || $sourceMtime > $cacheMtime) { + return null; + } + + $contents = file_get_contents($cacheFile); + if (!is_string($contents) || $contents === '') { + return null; + } + + try { + $payload = @unserialize($contents, ['allowed_classes' => true]); + } catch (\Throwable) { + return null; + } + if (!is_array($payload) + || ($payload['schema'] ?? null) !== self::SCHEMA_VERSION + || ($payload['phpVersion'] ?? null) !== $this->phpVersion + || ($payload['parserVersion'] ?? null) !== $this->parserVersion + || ($payload['sourceSize'] ?? null) !== strlen($source) + || ($payload['sourceHash'] ?? null) !== $sourceHash + || !isset($payload['ast']) + || !is_array($payload['ast']) + || !array_is_list($payload['ast']) + ) { + return null; + } + + try { + return $this->requireStatementList($payload['ast']); + } catch (\UnexpectedValueException) { + return null; + } + } + + /** @param list $ast */ + private function store(string $cacheFile, string $source, string $sourceHash, array $ast): void + { + try { + if (!is_dir($this->cacheDirectory) + && !mkdir($this->cacheDirectory, 0777, true) + && !is_dir($this->cacheDirectory) + ) { + return; + } + + $contents = serialize([ + 'schema' => self::SCHEMA_VERSION, + 'phpVersion' => $this->phpVersion, + 'parserVersion' => $this->parserVersion, + 'sourceSize' => strlen($source), + 'sourceHash' => $sourceHash, + 'ast' => $ast, + ]); + $temporary = tempnam($this->cacheDirectory, '.ast-'); + if ($temporary === false) { + return; + } + try { + if (file_put_contents($temporary, $contents, LOCK_EX) === false) { + return; + } + if (!@rename($temporary, $cacheFile)) { + // Windows cannot atomically replace an existing file. A + // concurrent reader may safely treat the short gap as a + // cache miss and rebuild the disposable snapshot. + @unlink($cacheFile); + @rename($temporary, $cacheFile); + } + } finally { + if (is_file($temporary)) { + @unlink($temporary); + } + } + } catch (\Throwable) { + // AST caching is an optimization. Serialization or filesystem + // failures must never prevent an otherwise valid compilation. + } + } + + private function getCacheFile(string $file): string + { + $normalized = str_replace('\\', '/', $file); + $cacheIdentity = implode("\0", [ + (string) self::SCHEMA_VERSION, + $this->phpVersion, + $this->parserVersion, + $normalized, + ]); + return $this->cacheDirectory . DIRECTORY_SEPARATOR + . hash('sha256', $cacheIdentity) . '.ast'; + } + + /** @param array $ast @return list */ + private function requireStatementList(array $ast): array + { + foreach ($ast as $node) { + if (!$node instanceof Node\Stmt) { + throw new \UnexpectedValueException('AST cache contains a non-statement root node'); + } + } + return $ast; + } + + private static function detectParserVersion(): string + { + if (class_exists(InstalledVersions::class) + && InstalledVersions::isInstalled('nikic/php-parser') + ) { + return InstalledVersions::getPrettyVersion('nikic/php-parser') + ?? InstalledVersions::getVersion('nikic/php-parser') + ?? InstalledVersions::getReference('nikic/php-parser') + ?? 'unknown'; + } + return 'unknown'; + } +} diff --git a/src/CompilerBase.php b/src/CompilerBase.php index d263090a..ab40f478 100644 --- a/src/CompilerBase.php +++ b/src/CompilerBase.php @@ -9,6 +9,7 @@ namespace TypePhp; use TypePhp\Analysis\CompilationStatistics; +use TypePhp\Build\AstCache; use TypePhp\Build\PhpxLocator; use League\CLImate\CLImate; @@ -622,6 +623,7 @@ class CompilerBase implements PropertyAccessContext protected bool $noProgress = false; protected bool $forTest = false; protected Parser $parser; + protected ?AstCache $astCache = null; protected string $phpVersion = self::DEFAULT_PHP_VERSION; protected PrettyPrinter $printer; protected bool $isPhpZts = false; // Whether the PHP build is thread-safe (ZTS) @@ -709,6 +711,7 @@ class CompilerBase implements PropertyAccessContext // php-parser's emulative lexer permits the compiler runtime to be // older than the selected PHP language version. $this->parser = (new ParserFactory())->createForVersion(PhpVersion::fromString($this->phpVersion)); + $this->astCache = null; } public function getPhpVersion(): string @@ -5200,6 +5203,12 @@ class CompilerBase implements PropertyAccessContext throw new \RuntimeException('Failed to resolve build path: ' . $string); } $this->buildDir = $resolved; + $this->astCache = null; + } + + protected function getAstCache(): AstCache + { + return $this->astCache ??= new AstCache($this->parser, $this->buildDir, $this->phpVersion); } protected function isStubFile(string $file): bool diff --git a/src/Preprocessor.php b/src/Preprocessor.php index 98e15fc4..e36d6036 100644 --- a/src/Preprocessor.php +++ b/src/Preprocessor.php @@ -161,7 +161,7 @@ class Preprocessor extends CompilerBase continue; } try { - $ast = $this->parser->parse($source); + $ast = $this->getAstCache()->load($file, $source); $traverser = new NodeTraverser(); $traverser->addVisitor(new NameResolver(null, ['replaceNodes' => false])); $ast = $this->requireStatementList($traverser->traverse($ast)); @@ -246,9 +246,9 @@ class Preprocessor extends CompilerBase $this->nativeGlobalTypeResolver = $resolver; $discovery = new NativeGlobalDiscovery($resolver, $functionReturns); - foreach ($candidateSources as $source) { + foreach ($candidateSources as $file => $source) { try { - $ast = $this->parser->parse($source); + $ast = $this->getAstCache()->load($file, $source); $traverser = new NodeTraverser(); $traverser->addVisitor(new NameResolver(null, ['replaceNodes' => false])); $ast = $this->requireStatementList($traverser->traverse($ast)); @@ -395,7 +395,7 @@ class Preprocessor extends CompilerBase $this->climate->info('prepare: ' . $this->getRelativePath($this->file)); try { - $ast = $this->parser->parse($phpCode); + $ast = $this->getAstCache()->load($this->file, $phpCode); } catch (\PhpParser\Error $e) { $this->climate->red("Fatal error: {$e->getMessage()} in {$this->file}"); throw new SyntaxError($e->getMessage(), $e->getCode()); diff --git a/src/Translator.php b/src/Translator.php index 5be496df..817d1798 100644 --- a/src/Translator.php +++ b/src/Translator.php @@ -3506,7 +3506,7 @@ CODE; { $this->climate->info('convert: ' . $this->getRelativePath($this->file)); - $ast = $this->parser->parse($phpCode); + $ast = $this->getAstCache()->load($this->file, $phpCode); $traverser = new NodeTraverser(); $traverser->addVisitor(new NameResolver(null, ['replaceNodes' => false])); if ($this->isNanoPolicyMode()) {