From 4216f21b02ffd231e1e3ad578155eda1dbff6505 Mon Sep 17 00:00:00 2001 From: tianfenghan Date: Tue, 22 Sep 2026 11:17:19 +0800 Subject: [PATCH] refactor(build): clarify embedded opcode pipeline --- .../src/Build/EmbeddedOpcodeGeneratorTest.php | 144 +++++++ src/Build/EmbeddedOpcodeBatch.php | 88 +++++ src/Build/EmbeddedOpcodeGenerator.php | 362 +++++++++++++++++ src/Build/SourcePipelineTrait.php | 365 +++++------------- 4 files changed, 688 insertions(+), 271 deletions(-) create mode 100644 phpunit/src/Build/EmbeddedOpcodeGeneratorTest.php create mode 100644 src/Build/EmbeddedOpcodeBatch.php create mode 100644 src/Build/EmbeddedOpcodeGenerator.php diff --git a/phpunit/src/Build/EmbeddedOpcodeGeneratorTest.php b/phpunit/src/Build/EmbeddedOpcodeGeneratorTest.php new file mode 100644 index 00000000..0e1f6662 --- /dev/null +++ b/phpunit/src/Build/EmbeddedOpcodeGeneratorTest.php @@ -0,0 +1,144 @@ +directory = sys_get_temp_dir() . '/typephp-opcode-generator-' . bin2hex(random_bytes(6)); + $this->buildDirectory = $this->directory . '/build'; + $vendor = $this->directory . '/vendor'; + mkdir($vendor . '/package', 0777, true); + file_put_contents($vendor . '/autoload.php', 'vendorFile = $vendor . '/package/library.php'; + $this->anonymousFile = $this->directory . '/anonymous.php'; + $this->ordinaryFile = $this->directory . '/ordinary.php'; + file_put_contents($this->vendorFile, 'anonymousFile, 'ordinaryFile, 'directory, \FilesystemIterator::SKIP_DOTS), + \RecursiveIteratorIterator::CHILD_FIRST, + ); + foreach ($iterator as $entry) { + $entry->isDir() ? rmdir($entry->getPathname()) : unlink($entry->getPathname()); + } + rmdir($this->directory); + } + + public function testPrepareReusesOnlyVendorAndAnonymousCaches(): void + { + $files = [$this->vendorFile, $this->anonymousFile, $this->ordinaryFile]; + $generator = $this->generator(); + $first = $generator->prepare($files, [$this->anonymousFile => '@typephp:anon:test']); + + self::assertSame($files, $first->pendingFiles()); + self::assertTrue($first->isVendorFile($this->vendorFile)); + self::assertTrue($first->isAnonymousFile($this->anonymousFile)); + self::assertNotSame( + $first->cacheDirectory($this->vendorFile), + $first->cacheDirectory($this->anonymousFile), + ); + + $vendorBlob = $this->writeBlob($first->cacheDirectory($this->vendorFile), $this->vendorFile, 'vendor'); + $anonymousBlob = $this->writeBlob( + $first->cacheDirectory($this->anonymousFile), + $this->anonymousFile, + 'anonymous', + ); + $ordinaryBlob = $this->writeBlob( + $first->cacheDirectory($this->ordinaryFile), + $this->ordinaryFile, + 'ordinary', + ); + + $second = $generator->prepare($files, [$this->anonymousFile => '@typephp:anon:test']); + + self::assertSame([ + $this->vendorFile => $vendorBlob, + $this->anonymousFile => $anonymousBlob, + ], $second->cachedBlobs()); + self::assertSame([$this->ordinaryFile], $second->pendingFiles()); + self::assertSame(1, $second->vendorHits()); + self::assertSame(1, $second->anonymousHits()); + self::assertFileDoesNotExist($ordinaryBlob); + self::assertFileExists($this->buildDirectory . '/cache/opcache/compile-embedded-opcodes.php'); + } + + public function testForceClearsPersistentOpcodeCaches(): void + { + $files = [$this->vendorFile, $this->anonymousFile]; + $anonymousKeys = [$this->anonymousFile => '@typephp:anon:test']; + $first = $this->generator()->prepare($files, $anonymousKeys); + $this->writeBlob($first->cacheDirectory($this->vendorFile), $this->vendorFile, 'vendor'); + $this->writeBlob($first->cacheDirectory($this->anonymousFile), $this->anonymousFile, 'anonymous'); + + $forced = $this->generator(true)->prepare($files, $anonymousKeys); + + self::assertSame([], $forced->cachedBlobs()); + self::assertSame($files, $forced->pendingFiles()); + } + + public function testVendorSkipMarkerAvoidsRepeatedCompilation(): void + { + $generator = $this->generator(); + $first = $generator->prepare([$this->vendorFile], []); + $cacheDirectory = $first->cacheDirectory($this->vendorFile); + mkdir($cacheDirectory, 0777, true); + file_put_contents( + $cacheDirectory . '/skipped-' . hash('sha256', $this->vendorFile) . '.txt', + $this->vendorFile, + ); + + $second = $generator->prepare([$this->vendorFile], []); + + self::assertSame([], $second->pendingFiles()); + self::assertSame(1, $second->skippedVendorCount()); + self::assertSame(0, $second->vendorHits()); + } + + private function generator(bool $force = false): EmbeddedOpcodeGenerator + { + return new EmbeddedOpcodeGenerator( + $this->buildDirectory, + 'app', + PHP_BINARY, + [], + PHP_VERSION, + 'build-signature', + $force, + ); + } + + private function writeBlob(string $cacheDirectory, string $source, string $contents): string + { + $blob = $cacheDirectory . '/system' . $source . '.bin'; + mkdir(dirname($blob), 0777, true); + file_put_contents($blob, $contents); + return $blob; + } +} diff --git a/src/Build/EmbeddedOpcodeBatch.php b/src/Build/EmbeddedOpcodeBatch.php new file mode 100644 index 00000000..ba7ba0ae --- /dev/null +++ b/src/Build/EmbeddedOpcodeBatch.php @@ -0,0 +1,88 @@ + $cachedBlobs + * @param list $pendingFiles + * @param array $cacheDirectories + * @param array $vendorFiles + * @param array $anonymousFiles + * @param array $skippedVendorFiles + */ + public function __construct( + private array $cachedBlobs, + private array $pendingFiles, + private array $cacheDirectories, + private array $vendorFiles, + private array $anonymousFiles, + private array $skippedVendorFiles, + ) { + } + + /** @return array */ + public function cachedBlobs(): array + { + return $this->cachedBlobs; + } + + /** @return list */ + public function pendingFiles(): array + { + return $this->pendingFiles; + } + + public function pendingCount(): int + { + return count($this->pendingFiles); + } + + public function cacheDirectory(string $file): string + { + return $this->cacheDirectories[$file] + ?? throw new \LogicException("Missing OPcache directory for: {$file}"); + } + + public function isVendorFile(string $file): bool + { + return isset($this->vendorFiles[$file]); + } + + public function isAnonymousFile(string $file): bool + { + return isset($this->anonymousFiles[$file]); + } + + public function vendorHits(): int + { + return count(array_intersect_key($this->cachedBlobs, $this->vendorFiles)); + } + + public function anonymousHits(): int + { + return count(array_intersect_key($this->cachedBlobs, $this->anonymousFiles)); + } + + public function vendorCount(): int + { + return count($this->vendorFiles); + } + + public function anonymousCount(): int + { + return count($this->anonymousFiles); + } + + public function skippedVendorCount(): int + { + return count($this->skippedVendorFiles); + } +} diff --git a/src/Build/EmbeddedOpcodeGenerator.php b/src/Build/EmbeddedOpcodeGenerator.php new file mode 100644 index 00000000..db971384 --- /dev/null +++ b/src/Build/EmbeddedOpcodeGenerator.php @@ -0,0 +1,362 @@ + $extensionArguments */ + public function __construct( + private readonly string $buildDirectory, + private readonly string $targetName, + private readonly string $php, + private readonly array $extensionArguments, + private readonly string $phpVersion, + private readonly string $buildSignature, + private readonly bool $force, + ) { + } + + /** + * Resolve persistent cache hits and produce an immutable compilation batch. + * + * @param list $files + * @param array $anonymousKeys + */ + public function prepare(array $files, array $anonymousKeys): EmbeddedOpcodeBatch + { + $cacheDirectories = []; + $vendorGroups = []; + $vendorFiles = []; + $anonymousFiles = []; + $otherFiles = []; + + foreach ($files as $file) { + if (isset($anonymousKeys[$file])) { + $cacheDirectory = $this->anonymousCacheDirectory($file); + if ($this->force) { + $this->clearDirectory($cacheDirectory); + } + $cacheDirectories[$file] = $cacheDirectory; + $anonymousFiles[$file] = true; + continue; + } + + $vendorRoot = $this->vendorRoot($file); + if ($vendorRoot === null) { + $otherFiles[] = $file; + continue; + } + $vendorGroups[$vendorRoot][] = $file; + $vendorFiles[$file] = true; + } + + foreach ($vendorGroups as $vendorRoot => $vendorGroup) { + $cacheDirectory = $this->vendorCacheDirectory($vendorRoot); + if ($this->force) { + $this->clearDirectory($cacheDirectory); + } + foreach ($vendorGroup as $file) { + $cacheDirectories[$file] = $cacheDirectory; + } + } + + if ($otherFiles !== []) { + // Ordinary directories have no reliable invalidation boundary. + // Recreate their shared staging directory for every build. + $cacheDirectory = $this->buildDirectory . '/cache/opcache/nonvendor-' + . substr(hash('sha256', $this->targetName), 0, 20); + $this->clearDirectory($cacheDirectory); + foreach ($otherFiles as $file) { + $cacheDirectories[$file] = $cacheDirectory; + } + } + + $this->writeDriver(); + return $this->resolve($files, $cacheDirectories, $vendorFiles, $anonymousFiles); + } + + /** + * Compile only the cache misses from a prepared batch. + * + * @param callable(string): void|null $onSkipped + * @param callable(int, int, string): void|null $onProgress + * @return array source file => OPcache blob + */ + public function compile( + EmbeddedOpcodeBatch $batch, + ?callable $onSkipped = null, + ?callable $onProgress = null, + ): array { + $blobs = $batch->cachedBlobs(); + $total = $batch->pendingCount(); + foreach ($batch->pendingFiles() as $index => $file) { + $cacheDirectory = $batch->cacheDirectory($file); + $this->ensureDirectory($cacheDirectory); + [$status, $stdout, $stderr] = $this->compileFile($file, $cacheDirectory); + + if ($status !== 0) { + if ($batch->isAnonymousFile($file) + || !str_contains($stderr, "OPcache could not compile: {$file}")) { + throw new \RuntimeException("Cannot compile embedded opcode {$file}:\n{$stderr}"); + } + if ($batch->isVendorFile($file)) { + AtomicFile::write($this->skippedMarker($cacheDirectory, $file), $file . PHP_EOL, '.skip-'); + } + if ($onSkipped !== null) { + $onSkipped($file); + } + } else { + // Compiled files may emit PHP deprecation notices after the + // version line printed by the driver. + if (strtok($stdout, "\r\n") !== $this->phpVersion) { + throw new \RuntimeException("Build PHP version changed while compiling {$file}"); + } + $blob = $this->findBlob($cacheDirectory, $file); + if ($blob === null) { + throw new \RuntimeException("Missing OPcache blob for {$file}"); + } + $blobs[$file] = $blob; + } + + if ($onProgress !== null) { + $onProgress($index + 1, $total, $file); + } + } + + uksort($blobs, static fn (string $left, string $right): int => strcmp($left, $right)); + return $blobs; + } + + /** + * @param list $files + * @param array $cacheDirectories + * @param array $vendorFiles + * @param array $anonymousFiles + */ + private function resolve( + array $files, + array $cacheDirectories, + array $vendorFiles, + array $anonymousFiles, + ): EmbeddedOpcodeBatch { + $cachedBlobs = []; + $pendingFiles = []; + $skippedVendorFiles = []; + foreach ($files as $file) { + $cacheDirectory = $cacheDirectories[$file]; + if (isset($vendorFiles[$file]) + && is_file($this->skippedMarker($cacheDirectory, $file))) { + $skippedVendorFiles[$file] = true; + continue; + } + if ((isset($vendorFiles[$file]) || isset($anonymousFiles[$file])) + && ($blob = $this->findBlob($cacheDirectory, $file)) !== null) { + $cachedBlobs[$file] = $blob; + continue; + } + $pendingFiles[] = $file; + } + + return new EmbeddedOpcodeBatch( + $cachedBlobs, + $pendingFiles, + $cacheDirectories, + $vendorFiles, + $anonymousFiles, + $skippedVendorFiles, + ); + } + + private function anonymousCacheDirectory(string $file): string + { + $fileHash = hash_file('sha256', $file); + if (!is_string($fileHash)) { + throw new \RuntimeException("Cannot hash anonymous class source: {$file}"); + } + $key = hash('sha256', implode("\n", [ + 'anonymous-opcodes-v1', + $file, + $fileHash, + $this->buildSignature, + ])); + return $this->buildDirectory . '/cache/opcache/anonymous-' . substr($key, 0, 20); + } + + private function vendorCacheDirectory(string $vendorRoot): string + { + clearstatcache(true, $vendorRoot); + $mtime = filemtime($vendorRoot); + if ($mtime === false) { + throw new \RuntimeException("Cannot read vendor directory mtime: {$vendorRoot}"); + } + $key = hash('sha256', implode("\n", [ + 'vendor-directory-mtime-opcodes-v2', + $vendorRoot, + (string) $mtime, + $this->buildSignature, + ])); + return $this->buildDirectory . '/cache/opcache/vendor-' . substr($key, 0, 20); + } + + /** Only files under a vendor directory with autoload.php use the persistent cache. */ + private function vendorRoot(string $file): ?string + { + for ($directory = dirname($file); $directory !== dirname($directory); $directory = dirname($directory)) { + if (basename($directory) === 'vendor' && is_file($directory . '/autoload.php')) { + return $directory; + } + } + return null; + } + + /** @return array{int, string, string} */ + private function compileFile(string $file, string $cacheDirectory): array + { + $command = [ + $this->php, + '-n', + ...$this->extensionArguments, + '-d', + 'opcache.enable_cli=1', + '-d', + 'opcache.file_cache_only=1', + '-d', + 'opcache.file_update_protection=0', + '-d', + 'opcache.file_cache=' . $cacheDirectory, + $this->driverPath(), + $file, + ]; + $process = proc_open($command, [ + 0 => ['pipe', 'r'], + 1 => ['pipe', 'w'], + 2 => ['pipe', 'w'], + ], $pipes); + if (!is_resource($process)) { + throw new \RuntimeException('Cannot start the PHP OPcache build process'); + } + fclose($pipes[0]); + $stdout = stream_get_contents($pipes[1]); + $stderr = stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + return [proc_close($process), (string) $stdout, (string) $stderr]; + } + + private function writeDriver(): void + { + $path = $this->driverPath(); + if (is_file($path) && file_get_contents($path) === self::DRIVER_SOURCE) { + return; + } + AtomicFile::write($path, self::DRIVER_SOURCE, '.driver-'); + } + + private function driverPath(): string + { + return $this->buildDirectory . '/cache/opcache/compile-embedded-opcodes.php'; + } + + private function skippedMarker(string $cacheDirectory, string $file): string + { + return $cacheDirectory . '/skipped-' . hash('sha256', $file) . '.txt'; + } + + private function findBlob(string $cacheDirectory, string $file): ?string + { + $matches = []; + foreach (glob($cacheDirectory . '/*', GLOB_ONLYDIR) ?: [] as $systemDirectory) { + $candidate = $systemDirectory . $file . '.bin'; + if (is_file($candidate) && filesize($candidate) > 0) { + $matches[] = $candidate; + } + } + if (count($matches) > 1) { + throw new \RuntimeException("Ambiguous OPcache blob for {$file}"); + } + if ($matches !== []) { + return $matches[0]; + } + if (DIRECTORY_SEPARATOR !== '\\' || !is_dir($cacheDirectory)) { + return null; + } + + // OPcache omits the drive colon from Windows cache paths (C:\foo + // becomes C\foo) after two system-specific cache directories. + $cacheSuffix = str_replace('/', '\\', preg_replace('/^([A-Za-z]):/', '$1', $file)) . '.bin'; + foreach (glob($cacheDirectory . '/*', GLOB_ONLYDIR) ?: [] as $systemDirectory) { + foreach (glob($systemDirectory . '/*', GLOB_ONLYDIR) ?: [] as $subDirectory) { + $candidate = $subDirectory . DIRECTORY_SEPARATOR . $cacheSuffix; + if (is_file($candidate) && filesize($candidate) > 0) { + $matches[] = $candidate; + } + } + } + if (count($matches) > 1) { + throw new \RuntimeException("Ambiguous OPcache blob for {$file}"); + } + if ($matches !== []) { + return $matches[0]; + } + + // Retain compatibility with PHP builds that use another number of + // system-specific directories. + $iterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($cacheDirectory, \FilesystemIterator::SKIP_DOTS), + ); + foreach ($iterator as $cached) { + if ($cached->isFile() && $cached->getSize() > 0 + && str_ends_with($cached->getPathname(), $cacheSuffix)) { + if ($matches !== []) { + throw new \RuntimeException("Ambiguous OPcache blob for {$file}"); + } + $matches[] = $cached->getPathname(); + } + } + return $matches[0] ?? null; + } + + private function clearDirectory(string $directory): void + { + if (!is_dir($directory)) { + return; + } + $iterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($directory, \FilesystemIterator::SKIP_DOTS), + \RecursiveIteratorIterator::CHILD_FIRST, + ); + foreach ($iterator as $entry) { + $path = $entry->getPathname(); + if ($entry->isDir() ? !rmdir($path) : !unlink($path)) { + throw new \RuntimeException("Cannot clear opcode build directory: {$path}"); + } + } + if (!rmdir($directory)) { + throw new \RuntimeException("Cannot clear opcode build directory: {$directory}"); + } + } + + private function ensureDirectory(string $directory): void + { + if (!is_dir($directory) && !mkdir($directory, 0777, true) && !is_dir($directory)) { + throw new \RuntimeException("Cannot create OPcache build directory: {$directory}"); + } + } +} diff --git a/src/Build/SourcePipelineTrait.php b/src/Build/SourcePipelineTrait.php index 2957a9c1..bb9afb68 100644 --- a/src/Build/SourcePipelineTrait.php +++ b/src/Build/SourcePipelineTrait.php @@ -301,98 +301,27 @@ trait SourcePipelineTrait } } - /** Only files under a vendor directory with autoload.php use the mtime cache. */ - private function vendorRootForOpcodeFile(string $file): ?string + /** Embed OPcache's file-cache bytes for scripts left to ZendVM. */ + private function genEmbeddedOpcodeTable(): array { - for ($directory = dirname($file); $directory !== dirname($directory); $directory = dirname($directory)) { - if (basename($directory) === 'vendor' && is_file($directory . '/autoload.php')) { - return $directory; - } - } - return null; - } + $output = $this->getBuildDir() . '/embedded-opcodes-' . $this->targetName . '.cc'; + $this->migrateEmbeddedOpcodeCaches(); + $files = array_values(array_unique($this->embeddedOpcodeFiles)); + sort($files, SORT_STRING); - private function findOpcodeBlob(string $cacheDir, string $file): ?string - { - $matches = []; - foreach (glob($cacheDir . '/*', GLOB_ONLYDIR) ?: [] as $systemDirectory) { - $candidate = $systemDirectory . $file . '.bin'; - if (is_file($candidate) && filesize($candidate) > 0) { - $matches[] = $candidate; - } - } - if (count($matches) > 1) { - throw new \RuntimeException("Ambiguous OPcache blob for {$file}"); - } - if ($matches !== []) { - return $matches[0]; - } - // OPcache uses a different file-cache path layout on Windows. - if (DIRECTORY_SEPARATOR === '\\' && is_dir($cacheDir)) { - // The drive colon is omitted from OPcache's cache path (C:\foo - // becomes C\foo), after two system-specific cache directories. - $windowsCacheSuffix = str_replace('/', '\\', preg_replace('/^([A-Za-z]):/', '$1', $file)) . '.bin'; - foreach (glob($cacheDir . '/*', GLOB_ONLYDIR) ?: [] as $systemDirectory) { - foreach (glob($systemDirectory . '/*', GLOB_ONLYDIR) ?: [] as $subDirectory) { - $candidate = $subDirectory . DIRECTORY_SEPARATOR . $windowsCacheSuffix; - if (is_file($candidate) && filesize($candidate) > 0) { - $matches[] = $candidate; - } - } - } - if (count($matches) > 1) { - throw new \RuntimeException("Ambiguous OPcache blob for {$file}"); - } - if ($matches !== []) { - return $matches[0]; - } - // Keep supporting other OPcache layouts without assuming the same - // number of system directories in every PHP build. - $iterator = new \RecursiveIteratorIterator( - new \RecursiveDirectoryIterator($cacheDir, \FilesystemIterator::SKIP_DOTS), - ); - foreach ($iterator as $cached) { - if ($cached->isFile() && $cached->getSize() > 0 - && str_ends_with($cached->getPathname(), $windowsCacheSuffix)) { - if ($matches !== []) { - throw new \RuntimeException("Ambiguous OPcache blob for {$file}"); - } - $matches[] = $cached->getPathname(); - } - } + $blobs = []; + $phpVersion = PHP_VERSION; + if ($files !== []) { + ['blobs' => $blobs, 'phpVersion' => $phpVersion] = $this->generateEmbeddedOpcodeBlobs($files); } - return $matches[0] ?? null; - } - private function getSkippedOpcodeMarker(string $cacheDir, string $file): string - { - return $cacheDir . '/skipped-' . hash('sha256', $file) . '.txt'; - } - - private function clearOpcodeCacheDirectory(string $directory): void - { - if (!is_dir($directory)) { - return; - } - $iterator = new \RecursiveIteratorIterator( - new \RecursiveDirectoryIterator($directory, \FilesystemIterator::SKIP_DOTS), - \RecursiveIteratorIterator::CHILD_FIRST, - ); - foreach ($iterator as $entry) { - $path = $entry->getPathname(); - if ($entry->isDir() ? !rmdir($path) : !unlink($path)) { - throw new \RuntimeException("Cannot clear opcode build directory: {$path}"); - } - } - if (!rmdir($directory)) { - throw new \RuntimeException("Cannot clear opcode build directory: {$directory}"); - } + $sources = $this->emitEmbeddedArchiveSources($output, $files, $blobs, $phpVersion); + $this->removeLegacyAnonymousArtifacts(); + return $sources; } - /** Embed OPcache's file-cache bytes for scripts left to ZendVM. */ - private function genEmbeddedOpcodeTable(): array + private function migrateEmbeddedOpcodeCaches(): void { - $output = $this->getBuildDir() . '/embedded-opcodes-' . $this->targetName . '.cc'; foreach (glob($this->getBuildDir() . '/opcache-*') ?: [] as $legacyCache) { if (is_dir($legacyCache)) { $this->moveLegacyBuildCache( @@ -401,195 +330,85 @@ trait SourcePipelineTrait ); } } - $files = array_values(array_unique($this->embeddedOpcodeFiles)); - sort($files, SORT_STRING); - $blobs = []; - $phpVersion = PHP_VERSION; + $this->moveLegacyBuildCache( + $this->getBuildDir() . '/compile-embedded-opcodes.php', + $this->getBuildDir() . '/cache/opcache/compile-embedded-opcodes.php', + ); + } - if ($files !== []) { - if ($this->isIosTarget() || $this->isAndroidTarget() || $this->isWasiTarget()) { - throw new \RuntimeException('Embedded opcodes require a native PHP build host matching the target'); - } - $this->output('Generating embedded opcodes for ' . count($files) . ' PHP files', 'lightBlue'); - $php = $this->getOpcodeBuildPhpCli(); - $extensionArgs = $this->getOpcodeBuildExtensionArgs(); - $phpVersion = $this->opcodeBuildPhpVersion; - $otherFiles = []; - $vendorRoots = []; - $vendorFiles = []; - $anonymousFiles = []; - $fileCacheDirs = []; - foreach ($files as $file) { - if (isset($this->anonymousOpcodeKeys[$file])) { - // These files are emitted by this compiler, so their exact - // contents provide a safe cache key independent of any - // user-supplied directory timestamp. - $key = hash('sha256', implode("\n", [ - 'anonymous-opcodes-v1', $file, - hash_file('sha256', $file), $this->opcodeBuildSignature, - ])); - $cacheDir = $this->getBuildDir() . '/cache/opcache/anonymous-' . substr($key, 0, 20); - if ($this->climate->arguments->defined('force')) { - $this->clearOpcodeCacheDirectory($cacheDir); - } - $fileCacheDirs[$file] = $cacheDir; - $anonymousFiles[$file] = true; - continue; - } - $vendorRoot = $this->vendorRootForOpcodeFile($file); - if ($vendorRoot === null) { - $otherFiles[] = $file; - continue; - } - $vendorRoots[$vendorRoot][] = $file; - $vendorFiles[$file] = true; - } - foreach ($vendorRoots as $vendorRoot => $filesInVendor) { - clearstatcache(true, $vendorRoot); - $mtime = filemtime($vendorRoot); - if ($mtime === false) { - throw new \RuntimeException("Cannot read vendor directory mtime: {$vendorRoot}"); - } - $key = hash('sha256', implode("\n", [ - 'vendor-directory-mtime-opcodes-v2', $vendorRoot, - (string) $mtime, $this->opcodeBuildSignature, - ])); - $cacheDir = $this->getBuildDir() . '/cache/opcache/vendor-' . substr($key, 0, 20); - if ($this->climate->arguments->defined('force')) { - $this->clearOpcodeCacheDirectory($cacheDir); - } - foreach ($filesInVendor as $file) { - $fileCacheDirs[$file] = $cacheDir; - } - } - if ($otherFiles !== []) { - // Recreate this staging directory on every build so OPcache - // cannot reuse bytecode for non-vendor or anonymous sources. - $cacheDir = $this->getBuildDir() . '/cache/opcache/nonvendor-' - . substr(hash('sha256', $this->targetName), 0, 20); - $this->clearOpcodeCacheDirectory($cacheDir); - foreach ($otherFiles as $file) { - $fileCacheDirs[$file] = $cacheDir; - } - } - $driver = $this->getBuildDir() . '/cache/opcache/compile-embedded-opcodes.php'; - $this->moveLegacyBuildCache( - $this->getBuildDir() . '/compile-embedded-opcodes.php', - $driver, - ); - $this->writeFile($driver, <<<'PHP' - $files @return array{blobs: array, phpVersion: string} */ + private function generateEmbeddedOpcodeBlobs(array $files): array + { + if ($this->isIosTarget() || $this->isAndroidTarget() || $this->isWasiTarget()) { + throw new \RuntimeException('Embedded opcodes require a native PHP build host matching the target'); + } + $this->output('Generating embedded opcodes for ' . count($files) . ' PHP files', 'lightBlue'); + + $generator = new EmbeddedOpcodeGenerator( + $this->getBuildDir(), + $this->targetName, + $this->getOpcodeBuildPhpCli(), + $this->getOpcodeBuildExtensionArgs(), + $this->opcodeBuildPhpVersion, + $this->opcodeBuildSignature, + $this->climate->arguments->defined('force'), + ); + $batch = $generator->prepare($files, $this->anonymousOpcodeKeys); + $this->reportEmbeddedOpcodeCache($batch); + + $progress = $this->startOpcodeProgress('Opcodes', $batch->pendingCount()); + $blobs = $generator->compile( + $batch, + fn(string $file) => $this->climate->warning( + 'Skipping non-executable embedded PHP file: ' . $file, + ), + fn(int $completed, int $total, string $file) => $this->updateOpcodeProgress( + $progress, + 'Opcodes', + $completed, + $total, + $this->noProgress ? $file : null, + ), + ); + if ($progress !== null) { + echo PHP_EOL; + } + return ['blobs' => $blobs, 'phpVersion' => $this->opcodeBuildPhpVersion]; } -} -PHP + + private function reportEmbeddedOpcodeCache(EmbeddedOpcodeBatch $batch): void + { + if ($batch->vendorCount() !== 0) { + $vendorHits = $batch->vendorHits(); + $vendorSkips = $batch->skippedVendorCount(); + $this->output( + 'Vendor opcode cache: ' . $vendorHits . ' reused, ' + . ($batch->vendorCount() - $vendorHits - $vendorSkips) . ' to generate' + . ($vendorSkips === 0 ? '' : ', ' . $vendorSkips . ' skipped'), + 'lightBlue', ); - $command = [ - $php, '-n', ...$extensionArgs, - '-d', 'opcache.enable_cli=1', - '-d', 'opcache.file_cache_only=1', - '-d', 'opcache.file_update_protection=0', - ]; - // OPcache installs declarations in the compiling CLI process. - // Isolate files so unrelated scripts with the same global - // function names do not conflict while creating their bytecode. - $pending = []; - $vendorHits = 0; - $anonymousHits = 0; - $vendorSkips = 0; - foreach ($files as $file) { - $cacheDir = $fileCacheDirs[$file]; - if (isset($vendorFiles[$file]) - && is_file($this->getSkippedOpcodeMarker($cacheDir, $file))) { - ++$vendorSkips; - continue; - } - if ((isset($vendorFiles[$file]) || isset($anonymousFiles[$file])) - && ($blob = $this->findOpcodeBlob($cacheDir, $file)) !== null) { - $blobs[$file] = $blob; - if (isset($anonymousFiles[$file])) { - ++$anonymousHits; - } else { - ++$vendorHits; - } - } else { - $pending[] = $file; - } - } - if ($vendorRoots !== []) { - $vendorCount = array_sum(array_map('count', $vendorRoots)); - $this->output( - 'Vendor opcode cache: ' . $vendorHits . ' reused, ' - . ($vendorCount - $vendorHits - $vendorSkips) . ' to generate' - . ($vendorSkips === 0 ? '' : ', ' . $vendorSkips . ' skipped'), - 'lightBlue', - ); - } - if ($anonymousFiles !== []) { - $this->output( - 'Anonymous opcode cache: ' . $anonymousHits . ' reused, ' - . (count($anonymousFiles) - $anonymousHits) . ' to generate', - 'lightBlue', - ); - } - $progress = $this->startOpcodeProgress('Opcodes', count($pending)); - $completed = 0; - foreach ($pending as $file) { - $cacheDir = $fileCacheDirs[$file]; - if (!is_dir($cacheDir) && !mkdir($cacheDir, 0777, true) && !is_dir($cacheDir)) { - throw new \RuntimeException("Cannot create OPcache build directory: {$cacheDir}"); - } - $process = proc_open([...$command, '-d', 'opcache.file_cache=' . $cacheDir, $driver, $file], [ - 0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w'], - ], $pipes); - if (!is_resource($process)) { - throw new \RuntimeException('Cannot start the PHP OPcache build process'); - } - fclose($pipes[0]); - $stdout = stream_get_contents($pipes[1]); - $stderr = stream_get_contents($pipes[2]); - fclose($pipes[1]); - fclose($pipes[2]); - if (proc_close($process) !== 0) { - if (isset($anonymousFiles[$file]) - || !str_contains((string) $stderr, "OPcache could not compile: {$file}")) { - throw new \RuntimeException("Cannot compile embedded opcode {$file}:\n{$stderr}"); - } - // Embedded directories can contain PHP-looking declaration - // files that are not scripts. Keep their raw bytes but do - // not treat them as executable bytecode. - $this->climate->warning('Skipping non-executable embedded PHP file: ' . $file); - if (isset($vendorFiles[$file])) { - $marker = $this->getSkippedOpcodeMarker($cacheDir, $file); - if (file_put_contents($marker, $file . PHP_EOL) === false) { - throw new \RuntimeException("Cannot cache skipped embedded PHP file: {$file}"); - } - } - $this->updateOpcodeProgress($progress, 'Opcodes', ++$completed, count($pending), $this->noProgress ? $file : null); - continue; - } - // Compiled files may emit PHP deprecation notices on stdout. - // The driver prints its version before compiling the file. - if (strtok((string) $stdout, "\r\n") !== $phpVersion) { - throw new \RuntimeException("Build PHP version changed while compiling {$file}"); - } - $blob = $this->findOpcodeBlob($cacheDir, $file); - if ($blob === null) { - throw new \RuntimeException("Missing OPcache blob for {$file}"); - } - $blobs[$file] = $blob; - $this->updateOpcodeProgress($progress, 'Opcodes', ++$completed, count($pending), $this->noProgress ? $file : null); - } - if ($progress !== null) { - echo PHP_EOL; - } - uksort($blobs, static fn(string $left, string $right): int => strcmp($left, $right)); } + if ($batch->anonymousCount() !== 0) { + $anonymousHits = $batch->anonymousHits(); + $this->output( + 'Anonymous opcode cache: ' . $anonymousHits . ' reused, ' + . ($batch->anonymousCount() - $anonymousHits) . ' to generate', + 'lightBlue', + ); + } + } + /** + * @param list $opcodeFiles + * @param array $blobs + * @return list + */ + private function emitEmbeddedArchiveSources( + string $output, + array $opcodeFiles, + array $blobs, + string $phpVersion, + ): array { $archive = $this->getBuildDir() . '/cache/embedded/embedded-files-' . $this->targetName . '.bin'; $this->moveLegacyBuildCache( $this->getBuildDir() . '/embedded-files-' . $this->targetName . '.bin', @@ -597,7 +416,7 @@ PHP ); $sources = [$output]; $embeddedArchive = EmbeddedArchive::empty($archive); - if ($this->embeddedFiles !== [] || $files !== []) { + if ($this->embeddedFiles !== [] || $opcodeFiles !== []) { $embeddedArchive = (new EmbeddedArchiveBuilder())->build( $archive, $this->embeddedFiles, @@ -629,13 +448,17 @@ PHP (new EmbeddedTableRenderer())->render($embeddedArchive, $phpVersion, $this->isWindows()), ); $this->generatedProjectSources[$output] = true; + return $sources; + } + + private function removeLegacyAnonymousArtifacts(): void + { foreach (glob($this->getBuildDir() . '/anonymous-*.json') ?: [] as $legacyManifest) { unlink($legacyManifest); } foreach (glob($this->getBuildDir() . '/anonymous-' . $this->targetName . '-*.php') ?: [] as $legacySource) { unlink($legacySource); } - return $sources; } public function getFiles(string $path): array