refactor(build): clarify embedded opcode pipeline

3 weeks ago · 4216f21b02
parent 5c3da4e406
commit 4216f21b02
4 changed files with 688 additions and 271 deletions
--- a/phpunit/src/Build/EmbeddedOpcodeGeneratorTest.php
+++ b/phpunit/src/Build/EmbeddedOpcodeGeneratorTest.php
@ -0,0 +1,144 @@
 <?php
 /**
 * This file is part of TypePHP(AOT).
 *
 * @link     https://www.swoole.com/aot/
 * @contact  service@swoole.com
 */
 namespace TypePhp\Tests\Build;
 use PHPUnit\Framework\TestCase;
 use TypePhp\Build\EmbeddedOpcodeGenerator;
 /**
 * @internal
 * @coversNothing
 */
 final class EmbeddedOpcodeGeneratorTest extends TestCase
 {
    private string $directory;
    private string $buildDirectory;
    private string $vendorFile;
    private string $anonymousFile;
    private string $ordinaryFile;
    protected function setUp(): void
    {
        $this->directory = sys_get_temp_dir() . '/typephp-opcode-generator-' . bin2hex(random_bytes(6));
        $this->buildDirectory = $this->directory . '/build';
        $vendor = $this->directory . '/vendor';
        mkdir($vendor . '/package', 0777, true);
        file_put_contents($vendor . '/autoload.php', '<?php');
        $this->vendorFile = $vendor . '/package/library.php';
        $this->anonymousFile = $this->directory . '/anonymous.php';
        $this->ordinaryFile = $this->directory . '/ordinary.php';
        file_put_contents($this->vendorFile, '<?php return 1;');
        file_put_contents($this->anonymousFile, '<?php return new class {};');
        file_put_contents($this->ordinaryFile, '<?php return 2;');
    }
    protected function tearDown(): void
    {
        $iterator = new \RecursiveIteratorIterator(
            new \RecursiveDirectoryIterator($this->directory, \FilesystemIterator::SKIP_DOTS),
            \RecursiveIteratorIterator::CHILD_FIRST,
        );
        foreach ($iterator as $entry) {
            $entry->isDir() ? rmdir($entry->getPathname()) : unlink($entry->getPathname());
        }
        rmdir($this->directory);
    }
    public function testPrepareReusesOnlyVendorAndAnonymousCaches(): void
    {
        $files = [$this->vendorFile, $this->anonymousFile, $this->ordinaryFile];
        $generator = $this->generator();
        $first = $generator->prepare($files, [$this->anonymousFile => '@typephp:anon:test']);
        self::assertSame($files, $first->pendingFiles());
        self::assertTrue($first->isVendorFile($this->vendorFile));
        self::assertTrue($first->isAnonymousFile($this->anonymousFile));
        self::assertNotSame(
            $first->cacheDirectory($this->vendorFile),
            $first->cacheDirectory($this->anonymousFile),
        );
        $vendorBlob = $this->writeBlob($first->cacheDirectory($this->vendorFile), $this->vendorFile, 'vendor');
        $anonymousBlob = $this->writeBlob(
            $first->cacheDirectory($this->anonymousFile),
            $this->anonymousFile,
            'anonymous',
        );
        $ordinaryBlob = $this->writeBlob(
            $first->cacheDirectory($this->ordinaryFile),
            $this->ordinaryFile,
            'ordinary',
        );
        $second = $generator->prepare($files, [$this->anonymousFile => '@typephp:anon:test']);
        self::assertSame([
            $this->vendorFile => $vendorBlob,
            $this->anonymousFile => $anonymousBlob,
        ], $second->cachedBlobs());
        self::assertSame([$this->ordinaryFile], $second->pendingFiles());
        self::assertSame(1, $second->vendorHits());
        self::assertSame(1, $second->anonymousHits());
        self::assertFileDoesNotExist($ordinaryBlob);
        self::assertFileExists($this->buildDirectory . '/cache/opcache/compile-embedded-opcodes.php');
    }
    public function testForceClearsPersistentOpcodeCaches(): void
    {
        $files = [$this->vendorFile, $this->anonymousFile];
        $anonymousKeys = [$this->anonymousFile => '@typephp:anon:test'];
        $first = $this->generator()->prepare($files, $anonymousKeys);
        $this->writeBlob($first->cacheDirectory($this->vendorFile), $this->vendorFile, 'vendor');
        $this->writeBlob($first->cacheDirectory($this->anonymousFile), $this->anonymousFile, 'anonymous');
        $forced = $this->generator(true)->prepare($files, $anonymousKeys);
        self::assertSame([], $forced->cachedBlobs());
        self::assertSame($files, $forced->pendingFiles());
    }
    public function testVendorSkipMarkerAvoidsRepeatedCompilation(): void
    {
        $generator = $this->generator();
        $first = $generator->prepare([$this->vendorFile], []);
        $cacheDirectory = $first->cacheDirectory($this->vendorFile);
        mkdir($cacheDirectory, 0777, true);
        file_put_contents(
            $cacheDirectory . '/skipped-' . hash('sha256', $this->vendorFile) . '.txt',
            $this->vendorFile,
        );
        $second = $generator->prepare([$this->vendorFile], []);
        self::assertSame([], $second->pendingFiles());
        self::assertSame(1, $second->skippedVendorCount());
        self::assertSame(0, $second->vendorHits());
    }
    private function generator(bool $force = false): EmbeddedOpcodeGenerator
    {
        return new EmbeddedOpcodeGenerator(
            $this->buildDirectory,
            'app',
            PHP_BINARY,
            [],
            PHP_VERSION,
            'build-signature',
            $force,
        );
    }
    private function writeBlob(string $cacheDirectory, string $source, string $contents): string
    {
        $blob = $cacheDirectory . '/system' . $source . '.bin';
        mkdir(dirname($blob), 0777, true);
        file_put_contents($blob, $contents);
        return $blob;
    }
 }
--- a/src/Build/EmbeddedOpcodeBatch.php
+++ b/src/Build/EmbeddedOpcodeBatch.php
@ -0,0 +1,88 @@
 <?php
 /**
 * This file is part of TypePHP(AOT).
 *
 * @link     https://www.swoole.com/aot/
 * @contact  service@swoole.com
 */
 namespace TypePhp\Build;
 final readonly class EmbeddedOpcodeBatch
 {
    /**
     * @param array<string, string> $cachedBlobs
     * @param list<string> $pendingFiles
     * @param array<string, string> $cacheDirectories
     * @param array<string, true> $vendorFiles
     * @param array<string, true> $anonymousFiles
     * @param array<string, true> $skippedVendorFiles
     */
    public function __construct(
        private array $cachedBlobs,
        private array $pendingFiles,
        private array $cacheDirectories,
        private array $vendorFiles,
        private array $anonymousFiles,
        private array $skippedVendorFiles,
    ) {
    }
    /** @return array<string, string> */
    public function cachedBlobs(): array
    {
        return $this->cachedBlobs;
    }
    /** @return list<string> */
    public function pendingFiles(): array
    {
        return $this->pendingFiles;
    }
    public function pendingCount(): int
    {
        return count($this->pendingFiles);
    }
    public function cacheDirectory(string $file): string
    {
        return $this->cacheDirectories[$file]
            ?? throw new \LogicException("Missing OPcache directory for: {$file}");
    }
    public function isVendorFile(string $file): bool
    {
        return isset($this->vendorFiles[$file]);
    }
    public function isAnonymousFile(string $file): bool
    {
        return isset($this->anonymousFiles[$file]);
    }
    public function vendorHits(): int
    {
        return count(array_intersect_key($this->cachedBlobs, $this->vendorFiles));
    }
    public function anonymousHits(): int
    {
        return count(array_intersect_key($this->cachedBlobs, $this->anonymousFiles));
    }
    public function vendorCount(): int
    {
        return count($this->vendorFiles);
    }
    public function anonymousCount(): int
    {
        return count($this->anonymousFiles);
    }
    public function skippedVendorCount(): int
    {
        return count($this->skippedVendorFiles);
    }
 }
--- a/src/Build/EmbeddedOpcodeGenerator.php
+++ b/src/Build/EmbeddedOpcodeGenerator.php
@ -0,0 +1,362 @@
 <?php
 /**
 * This file is part of TypePHP(AOT).
 *
 * @link     https://www.swoole.com/aot/
 * @contact  service@swoole.com
 */
 namespace TypePhp\Build;
 final class EmbeddedOpcodeGenerator
 {
    private const string DRIVER_SOURCE = <<<'PHP'
 <?php
 echo PHP_VERSION, "\n";
 foreach (array_slice($argv, 1) as $path) {
    if (!opcache_compile_file($path)) {
        fwrite(STDERR, "OPcache could not compile: {$path}\n");
        exit(1);
    }
 }
 PHP;
    /** @param list<string> $extensionArguments */
    public function __construct(
        private readonly string $buildDirectory,
        private readonly string $targetName,
        private readonly string $php,
        private readonly array $extensionArguments,
        private readonly string $phpVersion,
        private readonly string $buildSignature,
        private readonly bool $force,
    ) {
    }
    /**
     * Resolve persistent cache hits and produce an immutable compilation batch.
     *
     * @param list<string> $files
     * @param array<string, string> $anonymousKeys
     */
    public function prepare(array $files, array $anonymousKeys): EmbeddedOpcodeBatch
    {
        $cacheDirectories = [];
        $vendorGroups = [];
        $vendorFiles = [];
        $anonymousFiles = [];
        $otherFiles = [];
        foreach ($files as $file) {
            if (isset($anonymousKeys[$file])) {
                $cacheDirectory = $this->anonymousCacheDirectory($file);
                if ($this->force) {
                    $this->clearDirectory($cacheDirectory);
                }
                $cacheDirectories[$file] = $cacheDirectory;
                $anonymousFiles[$file] = true;
                continue;
            }
            $vendorRoot = $this->vendorRoot($file);
            if ($vendorRoot === null) {
                $otherFiles[] = $file;
                continue;
            }
            $vendorGroups[$vendorRoot][] = $file;
            $vendorFiles[$file] = true;
        }
        foreach ($vendorGroups as $vendorRoot => $vendorGroup) {
            $cacheDirectory = $this->vendorCacheDirectory($vendorRoot);
            if ($this->force) {
                $this->clearDirectory($cacheDirectory);
            }
            foreach ($vendorGroup as $file) {
                $cacheDirectories[$file] = $cacheDirectory;
            }
        }
        if ($otherFiles !== []) {
            // Ordinary directories have no reliable invalidation boundary.
            // Recreate their shared staging directory for every build.
            $cacheDirectory = $this->buildDirectory . '/cache/opcache/nonvendor-'
                . substr(hash('sha256', $this->targetName), 0, 20);
            $this->clearDirectory($cacheDirectory);
            foreach ($otherFiles as $file) {
                $cacheDirectories[$file] = $cacheDirectory;
            }
        }
        $this->writeDriver();
        return $this->resolve($files, $cacheDirectories, $vendorFiles, $anonymousFiles);
    }
    /**
     * Compile only the cache misses from a prepared batch.
     *
     * @param callable(string): void|null $onSkipped
     * @param callable(int, int, string): void|null $onProgress
     * @return array<string, string> source file => OPcache blob
     */
    public function compile(
        EmbeddedOpcodeBatch $batch,
        ?callable $onSkipped = null,
        ?callable $onProgress = null,
    ): array {
        $blobs = $batch->cachedBlobs();
        $total = $batch->pendingCount();
        foreach ($batch->pendingFiles() as $index => $file) {
            $cacheDirectory = $batch->cacheDirectory($file);
            $this->ensureDirectory($cacheDirectory);
            [$status, $stdout, $stderr] = $this->compileFile($file, $cacheDirectory);
            if ($status !== 0) {
                if ($batch->isAnonymousFile($file)
                    || !str_contains($stderr, "OPcache could not compile: {$file}")) {
                    throw new \RuntimeException("Cannot compile embedded opcode {$file}:\n{$stderr}");
                }
                if ($batch->isVendorFile($file)) {
                    AtomicFile::write($this->skippedMarker($cacheDirectory, $file), $file . PHP_EOL, '.skip-');
                }
                if ($onSkipped !== null) {
                    $onSkipped($file);
                }
            } else {
                // Compiled files may emit PHP deprecation notices after the
                // version line printed by the driver.
                if (strtok($stdout, "\r\n") !== $this->phpVersion) {
                    throw new \RuntimeException("Build PHP version changed while compiling {$file}");
                }
                $blob = $this->findBlob($cacheDirectory, $file);
                if ($blob === null) {
                    throw new \RuntimeException("Missing OPcache blob for {$file}");
                }
                $blobs[$file] = $blob;
            }
            if ($onProgress !== null) {
                $onProgress($index + 1, $total, $file);
            }
        }
        uksort($blobs, static fn (string $left, string $right): int => strcmp($left, $right));
        return $blobs;
    }
    /**
     * @param list<string> $files
     * @param array<string, string> $cacheDirectories
     * @param array<string, true> $vendorFiles
     * @param array<string, true> $anonymousFiles
     */
    private function resolve(
        array $files,
        array $cacheDirectories,
        array $vendorFiles,
        array $anonymousFiles,
    ): EmbeddedOpcodeBatch {
        $cachedBlobs = [];
        $pendingFiles = [];
        $skippedVendorFiles = [];
        foreach ($files as $file) {
            $cacheDirectory = $cacheDirectories[$file];
            if (isset($vendorFiles[$file])
                && is_file($this->skippedMarker($cacheDirectory, $file))) {
                $skippedVendorFiles[$file] = true;
                continue;
            }
            if ((isset($vendorFiles[$file]) || isset($anonymousFiles[$file]))
                && ($blob = $this->findBlob($cacheDirectory, $file)) !== null) {
                $cachedBlobs[$file] = $blob;
                continue;
            }
            $pendingFiles[] = $file;
        }
        return new EmbeddedOpcodeBatch(
            $cachedBlobs,
            $pendingFiles,
            $cacheDirectories,
            $vendorFiles,
            $anonymousFiles,
            $skippedVendorFiles,
        );
    }
    private function anonymousCacheDirectory(string $file): string
    {
        $fileHash = hash_file('sha256', $file);
        if (!is_string($fileHash)) {
            throw new \RuntimeException("Cannot hash anonymous class source: {$file}");
        }
        $key = hash('sha256', implode("\n", [
            'anonymous-opcodes-v1',
            $file,
            $fileHash,
            $this->buildSignature,
        ]));
        return $this->buildDirectory . '/cache/opcache/anonymous-' . substr($key, 0, 20);
    }
    private function vendorCacheDirectory(string $vendorRoot): string
    {
        clearstatcache(true, $vendorRoot);
        $mtime = filemtime($vendorRoot);
        if ($mtime === false) {
            throw new \RuntimeException("Cannot read vendor directory mtime: {$vendorRoot}");
        }
        $key = hash('sha256', implode("\n", [
            'vendor-directory-mtime-opcodes-v2',
            $vendorRoot,
            (string) $mtime,
            $this->buildSignature,
        ]));
        return $this->buildDirectory . '/cache/opcache/vendor-' . substr($key, 0, 20);
    }
    /** Only files under a vendor directory with autoload.php use the persistent cache. */
    private function vendorRoot(string $file): ?string
    {
        for ($directory = dirname($file); $directory !== dirname($directory); $directory = dirname($directory)) {
            if (basename($directory) === 'vendor' && is_file($directory . '/autoload.php')) {
                return $directory;
            }
        }
        return null;
    }
    /** @return array{int, string, string} */
    private function compileFile(string $file, string $cacheDirectory): array
    {
        $command = [
            $this->php,
            '-n',
            ...$this->extensionArguments,
            '-d',
            'opcache.enable_cli=1',
            '-d',
            'opcache.file_cache_only=1',
            '-d',
            'opcache.file_update_protection=0',
            '-d',
            'opcache.file_cache=' . $cacheDirectory,
            $this->driverPath(),
            $file,
        ];
        $process = proc_open($command, [
            0 => ['pipe', 'r'],
            1 => ['pipe', 'w'],
            2 => ['pipe', 'w'],
        ], $pipes);
        if (!is_resource($process)) {
            throw new \RuntimeException('Cannot start the PHP OPcache build process');
        }
        fclose($pipes[0]);
        $stdout = stream_get_contents($pipes[1]);
        $stderr = stream_get_contents($pipes[2]);
        fclose($pipes[1]);
        fclose($pipes[2]);
        return [proc_close($process), (string) $stdout, (string) $stderr];
    }
    private function writeDriver(): void
    {
        $path = $this->driverPath();
        if (is_file($path) && file_get_contents($path) === self::DRIVER_SOURCE) {
            return;
        }
        AtomicFile::write($path, self::DRIVER_SOURCE, '.driver-');
    }
    private function driverPath(): string
    {
        return $this->buildDirectory . '/cache/opcache/compile-embedded-opcodes.php';
    }
    private function skippedMarker(string $cacheDirectory, string $file): string
    {
        return $cacheDirectory . '/skipped-' . hash('sha256', $file) . '.txt';
    }
    private function findBlob(string $cacheDirectory, string $file): ?string
    {
        $matches = [];
        foreach (glob($cacheDirectory . '/*', GLOB_ONLYDIR) ?: [] as $systemDirectory) {
            $candidate = $systemDirectory . $file . '.bin';
            if (is_file($candidate) && filesize($candidate) > 0) {
                $matches[] = $candidate;
            }
        }
        if (count($matches) > 1) {
            throw new \RuntimeException("Ambiguous OPcache blob for {$file}");
        }
        if ($matches !== []) {
            return $matches[0];
        }
        if (DIRECTORY_SEPARATOR !== '\\' || !is_dir($cacheDirectory)) {
            return null;
        }
        // OPcache omits the drive colon from Windows cache paths (C:\foo
        // becomes C\foo) after two system-specific cache directories.
        $cacheSuffix = str_replace('/', '\\', preg_replace('/^([A-Za-z]):/', '$1', $file)) . '.bin';
        foreach (glob($cacheDirectory . '/*', GLOB_ONLYDIR) ?: [] as $systemDirectory) {
            foreach (glob($systemDirectory . '/*', GLOB_ONLYDIR) ?: [] as $subDirectory) {
                $candidate = $subDirectory . DIRECTORY_SEPARATOR . $cacheSuffix;
                if (is_file($candidate) && filesize($candidate) > 0) {
                    $matches[] = $candidate;
                }
            }
        }
        if (count($matches) > 1) {
            throw new \RuntimeException("Ambiguous OPcache blob for {$file}");
        }
        if ($matches !== []) {
            return $matches[0];
        }
        // Retain compatibility with PHP builds that use another number of
        // system-specific directories.
        $iterator = new \RecursiveIteratorIterator(
            new \RecursiveDirectoryIterator($cacheDirectory, \FilesystemIterator::SKIP_DOTS),
        );
        foreach ($iterator as $cached) {
            if ($cached->isFile() && $cached->getSize() > 0
                && str_ends_with($cached->getPathname(), $cacheSuffix)) {
                if ($matches !== []) {
                    throw new \RuntimeException("Ambiguous OPcache blob for {$file}");
                }
                $matches[] = $cached->getPathname();
            }
        }
        return $matches[0] ?? null;
    }
    private function clearDirectory(string $directory): void
    {
        if (!is_dir($directory)) {
            return;
        }
        $iterator = new \RecursiveIteratorIterator(
            new \RecursiveDirectoryIterator($directory, \FilesystemIterator::SKIP_DOTS),
            \RecursiveIteratorIterator::CHILD_FIRST,
        );
        foreach ($iterator as $entry) {
            $path = $entry->getPathname();
            if ($entry->isDir() ? !rmdir($path) : !unlink($path)) {
                throw new \RuntimeException("Cannot clear opcode build directory: {$path}");
            }
        }
        if (!rmdir($directory)) {
            throw new \RuntimeException("Cannot clear opcode build directory: {$directory}");
        }
    }
    private function ensureDirectory(string $directory): void
    {
        if (!is_dir($directory) && !mkdir($directory, 0777, true) && !is_dir($directory)) {
            throw new \RuntimeException("Cannot create OPcache build directory: {$directory}");
        }
    }
 }
--- a/src/Build/SourcePipelineTrait.php
+++ b/src/Build/SourcePipelineTrait.php
@ -301,98 +301,27 @@ trait SourcePipelineTrait
        }
    }
-    /** Only files under a vendor directory with autoload.php use the mtime cache. */
+    /** Embed OPcache's file-cache bytes for scripts left to ZendVM. */
-    private function vendorRootForOpcodeFile(string $file): ?string
+    private function genEmbeddedOpcodeTable(): array
    {
-        for ($directory = dirname($file); $directory !== dirname($directory); $directory = dirname($directory)) {
+        $output = $this->getBuildDir() . '/embedded-opcodes-' . $this->targetName . '.cc';
-            if (basename($directory) === 'vendor' && is_file($directory . '/autoload.php')) {
+        $this->migrateEmbeddedOpcodeCaches();
-                return $directory;
+        $files = array_values(array_unique($this->embeddedOpcodeFiles));
-            }
+        sort($files, SORT_STRING);
        }
        return null;
    }
-    private function findOpcodeBlob(string $cacheDir, string $file): ?string
+        $blobs = [];
-    {
+        $phpVersion = PHP_VERSION;
-        $matches = [];
+        if ($files !== []) {
-        foreach (glob($cacheDir . '/*', GLOB_ONLYDIR) ?: [] as $systemDirectory) {
+            ['blobs' => $blobs, 'phpVersion' => $phpVersion] = $this->generateEmbeddedOpcodeBlobs($files);
            $candidate = $systemDirectory . $file . '.bin';
            if (is_file($candidate) && filesize($candidate) > 0) {
                $matches[] = $candidate;
            }
        }
        if (count($matches) > 1) {
            throw new \RuntimeException("Ambiguous OPcache blob for {$file}");
        }
        if ($matches !== []) {
            return $matches[0];
        }
        // OPcache uses a different file-cache path layout on Windows.
        if (DIRECTORY_SEPARATOR === '\\' && is_dir($cacheDir)) {
            // The drive colon is omitted from OPcache's cache path (C:\foo
            // becomes C\foo), after two system-specific cache directories.
            $windowsCacheSuffix = str_replace('/', '\\', preg_replace('/^([A-Za-z]):/', '$1', $file)) . '.bin';
            foreach (glob($cacheDir . '/*', GLOB_ONLYDIR) ?: [] as $systemDirectory) {
                foreach (glob($systemDirectory . '/*', GLOB_ONLYDIR) ?: [] as $subDirectory) {
                    $candidate = $subDirectory . DIRECTORY_SEPARATOR . $windowsCacheSuffix;
                    if (is_file($candidate) && filesize($candidate) > 0) {
                        $matches[] = $candidate;
                    }
                }
            }
            if (count($matches) > 1) {
                throw new \RuntimeException("Ambiguous OPcache blob for {$file}");
            }
            if ($matches !== []) {
                return $matches[0];
            }
            // Keep supporting other OPcache layouts without assuming the same
            // number of system directories in every PHP build.
            $iterator = new \RecursiveIteratorIterator(
                new \RecursiveDirectoryIterator($cacheDir, \FilesystemIterator::SKIP_DOTS),
            );
            foreach ($iterator as $cached) {
                if ($cached->isFile() && $cached->getSize() > 0
                    && str_ends_with($cached->getPathname(), $windowsCacheSuffix)) {
                    if ($matches !== []) {
                        throw new \RuntimeException("Ambiguous OPcache blob for {$file}");
                    }
                    $matches[] = $cached->getPathname();
                }
            }
        }
        return $matches[0] ?? null;
    }
-    private function getSkippedOpcodeMarker(string $cacheDir, string $file): string
+        $sources = $this->emitEmbeddedArchiveSources($output, $files, $blobs, $phpVersion);
-    {
+        $this->removeLegacyAnonymousArtifacts();
-        return $cacheDir . '/skipped-' . hash('sha256', $file) . '.txt';
+        return $sources;
    }
    private function clearOpcodeCacheDirectory(string $directory): void
    {
        if (!is_dir($directory)) {
            return;
        }
        $iterator = new \RecursiveIteratorIterator(
            new \RecursiveDirectoryIterator($directory, \FilesystemIterator::SKIP_DOTS),
            \RecursiveIteratorIterator::CHILD_FIRST,
        );
        foreach ($iterator as $entry) {
            $path = $entry->getPathname();
            if ($entry->isDir() ? !rmdir($path) : !unlink($path)) {
                throw new \RuntimeException("Cannot clear opcode build directory: {$path}");
            }
        }
        if (!rmdir($directory)) {
            throw new \RuntimeException("Cannot clear opcode build directory: {$directory}");
        }
    }
-    /** Embed OPcache's file-cache bytes for scripts left to ZendVM. */
+    private function migrateEmbeddedOpcodeCaches(): void
    private function genEmbeddedOpcodeTable(): array
    {
        $output = $this->getBuildDir() . '/embedded-opcodes-' . $this->targetName . '.cc';
        foreach (glob($this->getBuildDir() . '/opcache-*') ?: [] as $legacyCache) {
            if (is_dir($legacyCache)) {
                $this->moveLegacyBuildCache(
@ -401,195 +330,85 @@ trait SourcePipelineTrait
                );
            }
        }
-        $files = array_values(array_unique($this->embeddedOpcodeFiles));
+        $this->moveLegacyBuildCache(
-        sort($files, SORT_STRING);
+            $this->getBuildDir() . '/compile-embedded-opcodes.php',
-        $blobs = [];
+            $this->getBuildDir() . '/cache/opcache/compile-embedded-opcodes.php',
-        $phpVersion = PHP_VERSION;
+        );
    }
-        if ($files !== []) {
+    /** @param list<string> $files @return array{blobs: array<string, string>, phpVersion: string} */
-            if ($this->isIosTarget() || $this->isAndroidTarget() || $this->isWasiTarget()) {
+    private function generateEmbeddedOpcodeBlobs(array $files): array
-                throw new \RuntimeException('Embedded opcodes require a native PHP build host matching the target');
+    {
-            }
+        if ($this->isIosTarget() || $this->isAndroidTarget() || $this->isWasiTarget()) {
-            $this->output('Generating embedded opcodes for ' . count($files) . ' PHP files', 'lightBlue');
+            throw new \RuntimeException('Embedded opcodes require a native PHP build host matching the target');
-            $php = $this->getOpcodeBuildPhpCli();
+        }
-            $extensionArgs = $this->getOpcodeBuildExtensionArgs();
+        $this->output('Generating embedded opcodes for ' . count($files) . ' PHP files', 'lightBlue');
-            $phpVersion = $this->opcodeBuildPhpVersion;
+
-            $otherFiles = [];
+        $generator = new EmbeddedOpcodeGenerator(
-            $vendorRoots = [];
+            $this->getBuildDir(),
-            $vendorFiles = [];
+            $this->targetName,
-            $anonymousFiles = [];
+            $this->getOpcodeBuildPhpCli(),
-            $fileCacheDirs = [];
+            $this->getOpcodeBuildExtensionArgs(),
-            foreach ($files as $file) {
+            $this->opcodeBuildPhpVersion,
-                if (isset($this->anonymousOpcodeKeys[$file])) {
+            $this->opcodeBuildSignature,
-                    // These files are emitted by this compiler, so their exact
+            $this->climate->arguments->defined('force'),
-                    // contents provide a safe cache key independent of any
+        );
-                    // user-supplied directory timestamp.
+        $batch = $generator->prepare($files, $this->anonymousOpcodeKeys);
-                    $key = hash('sha256', implode("\n", [
+        $this->reportEmbeddedOpcodeCache($batch);
-                        'anonymous-opcodes-v1', $file,
+
-                        hash_file('sha256', $file), $this->opcodeBuildSignature,
+        $progress = $this->startOpcodeProgress('Opcodes', $batch->pendingCount());
-                    ]));
+        $blobs = $generator->compile(
-                    $cacheDir = $this->getBuildDir() . '/cache/opcache/anonymous-' . substr($key, 0, 20);
+            $batch,
-                    if ($this->climate->arguments->defined('force')) {
+            fn(string $file) => $this->climate->warning(
-                        $this->clearOpcodeCacheDirectory($cacheDir);
+                'Skipping non-executable embedded PHP file: ' . $file,
-                    }
+            ),
-                    $fileCacheDirs[$file] = $cacheDir;
+            fn(int $completed, int $total, string $file) => $this->updateOpcodeProgress(
-                    $anonymousFiles[$file] = true;
+                $progress,
-                    continue;
+                'Opcodes',
-                }
+                $completed,
-                $vendorRoot = $this->vendorRootForOpcodeFile($file);
+                $total,
-                if ($vendorRoot === null) {
+                $this->noProgress ? $file : null,
-                    $otherFiles[] = $file;
+            ),
-                    continue;
+        );
-                }
+        if ($progress !== null) {
-                $vendorRoots[$vendorRoot][] = $file;
+            echo PHP_EOL;
-                $vendorFiles[$file] = true;
+        }
-            }
+        return ['blobs' => $blobs, 'phpVersion' => $this->opcodeBuildPhpVersion];
            foreach ($vendorRoots as $vendorRoot => $filesInVendor) {
                clearstatcache(true, $vendorRoot);
                $mtime = filemtime($vendorRoot);
                if ($mtime === false) {
                    throw new \RuntimeException("Cannot read vendor directory mtime: {$vendorRoot}");
                }
                $key = hash('sha256', implode("\n", [
                    'vendor-directory-mtime-opcodes-v2', $vendorRoot,
                    (string) $mtime, $this->opcodeBuildSignature,
                ]));
                $cacheDir = $this->getBuildDir() . '/cache/opcache/vendor-' . substr($key, 0, 20);
                if ($this->climate->arguments->defined('force')) {
                    $this->clearOpcodeCacheDirectory($cacheDir);
                }
                foreach ($filesInVendor as $file) {
                    $fileCacheDirs[$file] = $cacheDir;
                }
            }
            if ($otherFiles !== []) {
                // Recreate this staging directory on every build so OPcache
                // cannot reuse bytecode for non-vendor or anonymous sources.
                $cacheDir = $this->getBuildDir() . '/cache/opcache/nonvendor-'
                    . substr(hash('sha256', $this->targetName), 0, 20);
                $this->clearOpcodeCacheDirectory($cacheDir);
                foreach ($otherFiles as $file) {
                    $fileCacheDirs[$file] = $cacheDir;
                }
            }
            $driver = $this->getBuildDir() . '/cache/opcache/compile-embedded-opcodes.php';
            $this->moveLegacyBuildCache(
                $this->getBuildDir() . '/compile-embedded-opcodes.php',
                $driver,
            );
            $this->writeFile($driver, <<<'PHP'
 <?php
 echo PHP_VERSION, "\n";
 foreach (array_slice($argv, 1) as $path) {
    if (!opcache_compile_file($path)) {
        fwrite(STDERR, "OPcache could not compile: {$path}\n");
        exit(1);
    }
-}
+
-PHP
+    private function reportEmbeddedOpcodeCache(EmbeddedOpcodeBatch $batch): void
    {
        if ($batch->vendorCount() !== 0) {
            $vendorHits = $batch->vendorHits();
            $vendorSkips = $batch->skippedVendorCount();
            $this->output(
                'Vendor opcode cache: ' . $vendorHits . ' reused, '
                . ($batch->vendorCount() - $vendorHits - $vendorSkips) . ' to generate'
                . ($vendorSkips === 0 ? '' : ', ' . $vendorSkips . ' skipped'),
                'lightBlue',
            );
            $command = [
                $php, '-n', ...$extensionArgs,
                '-d', 'opcache.enable_cli=1',
                '-d', 'opcache.file_cache_only=1',
                '-d', 'opcache.file_update_protection=0',
            ];
            // OPcache installs declarations in the compiling CLI process.
            // Isolate files so unrelated scripts with the same global
            // function names do not conflict while creating their bytecode.
            $pending = [];
            $vendorHits = 0;
            $anonymousHits = 0;
            $vendorSkips = 0;
            foreach ($files as $file) {
                $cacheDir = $fileCacheDirs[$file];
                if (isset($vendorFiles[$file])
                    && is_file($this->getSkippedOpcodeMarker($cacheDir, $file))) {
                    ++$vendorSkips;
                    continue;
                }
                if ((isset($vendorFiles[$file]) || isset($anonymousFiles[$file]))
                    && ($blob = $this->findOpcodeBlob($cacheDir, $file)) !== null) {
                    $blobs[$file] = $blob;
                    if (isset($anonymousFiles[$file])) {
                        ++$anonymousHits;
                    } else {
                        ++$vendorHits;
                    }
                } else {
                    $pending[] = $file;
                }
            }
            if ($vendorRoots !== []) {
                $vendorCount = array_sum(array_map('count', $vendorRoots));
                $this->output(
                    'Vendor opcode cache: ' . $vendorHits . ' reused, '
                    . ($vendorCount - $vendorHits - $vendorSkips) . ' to generate'
                    . ($vendorSkips === 0 ? '' : ', ' . $vendorSkips . ' skipped'),
                    'lightBlue',
                );
            }
            if ($anonymousFiles !== []) {
                $this->output(
                    'Anonymous opcode cache: ' . $anonymousHits . ' reused, '
                    . (count($anonymousFiles) - $anonymousHits) . ' to generate',
                    'lightBlue',
                );
            }
            $progress = $this->startOpcodeProgress('Opcodes', count($pending));
            $completed = 0;
            foreach ($pending as $file) {
                $cacheDir = $fileCacheDirs[$file];
                if (!is_dir($cacheDir) && !mkdir($cacheDir, 0777, true) && !is_dir($cacheDir)) {
                    throw new \RuntimeException("Cannot create OPcache build directory: {$cacheDir}");
                }
                $process = proc_open([...$command, '-d', 'opcache.file_cache=' . $cacheDir, $driver, $file], [
                    0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w'],
                ], $pipes);
                if (!is_resource($process)) {
                    throw new \RuntimeException('Cannot start the PHP OPcache build process');
                }
                fclose($pipes[0]);
                $stdout = stream_get_contents($pipes[1]);
                $stderr = stream_get_contents($pipes[2]);
                fclose($pipes[1]);
                fclose($pipes[2]);
                if (proc_close($process) !== 0) {
                    if (isset($anonymousFiles[$file])
                        || !str_contains((string) $stderr, "OPcache could not compile: {$file}")) {
                        throw new \RuntimeException("Cannot compile embedded opcode {$file}:\n{$stderr}");
                    }
                    // Embedded directories can contain PHP-looking declaration
                    // files that are not scripts. Keep their raw bytes but do
                    // not treat them as executable bytecode.
                    $this->climate->warning('Skipping non-executable embedded PHP file: ' . $file);
                    if (isset($vendorFiles[$file])) {
                        $marker = $this->getSkippedOpcodeMarker($cacheDir, $file);
                        if (file_put_contents($marker, $file . PHP_EOL) === false) {
                            throw new \RuntimeException("Cannot cache skipped embedded PHP file: {$file}");
                        }
                    }
                    $this->updateOpcodeProgress($progress, 'Opcodes', ++$completed, count($pending), $this->noProgress ? $file : null);
                    continue;
                }
                // Compiled files may emit PHP deprecation notices on stdout.
                // The driver prints its version before compiling the file.
                if (strtok((string) $stdout, "\r\n") !== $phpVersion) {
                    throw new \RuntimeException("Build PHP version changed while compiling {$file}");
                }
                $blob = $this->findOpcodeBlob($cacheDir, $file);
                if ($blob === null) {
                    throw new \RuntimeException("Missing OPcache blob for {$file}");
                }
                $blobs[$file] = $blob;
                $this->updateOpcodeProgress($progress, 'Opcodes', ++$completed, count($pending), $this->noProgress ? $file : null);
            }
            if ($progress !== null) {
                echo PHP_EOL;
            }
            uksort($blobs, static fn(string $left, string $right): int => strcmp($left, $right));
        }
        if ($batch->anonymousCount() !== 0) {
            $anonymousHits = $batch->anonymousHits();
            $this->output(
                'Anonymous opcode cache: ' . $anonymousHits . ' reused, '
                . ($batch->anonymousCount() - $anonymousHits) . ' to generate',
                'lightBlue',
            );
        }
    }
    /**
     * @param list<string> $opcodeFiles
     * @param array<string, string> $blobs
     * @return list<string>
     */
    private function emitEmbeddedArchiveSources(
        string $output,
        array $opcodeFiles,
        array $blobs,
        string $phpVersion,
    ): array {
        $archive = $this->getBuildDir() . '/cache/embedded/embedded-files-' . $this->targetName . '.bin';
        $this->moveLegacyBuildCache(
            $this->getBuildDir() . '/embedded-files-' . $this->targetName . '.bin',
@ -597,7 +416,7 @@ PHP
        );
        $sources = [$output];
        $embeddedArchive = EmbeddedArchive::empty($archive);
-        if ($this->embeddedFiles !== [] || $files !== []) {
+        if ($this->embeddedFiles !== [] || $opcodeFiles !== []) {
            $embeddedArchive = (new EmbeddedArchiveBuilder())->build(
                $archive,
                $this->embeddedFiles,
@ -629,13 +448,17 @@ PHP
            (new EmbeddedTableRenderer())->render($embeddedArchive, $phpVersion, $this->isWindows()),
        );
        $this->generatedProjectSources[$output] = true;
        return $sources;
    }
    private function removeLegacyAnonymousArtifacts(): void
    {
        foreach (glob($this->getBuildDir() . '/anonymous-*.json') ?: [] as $legacyManifest) {
            unlink($legacyManifest);
        }
        foreach (glob($this->getBuildDir() . '/anonymous-' . $this->targetName . '-*.php') ?: [] as $legacySource) {
            unlink($legacySource);
        }
        return $sources;
    }
    public function getFiles(string $path): array