aot
/
compiler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(build): clarify embedded opcode pipeline

Browse Source
master
韩天峰 3 weeks ago
parent 5c3da4e406
commit 4216f21b02
4 changed files with 688 additions and 271 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 144
      phpunit/src/Build/EmbeddedOpcodeGeneratorTest.php
  2. 88
      src/Build/EmbeddedOpcodeBatch.php
  3. 362
      src/Build/EmbeddedOpcodeGenerator.php
  4. 365
      src/Build/SourcePipelineTrait.php

144
phpunit/src/Build/EmbeddedOpcodeGeneratorTest.php
Unescape View File

@ -0,0 +1,144 @@
<?php
/**
* This file is part of TypePHP(AOT).
*
* @link https://www.swoole.com/aot/
* @contact service@swoole.com
*/
namespace TypePhp\Tests\Build;
use PHPUnit\Framework\TestCase;
use TypePhp\Build\EmbeddedOpcodeGenerator;
/**
* @internal
* @coversNothing
*/
final class EmbeddedOpcodeGeneratorTest extends TestCase
{
private string $directory;
private string $buildDirectory;
private string $vendorFile;
private string $anonymousFile;
private string $ordinaryFile;
protected function setUp(): void
{
$this->directory = sys_get_temp_dir() . '/typephp-opcode-generator-' . bin2hex(random_bytes(6));
$this->buildDirectory = $this->directory . '/build';
$vendor = $this->directory . '/vendor';
mkdir($vendor . '/package', 0777, true);
file_put_contents($vendor . '/autoload.php', '<?php');
$this->vendorFile = $vendor . '/package/library.php';
$this->anonymousFile = $this->directory . '/anonymous.php';
$this->ordinaryFile = $this->directory . '/ordinary.php';
file_put_contents($this->vendorFile, '<?php return 1;');
file_put_contents($this->anonymousFile, '<?php return new class {};');
file_put_contents($this->ordinaryFile, '<?php return 2;');
}
protected function tearDown(): void
{
$iterator = new \RecursiveIteratorIterator(
new \RecursiveDirectoryIterator($this->directory, \FilesystemIterator::SKIP_DOTS),
\RecursiveIteratorIterator::CHILD_FIRST,
);
foreach ($iterator as $entry) {
$entry->isDir() ? rmdir($entry->getPathname()) : unlink($entry->getPathname());
}
rmdir($this->directory);
}
public function testPrepareReusesOnlyVendorAndAnonymousCaches(): void
{
$files = [$this->vendorFile, $this->anonymousFile, $this->ordinaryFile];
$generator = $this->generator();
$first = $generator->prepare($files, [$this->anonymousFile => '@typephp:anon:test']);
self::assertSame($files, $first->pendingFiles());
self::assertTrue($first->isVendorFile($this->vendorFile));
self::assertTrue($first->isAnonymousFile($this->anonymousFile));
self::assertNotSame(
$first->cacheDirectory($this->vendorFile),
$first->cacheDirectory($this->anonymousFile),
);
$vendorBlob = $this->writeBlob($first->cacheDirectory($this->vendorFile), $this->vendorFile, 'vendor');
$anonymousBlob = $this->writeBlob(
$first->cacheDirectory($this->anonymousFile),
$this->anonymousFile,
'anonymous',
);
$ordinaryBlob = $this->writeBlob(
$first->cacheDirectory($this->ordinaryFile),
$this->ordinaryFile,
'ordinary',
);
$second = $generator->prepare($files, [$this->anonymousFile => '@typephp:anon:test']);
self::assertSame([
$this->vendorFile => $vendorBlob,
$this->anonymousFile => $anonymousBlob,
], $second->cachedBlobs());
self::assertSame([$this->ordinaryFile], $second->pendingFiles());
self::assertSame(1, $second->vendorHits());
self::assertSame(1, $second->anonymousHits());
self::assertFileDoesNotExist($ordinaryBlob);
self::assertFileExists($this->buildDirectory . '/cache/opcache/compile-embedded-opcodes.php');
}
public function testForceClearsPersistentOpcodeCaches(): void
{
$files = [$this->vendorFile, $this->anonymousFile];
$anonymousKeys = [$this->anonymousFile => '@typephp:anon:test'];
$first = $this->generator()->prepare($files, $anonymousKeys);
$this->writeBlob($first->cacheDirectory($this->vendorFile), $this->vendorFile, 'vendor');
$this->writeBlob($first->cacheDirectory($this->anonymousFile), $this->anonymousFile, 'anonymous');
$forced = $this->generator(true)->prepare($files, $anonymousKeys);
self::assertSame([], $forced->cachedBlobs());
self::assertSame($files, $forced->pendingFiles());
}
public function testVendorSkipMarkerAvoidsRepeatedCompilation(): void
{
$generator = $this->generator();
$first = $generator->prepare([$this->vendorFile], []);
$cacheDirectory = $first->cacheDirectory($this->vendorFile);
mkdir($cacheDirectory, 0777, true);
file_put_contents(
$cacheDirectory . '/skipped-' . hash('sha256', $this->vendorFile) . '.txt',
$this->vendorFile,
);
$second = $generator->prepare([$this->vendorFile], []);
self::assertSame([], $second->pendingFiles());
self::assertSame(1, $second->skippedVendorCount());
self::assertSame(0, $second->vendorHits());
}
private function generator(bool $force = false): EmbeddedOpcodeGenerator
{
return new EmbeddedOpcodeGenerator(
$this->buildDirectory,
'app',
PHP_BINARY,
[],
PHP_VERSION,
'build-signature',
$force,
);
}
private function writeBlob(string $cacheDirectory, string $source, string $contents): string
{
$blob = $cacheDirectory . '/system' . $source . '.bin';
mkdir(dirname($blob), 0777, true);
file_put_contents($blob, $contents);
return $blob;
}
}

88
src/Build/EmbeddedOpcodeBatch.php
Unescape View File

@ -0,0 +1,88 @@
<?php
/**
* This file is part of TypePHP(AOT).
*
* @link https://www.swoole.com/aot/
* @contact service@swoole.com
*/
namespace TypePhp\Build;
final readonly class EmbeddedOpcodeBatch
{
/**
* @param array<string, string> $cachedBlobs
* @param list<string> $pendingFiles
* @param array<string, string> $cacheDirectories
* @param array<string, true> $vendorFiles
* @param array<string, true> $anonymousFiles
* @param array<string, true> $skippedVendorFiles
*/
public function __construct(
private array $cachedBlobs,
private array $pendingFiles,
private array $cacheDirectories,
private array $vendorFiles,
private array $anonymousFiles,
private array $skippedVendorFiles,
) {
}
/** @return array<string, string> */
public function cachedBlobs(): array
{
return $this->cachedBlobs;
}
/** @return list<string> */
public function pendingFiles(): array
{
return $this->pendingFiles;
}
public function pendingCount(): int
{
return count($this->pendingFiles);
}
public function cacheDirectory(string $file): string
{
return $this->cacheDirectories[$file]
?? throw new \LogicException("Missing OPcache directory for: {$file}");
}
public function isVendorFile(string $file): bool
{
return isset($this->vendorFiles[$file]);
}
public function isAnonymousFile(string $file): bool
{
return isset($this->anonymousFiles[$file]);
}
public function vendorHits(): int
{
return count(array_intersect_key($this->cachedBlobs, $this->vendorFiles));
}
public function anonymousHits(): int
{
return count(array_intersect_key($this->cachedBlobs, $this->anonymousFiles));
}
public function vendorCount(): int
{
return count($this->vendorFiles);
}
public function anonymousCount(): int
{
return count($this->anonymousFiles);
}
public function skippedVendorCount(): int
{
return count($this->skippedVendorFiles);
}
}

362
src/Build/EmbeddedOpcodeGenerator.php
Unescape View File

@ -0,0 +1,362 @@
<?php
/**
* This file is part of TypePHP(AOT).
*
* @link https://www.swoole.com/aot/
* @contact service@swoole.com
*/
namespace TypePhp\Build;
final class EmbeddedOpcodeGenerator
{
private const string DRIVER_SOURCE = <<<'PHP'
<?php
echo PHP_VERSION, "\n";
foreach (array_slice($argv, 1) as $path) {
if (!opcache_compile_file($path)) {
fwrite(STDERR, "OPcache could not compile: {$path}\n");
exit(1);
}
}
PHP;
/** @param list<string> $extensionArguments */
public function __construct(
private readonly string $buildDirectory,
private readonly string $targetName,
private readonly string $php,
private readonly array $extensionArguments,
private readonly string $phpVersion,
private readonly string $buildSignature,
private readonly bool $force,
) {
}
/**
* Resolve persistent cache hits and produce an immutable compilation batch.
*
* @param list<string> $files
* @param array<string, string> $anonymousKeys
*/
public function prepare(array $files, array $anonymousKeys): EmbeddedOpcodeBatch
{
$cacheDirectories = [];
$vendorGroups = [];
$vendorFiles = [];
$anonymousFiles = [];
$otherFiles = [];
foreach ($files as $file) {
if (isset($anonymousKeys[$file])) {
$cacheDirectory = $this->anonymousCacheDirectory($file);
if ($this->force) {
$this->clearDirectory($cacheDirectory);
}
$cacheDirectories[$file] = $cacheDirectory;
$anonymousFiles[$file] = true;
continue;
}
$vendorRoot = $this->vendorRoot($file);
if ($vendorRoot === null) {
$otherFiles[] = $file;
continue;
}
$vendorGroups[$vendorRoot][] = $file;
$vendorFiles[$file] = true;
}
foreach ($vendorGroups as $vendorRoot => $vendorGroup) {
$cacheDirectory = $this->vendorCacheDirectory($vendorRoot);
if ($this->force) {
$this->clearDirectory($cacheDirectory);
}
foreach ($vendorGroup as $file) {
$cacheDirectories[$file] = $cacheDirectory;
}
}
if ($otherFiles !== []) {
// Ordinary directories have no reliable invalidation boundary.
// Recreate their shared staging directory for every build.
$cacheDirectory = $this->buildDirectory . '/cache/opcache/nonvendor-'
. substr(hash('sha256', $this->targetName), 0, 20);
$this->clearDirectory($cacheDirectory);
foreach ($otherFiles as $file) {
$cacheDirectories[$file] = $cacheDirectory;
}
}
$this->writeDriver();
return $this->resolve($files, $cacheDirectories, $vendorFiles, $anonymousFiles);
}
/**
* Compile only the cache misses from a prepared batch.
*
* @param callable(string): void|null $onSkipped
* @param callable(int, int, string): void|null $onProgress
* @return array<string, string> source file => OPcache blob
*/
public function compile(
EmbeddedOpcodeBatch $batch,
?callable $onSkipped = null,
?callable $onProgress = null,
): array {
$blobs = $batch->cachedBlobs();
$total = $batch->pendingCount();
foreach ($batch->pendingFiles() as $index => $file) {
$cacheDirectory = $batch->cacheDirectory($file);
$this->ensureDirectory($cacheDirectory);
[$status, $stdout, $stderr] = $this->compileFile($file, $cacheDirectory);
if ($status !== 0) {
if ($batch->isAnonymousFile($file)
|| !str_contains($stderr, "OPcache could not compile: {$file}")) {
throw new \RuntimeException("Cannot compile embedded opcode {$file}:\n{$stderr}");
}
if ($batch->isVendorFile($file)) {
AtomicFile::write($this->skippedMarker($cacheDirectory, $file), $file . PHP_EOL, '.skip-');
}
if ($onSkipped !== null) {
$onSkipped($file);
}
} else {
// Compiled files may emit PHP deprecation notices after the
// version line printed by the driver.
if (strtok($stdout, "\r\n") !== $this->phpVersion) {
throw new \RuntimeException("Build PHP version changed while compiling {$file}");
}
$blob = $this->findBlob($cacheDirectory, $file);
if ($blob === null) {
throw new \RuntimeException("Missing OPcache blob for {$file}");
}
$blobs[$file] = $blob;
}
if ($onProgress !== null) {
$onProgress($index + 1, $total, $file);
}
}
uksort($blobs, static fn (string $left, string $right): int => strcmp($left, $right));
return $blobs;
}
/**
* @param list<string> $files
* @param array<string, string> $cacheDirectories
* @param array<string, true> $vendorFiles
* @param array<string, true> $anonymousFiles
*/
private function resolve(
array $files,
array $cacheDirectories,
array $vendorFiles,
array $anonymousFiles,
): EmbeddedOpcodeBatch {
$cachedBlobs = [];
$pendingFiles = [];
$skippedVendorFiles = [];
foreach ($files as $file) {
$cacheDirectory = $cacheDirectories[$file];
if (isset($vendorFiles[$file])
&& is_file($this->skippedMarker($cacheDirectory, $file))) {
$skippedVendorFiles[$file] = true;
continue;
}
if ((isset($vendorFiles[$file]) || isset($anonymousFiles[$file]))
&& ($blob = $this->findBlob($cacheDirectory, $file)) !== null) {
$cachedBlobs[$file] = $blob;
continue;
}
$pendingFiles[] = $file;
}
return new EmbeddedOpcodeBatch(
$cachedBlobs,
$pendingFiles,
$cacheDirectories,
$vendorFiles,
$anonymousFiles,
$skippedVendorFiles,
);
}
private function anonymousCacheDirectory(string $file): string
{
$fileHash = hash_file('sha256', $file);
if (!is_string($fileHash)) {
throw new \RuntimeException("Cannot hash anonymous class source: {$file}");
}
$key = hash('sha256', implode("\n", [
'anonymous-opcodes-v1',
$file,
$fileHash,
$this->buildSignature,
]));
return $this->buildDirectory . '/cache/opcache/anonymous-' . substr($key, 0, 20);
}
private function vendorCacheDirectory(string $vendorRoot): string
{
clearstatcache(true, $vendorRoot);
$mtime = filemtime($vendorRoot);
if ($mtime === false) {
throw new \RuntimeException("Cannot read vendor directory mtime: {$vendorRoot}");
}
$key = hash('sha256', implode("\n", [
'vendor-directory-mtime-opcodes-v2',
$vendorRoot,
(string) $mtime,
$this->buildSignature,
]));
return $this->buildDirectory . '/cache/opcache/vendor-' . substr($key, 0, 20);
}
/** Only files under a vendor directory with autoload.php use the persistent cache. */
private function vendorRoot(string $file): ?string
{
for ($directory = dirname($file); $directory !== dirname($directory); $directory = dirname($directory)) {
if (basename($directory) === 'vendor' && is_file($directory . '/autoload.php')) {
return $directory;
}
}
return null;
}
/** @return array{int, string, string} */
private function compileFile(string $file, string $cacheDirectory): array
{
$command = [
$this->php,
'-n',
...$this->extensionArguments,
'-d',
'opcache.enable_cli=1',
'-d',
'opcache.file_cache_only=1',
'-d',
'opcache.file_update_protection=0',
'-d',
'opcache.file_cache=' . $cacheDirectory,
$this->driverPath(),
$file,
];
$process = proc_open($command, [
0 => ['pipe', 'r'],
1 => ['pipe', 'w'],
2 => ['pipe', 'w'],
], $pipes);
if (!is_resource($process)) {
throw new \RuntimeException('Cannot start the PHP OPcache build process');
}
fclose($pipes[0]);
$stdout = stream_get_contents($pipes[1]);
$stderr = stream_get_contents($pipes[2]);
fclose($pipes[1]);
fclose($pipes[2]);
return [proc_close($process), (string) $stdout, (string) $stderr];
}
private function writeDriver(): void
{
$path = $this->driverPath();
if (is_file($path) && file_get_contents($path) === self::DRIVER_SOURCE) {
return;
}
AtomicFile::write($path, self::DRIVER_SOURCE, '.driver-');
}
private function driverPath(): string
{
return $this->buildDirectory . '/cache/opcache/compile-embedded-opcodes.php';
}
private function skippedMarker(string $cacheDirectory, string $file): string
{
return $cacheDirectory . '/skipped-' . hash('sha256', $file) . '.txt';
}
private function findBlob(string $cacheDirectory, string $file): ?string
{
$matches = [];
foreach (glob($cacheDirectory . '/*', GLOB_ONLYDIR) ?: [] as $systemDirectory) {
$candidate = $systemDirectory . $file . '.bin';
if (is_file($candidate) && filesize($candidate) > 0) {
$matches[] = $candidate;
}
}
if (count($matches) > 1) {
throw new \RuntimeException("Ambiguous OPcache blob for {$file}");
}
if ($matches !== []) {
return $matches[0];
}
if (DIRECTORY_SEPARATOR !== '\\' || !is_dir($cacheDirectory)) {
return null;
}
// OPcache omits the drive colon from Windows cache paths (C:\foo
// becomes C\foo) after two system-specific cache directories.
$cacheSuffix = str_replace('/', '\\', preg_replace('/^([A-Za-z]):/', '$1', $file)) . '.bin';
foreach (glob($cacheDirectory . '/*', GLOB_ONLYDIR) ?: [] as $systemDirectory) {
foreach (glob($systemDirectory . '/*', GLOB_ONLYDIR) ?: [] as $subDirectory) {
$candidate = $subDirectory . DIRECTORY_SEPARATOR . $cacheSuffix;
if (is_file($candidate) && filesize($candidate) > 0) {
$matches[] = $candidate;
}
}
}
if (count($matches) > 1) {
throw new \RuntimeException("Ambiguous OPcache blob for {$file}");
}
if ($matches !== []) {
return $matches[0];
}
// Retain compatibility with PHP builds that use another number of
// system-specific directories.
$iterator = new \RecursiveIteratorIterator(
new \RecursiveDirectoryIterator($cacheDirectory, \FilesystemIterator::SKIP_DOTS),
);
foreach ($iterator as $cached) {
if ($cached->isFile() && $cached->getSize() > 0
&& str_ends_with($cached->getPathname(), $cacheSuffix)) {
if ($matches !== []) {
throw new \RuntimeException("Ambiguous OPcache blob for {$file}");
}
$matches[] = $cached->getPathname();
}
}
return $matches[0] ?? null;
}
private function clearDirectory(string $directory): void
{
if (!is_dir($directory)) {
return;
}
$iterator = new \RecursiveIteratorIterator(
new \RecursiveDirectoryIterator($directory, \FilesystemIterator::SKIP_DOTS),
\RecursiveIteratorIterator::CHILD_FIRST,
);
foreach ($iterator as $entry) {
$path = $entry->getPathname();
if ($entry->isDir() ? !rmdir($path) : !unlink($path)) {
throw new \RuntimeException("Cannot clear opcode build directory: {$path}");
}
}
if (!rmdir($directory)) {
throw new \RuntimeException("Cannot clear opcode build directory: {$directory}");
}
}
private function ensureDirectory(string $directory): void
{
if (!is_dir($directory) && !mkdir($directory, 0777, true) && !is_dir($directory)) {
throw new \RuntimeException("Cannot create OPcache build directory: {$directory}");
}
}
}

365
src/Build/SourcePipelineTrait.php
Unescape View File

@ -301,98 +301,27 @@ trait SourcePipelineTrait
}
}
/** Only files under a vendor directory with autoload.php use the mtime cache. */
private function vendorRootForOpcodeFile(string $file): ?string
/** Embed OPcache's file-cache bytes for scripts left to ZendVM. */
private function genEmbeddedOpcodeTable(): array
{
for ($directory = dirname($file); $directory !== dirname($directory); $directory = dirname($directory)) {
if (basename($directory) === 'vendor' && is_file($directory . '/autoload.php')) {
return $directory;
}
}
return null;
}
$output = $this->getBuildDir() . '/embedded-opcodes-' . $this->targetName . '.cc';
$this->migrateEmbeddedOpcodeCaches();
$files = array_values(array_unique($this->embeddedOpcodeFiles));
sort($files, SORT_STRING);
private function findOpcodeBlob(string $cacheDir, string $file): ?string
{
$matches = [];
foreach (glob($cacheDir . '/*', GLOB_ONLYDIR) ?: [] as $systemDirectory) {
$candidate = $systemDirectory . $file . '.bin';
if (is_file($candidate) && filesize($candidate) > 0) {
$matches[] = $candidate;
}
}
if (count($matches) > 1) {
throw new \RuntimeException("Ambiguous OPcache blob for {$file}");
}
if ($matches !== []) {
return $matches[0];
}
// OPcache uses a different file-cache path layout on Windows.
if (DIRECTORY_SEPARATOR === '\\' && is_dir($cacheDir)) {
// The drive colon is omitted from OPcache's cache path (C:\foo
// becomes C\foo), after two system-specific cache directories.
$windowsCacheSuffix = str_replace('/', '\\', preg_replace('/^([A-Za-z]):/', '$1', $file)) . '.bin';
foreach (glob($cacheDir . '/*', GLOB_ONLYDIR) ?: [] as $systemDirectory) {
foreach (glob($systemDirectory . '/*', GLOB_ONLYDIR) ?: [] as $subDirectory) {
$candidate = $subDirectory . DIRECTORY_SEPARATOR . $windowsCacheSuffix;
if (is_file($candidate) && filesize($candidate) > 0) {
$matches[] = $candidate;
}
}
}
if (count($matches) > 1) {
throw new \RuntimeException("Ambiguous OPcache blob for {$file}");
}
if ($matches !== []) {
return $matches[0];
}
// Keep supporting other OPcache layouts without assuming the same
// number of system directories in every PHP build.
$iterator = new \RecursiveIteratorIterator(
new \RecursiveDirectoryIterator($cacheDir, \FilesystemIterator::SKIP_DOTS),
);
foreach ($iterator as $cached) {
if ($cached->isFile() && $cached->getSize() > 0
&& str_ends_with($cached->getPathname(), $windowsCacheSuffix)) {
if ($matches !== []) {
throw new \RuntimeException("Ambiguous OPcache blob for {$file}");
}
$matches[] = $cached->getPathname();
}
}
$blobs = [];
$phpVersion = PHP_VERSION;
if ($files !== []) {
['blobs' => $blobs, 'phpVersion' => $phpVersion] = $this->generateEmbeddedOpcodeBlobs($files);
}
return $matches[0] ?? null;
}
private function getSkippedOpcodeMarker(string $cacheDir, string $file): string
{
return $cacheDir . '/skipped-' . hash('sha256', $file) . '.txt';
}
private function clearOpcodeCacheDirectory(string $directory): void
{
if (!is_dir($directory)) {
return;
}
$iterator = new \RecursiveIteratorIterator(
new \RecursiveDirectoryIterator($directory, \FilesystemIterator::SKIP_DOTS),
\RecursiveIteratorIterator::CHILD_FIRST,
);
foreach ($iterator as $entry) {
$path = $entry->getPathname();
if ($entry->isDir() ? !rmdir($path) : !unlink($path)) {
throw new \RuntimeException("Cannot clear opcode build directory: {$path}");
}
}
if (!rmdir($directory)) {
throw new \RuntimeException("Cannot clear opcode build directory: {$directory}");
}
$sources = $this->emitEmbeddedArchiveSources($output, $files, $blobs, $phpVersion);
$this->removeLegacyAnonymousArtifacts();
return $sources;
}
/** Embed OPcache's file-cache bytes for scripts left to ZendVM. */
private function genEmbeddedOpcodeTable(): array
private function migrateEmbeddedOpcodeCaches(): void
{
$output = $this->getBuildDir() . '/embedded-opcodes-' . $this->targetName . '.cc';
foreach (glob($this->getBuildDir() . '/opcache-*') ?: [] as $legacyCache) {
if (is_dir($legacyCache)) {
$this->moveLegacyBuildCache(
@ -401,195 +330,85 @@ trait SourcePipelineTrait
);
}
}
$files = array_values(array_unique($this->embeddedOpcodeFiles));
sort($files, SORT_STRING);
$blobs = [];
$phpVersion = PHP_VERSION;
$this->moveLegacyBuildCache(
$this->getBuildDir() . '/compile-embedded-opcodes.php',
$this->getBuildDir() . '/cache/opcache/compile-embedded-opcodes.php',
);
}
if ($files !== []) {
if ($this->isIosTarget() || $this->isAndroidTarget() || $this->isWasiTarget()) {
throw new \RuntimeException('Embedded opcodes require a native PHP build host matching the target');
}
$this->output('Generating embedded opcodes for ' . count($files) . ' PHP files', 'lightBlue');
$php = $this->getOpcodeBuildPhpCli();
$extensionArgs = $this->getOpcodeBuildExtensionArgs();
$phpVersion = $this->opcodeBuildPhpVersion;
$otherFiles = [];
$vendorRoots = [];
$vendorFiles = [];
$anonymousFiles = [];
$fileCacheDirs = [];
foreach ($files as $file) {
if (isset($this->anonymousOpcodeKeys[$file])) {
// These files are emitted by this compiler, so their exact
// contents provide a safe cache key independent of any
// user-supplied directory timestamp.
$key = hash('sha256', implode("\n", [
'anonymous-opcodes-v1', $file,
hash_file('sha256', $file), $this->opcodeBuildSignature,
]));
$cacheDir = $this->getBuildDir() . '/cache/opcache/anonymous-' . substr($key, 0, 20);
if ($this->climate->arguments->defined('force')) {
$this->clearOpcodeCacheDirectory($cacheDir);
}
$fileCacheDirs[$file] = $cacheDir;
$anonymousFiles[$file] = true;
continue;
}
$vendorRoot = $this->vendorRootForOpcodeFile($file);
if ($vendorRoot === null) {
$otherFiles[] = $file;
continue;
}
$vendorRoots[$vendorRoot][] = $file;
$vendorFiles[$file] = true;
}
foreach ($vendorRoots as $vendorRoot => $filesInVendor) {
clearstatcache(true, $vendorRoot);
$mtime = filemtime($vendorRoot);
if ($mtime === false) {
throw new \RuntimeException("Cannot read vendor directory mtime: {$vendorRoot}");
}
$key = hash('sha256', implode("\n", [
'vendor-directory-mtime-opcodes-v2', $vendorRoot,
(string) $mtime, $this->opcodeBuildSignature,
]));
$cacheDir = $this->getBuildDir() . '/cache/opcache/vendor-' . substr($key, 0, 20);
if ($this->climate->arguments->defined('force')) {
$this->clearOpcodeCacheDirectory($cacheDir);
}
foreach ($filesInVendor as $file) {
$fileCacheDirs[$file] = $cacheDir;
}
}
if ($otherFiles !== []) {
// Recreate this staging directory on every build so OPcache
// cannot reuse bytecode for non-vendor or anonymous sources.
$cacheDir = $this->getBuildDir() . '/cache/opcache/nonvendor-'
. substr(hash('sha256', $this->targetName), 0, 20);
$this->clearOpcodeCacheDirectory($cacheDir);
foreach ($otherFiles as $file) {
$fileCacheDirs[$file] = $cacheDir;
}
}
$driver = $this->getBuildDir() . '/cache/opcache/compile-embedded-opcodes.php';
$this->moveLegacyBuildCache(
$this->getBuildDir() . '/compile-embedded-opcodes.php',
$driver,
);
$this->writeFile($driver, <<<'PHP'
<?php
echo PHP_VERSION, "\n";
foreach (array_slice($argv, 1) as $path) {
if (!opcache_compile_file($path)) {
fwrite(STDERR, "OPcache could not compile: {$path}\n");
exit(1);
/** @param list<string> $files @return array{blobs: array<string, string>, phpVersion: string} */
private function generateEmbeddedOpcodeBlobs(array $files): array
{
if ($this->isIosTarget() || $this->isAndroidTarget() || $this->isWasiTarget()) {
throw new \RuntimeException('Embedded opcodes require a native PHP build host matching the target');
}
$this->output('Generating embedded opcodes for ' . count($files) . ' PHP files', 'lightBlue');
$generator = new EmbeddedOpcodeGenerator(
$this->getBuildDir(),
$this->targetName,
$this->getOpcodeBuildPhpCli(),
$this->getOpcodeBuildExtensionArgs(),
$this->opcodeBuildPhpVersion,
$this->opcodeBuildSignature,
$this->climate->arguments->defined('force'),
);
$batch = $generator->prepare($files, $this->anonymousOpcodeKeys);
$this->reportEmbeddedOpcodeCache($batch);
$progress = $this->startOpcodeProgress('Opcodes', $batch->pendingCount());
$blobs = $generator->compile(
$batch,
fn(string $file) => $this->climate->warning(
'Skipping non-executable embedded PHP file: ' . $file,
),
fn(int $completed, int $total, string $file) => $this->updateOpcodeProgress(
$progress,
'Opcodes',
$completed,
$total,
$this->noProgress ? $file : null,
),
);
if ($progress !== null) {
echo PHP_EOL;
}
return ['blobs' => $blobs, 'phpVersion' => $this->opcodeBuildPhpVersion];
}
}
PHP
private function reportEmbeddedOpcodeCache(EmbeddedOpcodeBatch $batch): void
{
if ($batch->vendorCount() !== 0) {
$vendorHits = $batch->vendorHits();
$vendorSkips = $batch->skippedVendorCount();
$this->output(
'Vendor opcode cache: ' . $vendorHits . ' reused, '
. ($batch->vendorCount() - $vendorHits - $vendorSkips) . ' to generate'
. ($vendorSkips === 0 ? '' : ', ' . $vendorSkips . ' skipped'),
'lightBlue',
);
$command = [
$php, '-n', ...$extensionArgs,
'-d', 'opcache.enable_cli=1',
'-d', 'opcache.file_cache_only=1',
'-d', 'opcache.file_update_protection=0',
];
// OPcache installs declarations in the compiling CLI process.
// Isolate files so unrelated scripts with the same global
// function names do not conflict while creating their bytecode.
$pending = [];
$vendorHits = 0;
$anonymousHits = 0;
$vendorSkips = 0;
foreach ($files as $file) {
$cacheDir = $fileCacheDirs[$file];
if (isset($vendorFiles[$file])
&& is_file($this->getSkippedOpcodeMarker($cacheDir, $file))) {
++$vendorSkips;
continue;
}
if ((isset($vendorFiles[$file]) || isset($anonymousFiles[$file]))
&& ($blob = $this->findOpcodeBlob($cacheDir, $file)) !== null) {
$blobs[$file] = $blob;
if (isset($anonymousFiles[$file])) {
++$anonymousHits;
} else {
++$vendorHits;
}
} else {
$pending[] = $file;
}
}
if ($vendorRoots !== []) {
$vendorCount = array_sum(array_map('count', $vendorRoots));
$this->output(
'Vendor opcode cache: ' . $vendorHits . ' reused, '
. ($vendorCount - $vendorHits - $vendorSkips) . ' to generate'
. ($vendorSkips === 0 ? '' : ', ' . $vendorSkips . ' skipped'),
'lightBlue',
);
}
if ($anonymousFiles !== []) {
$this->output(
'Anonymous opcode cache: ' . $anonymousHits . ' reused, '
. (count($anonymousFiles) - $anonymousHits) . ' to generate',
'lightBlue',
);
}
$progress = $this->startOpcodeProgress('Opcodes', count($pending));
$completed = 0;
foreach ($pending as $file) {
$cacheDir = $fileCacheDirs[$file];
if (!is_dir($cacheDir) && !mkdir($cacheDir, 0777, true) && !is_dir($cacheDir)) {
throw new \RuntimeException("Cannot create OPcache build directory: {$cacheDir}");
}
$process = proc_open([...$command, '-d', 'opcache.file_cache=' . $cacheDir, $driver, $file], [
0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w'],
], $pipes);
if (!is_resource($process)) {
throw new \RuntimeException('Cannot start the PHP OPcache build process');
}
fclose($pipes[0]);
$stdout = stream_get_contents($pipes[1]);
$stderr = stream_get_contents($pipes[2]);
fclose($pipes[1]);
fclose($pipes[2]);
if (proc_close($process) !== 0) {
if (isset($anonymousFiles[$file])
|| !str_contains((string) $stderr, "OPcache could not compile: {$file}")) {
throw new \RuntimeException("Cannot compile embedded opcode {$file}:\n{$stderr}");
}
// Embedded directories can contain PHP-looking declaration
// files that are not scripts. Keep their raw bytes but do
// not treat them as executable bytecode.
$this->climate->warning('Skipping non-executable embedded PHP file: ' . $file);
if (isset($vendorFiles[$file])) {
$marker = $this->getSkippedOpcodeMarker($cacheDir, $file);
if (file_put_contents($marker, $file . PHP_EOL) === false) {
throw new \RuntimeException("Cannot cache skipped embedded PHP file: {$file}");
}
}
$this->updateOpcodeProgress($progress, 'Opcodes', ++$completed, count($pending), $this->noProgress ? $file : null);
continue;
}
// Compiled files may emit PHP deprecation notices on stdout.
// The driver prints its version before compiling the file.
if (strtok((string) $stdout, "\r\n") !== $phpVersion) {
throw new \RuntimeException("Build PHP version changed while compiling {$file}");
}
$blob = $this->findOpcodeBlob($cacheDir, $file);
if ($blob === null) {
throw new \RuntimeException("Missing OPcache blob for {$file}");
}
$blobs[$file] = $blob;
$this->updateOpcodeProgress($progress, 'Opcodes', ++$completed, count($pending), $this->noProgress ? $file : null);
}
if ($progress !== null) {
echo PHP_EOL;
}
uksort($blobs, static fn(string $left, string $right): int => strcmp($left, $right));
}
if ($batch->anonymousCount() !== 0) {
$anonymousHits = $batch->anonymousHits();
$this->output(
'Anonymous opcode cache: ' . $anonymousHits . ' reused, '
. ($batch->anonymousCount() - $anonymousHits) . ' to generate',
'lightBlue',
);
}
}
/**
* @param list<string> $opcodeFiles
* @param array<string, string> $blobs
* @return list<string>
*/
private function emitEmbeddedArchiveSources(
string $output,
array $opcodeFiles,
array $blobs,
string $phpVersion,
): array {
$archive = $this->getBuildDir() . '/cache/embedded/embedded-files-' . $this->targetName . '.bin';
$this->moveLegacyBuildCache(
$this->getBuildDir() . '/embedded-files-' . $this->targetName . '.bin',
@ -597,7 +416,7 @@ PHP
);
$sources = [$output];
$embeddedArchive = EmbeddedArchive::empty($archive);
if ($this->embeddedFiles !== [] || $files !== []) {
if ($this->embeddedFiles !== [] || $opcodeFiles !== []) {
$embeddedArchive = (new EmbeddedArchiveBuilder())->build(
$archive,
$this->embeddedFiles,
@ -629,13 +448,17 @@ PHP
(new EmbeddedTableRenderer())->render($embeddedArchive, $phpVersion, $this->isWindows()),
);
$this->generatedProjectSources[$output] = true;
return $sources;
}
private function removeLegacyAnonymousArtifacts(): void
{
foreach (glob($this->getBuildDir() . '/anonymous-*.json') ?: [] as $legacyManifest) {
unlink($legacyManifest);
}
foreach (glob($this->getBuildDir() . '/anonymous-' . $this->targetName . '-*.php') ?: [] as $legacySource) {
unlink($legacySource);
}
return $sources;
}
public function getFiles(string $path): array

Write Preview
Loading…
Cancel
Save