diff --git a/phpunit/src/Build/EmbeddedArchiveBuilderTest.php b/phpunit/src/Build/EmbeddedArchiveBuilderTest.php new file mode 100644 index 00000000..6dd350f0 --- /dev/null +++ b/phpunit/src/Build/EmbeddedArchiveBuilderTest.php @@ -0,0 +1,105 @@ +directory = sys_get_temp_dir() . '/typephp-embedded-archive-' . bin2hex(random_bytes(6)); + mkdir($this->directory, 0777, true); + } + + protected function tearDown(): void + { + $iterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($this->directory, \FilesystemIterator::SKIP_DOTS), + \RecursiveIteratorIterator::CHILD_FIRST, + ); + foreach ($iterator as $entry) { + $entry->isDir() ? rmdir($entry->getPathname()) : unlink($entry->getPathname()); + } + rmdir($this->directory); + } + + public function testFilesAndOpcodesShareOneDeterministicArchive(): void + { + $first = $this->write('first.txt', 'alpha'); + $second = $this->write('second.bin', "\0beta"); + $source = $this->write('source.php', 'write('source.bin', 'opcode'); + $archivePath = $this->directory . '/cache/archive.bin'; + + $archive = (new EmbeddedArchiveBuilder())->build( + $archivePath, + [$first, $second], + [$source => $blob], + [$source => '@typephp:anon:test'], + ); + + self::assertSame("alpha\0betaopcode", file_get_contents($archivePath)); + self::assertSame([$first => [0, 5], $second => [5, 5]], $archive->fileIndex); + self::assertSame(['@typephp:anon:test' => [10, 6]], $archive->opcodeIndex); + self::assertSame(hash_file('sha256', $archivePath), $archive->hash); + + file_put_contents($blob, 'new-opcode'); + $updated = (new EmbeddedArchiveBuilder())->build( + $archivePath, + [$first, $second], + [$source => $blob], + [$source => '@typephp:anon:test'], + ); + self::assertSame("alpha\0betanew-opcode", file_get_contents($archivePath)); + self::assertSame(['@typephp:anon:test' => [10, 10]], $updated->opcodeIndex); + } + + public function testTableRendererKeepsPlatformSpecificArchiveAccessSymmetric(): void + { + $archive = new EmbeddedArchive( + '/tmp/archive.bin', + 'hash', + ['/app/config.json' => [0, 4]], + ['@typephp:anon:test' => [4, 8]], + ); + $renderer = new EmbeddedTableRenderer(); + + $windows = $renderer->render($archive, '8.4.14', true); + self::assertStringContainsString( + 'typephp_embedded_archive_start = typephp_embedded_archive_data()', + $windows, + ); + self::assertStringContainsString('{"@typephp:anon:test", typephp_embedded_archive_start + 4, 8}', $windows); + self::assertStringContainsString('*count = 1; return typephp_opcodes;', $windows); + + $unix = $renderer->render($archive, '8.4.14', false); + self::assertStringContainsString( + 'extern "C" const uint8_t typephp_embedded_archive_start[];', + $unix, + ); + self::assertStringNotContainsString('typephp_opcode_table_install(void) {}', $unix); + } + + private function write(string $name, string $contents): string + { + $path = $this->directory . '/' . $name; + file_put_contents($path, $contents); + return $path; + } +} diff --git a/src/Build/EmbeddedArchive.php b/src/Build/EmbeddedArchive.php new file mode 100644 index 00000000..bd6af1ee --- /dev/null +++ b/src/Build/EmbeddedArchive.php @@ -0,0 +1,34 @@ + $fileIndex + * @param array $opcodeIndex + */ + public function __construct( + public string $path, + public string $hash, + public array $fileIndex, + public array $opcodeIndex, + ) { + } + + public static function empty(string $path): self + { + return new self($path, '', [], []); + } + + public function isEmpty(): bool + { + return $this->fileIndex === [] && $this->opcodeIndex === []; + } +} diff --git a/src/Build/EmbeddedArchiveBuilder.php b/src/Build/EmbeddedArchiveBuilder.php new file mode 100644 index 00000000..5204ca0a --- /dev/null +++ b/src/Build/EmbeddedArchiveBuilder.php @@ -0,0 +1,104 @@ + $files + * @param array $opcodeBlobs source file => blob file + * @param array $opcodeKeys source file => runtime lookup key + */ + public function build( + string $archivePath, + array $files, + array $opcodeBlobs, + array $opcodeKeys = [], + ): EmbeddedArchive { + $this->ensureDirectory(dirname($archivePath)); + $temporaryPath = $archivePath . '.tmp'; + $stream = fopen($temporaryPath, 'wb'); + if ($stream === false) { + throw new \RuntimeException("Cannot create embedded file archive: {$temporaryPath}"); + } + + $fileIndex = []; + $opcodeIndex = []; + try { + foreach ($files as $file) { + $fileIndex[$file] = $this->append($stream, $file, 'embedded file'); + } + foreach ($opcodeBlobs as $sourceFile => $blobFile) { + $key = $opcodeKeys[$sourceFile] ?? $sourceFile; + $opcodeIndex[$key] = $this->append($stream, $blobFile, 'opcode blob'); + } + } catch (\Throwable $error) { + fclose($stream); + @unlink($temporaryPath); + throw $error; + } + + if (!fclose($stream)) { + @unlink($temporaryPath); + throw new \RuntimeException("Cannot close embedded file archive: {$temporaryPath}"); + } + + $temporaryHash = hash_file('sha256', $temporaryPath); + if (!is_string($temporaryHash)) { + @unlink($temporaryPath); + throw new \RuntimeException("Cannot hash embedded file archive: {$temporaryPath}"); + } + $currentHash = is_file($archivePath) ? hash_file('sha256', $archivePath) : false; + if ($currentHash === $temporaryHash) { + if (!unlink($temporaryPath)) { + throw new \RuntimeException("Cannot remove unchanged embedded archive: {$temporaryPath}"); + } + } else { + if (is_file($archivePath) && !unlink($archivePath)) { + @unlink($temporaryPath); + throw new \RuntimeException("Cannot replace embedded file archive: {$archivePath}"); + } + if (!rename($temporaryPath, $archivePath)) { + @unlink($temporaryPath); + throw new \RuntimeException("Cannot install embedded file archive: {$archivePath}"); + } + } + + return new EmbeddedArchive($archivePath, $temporaryHash, $fileIndex, $opcodeIndex); + } + + /** @return array{int, int} */ + private function append(mixed $output, string $inputPath, string $description): array + { + $offset = ftell($output); + if (!is_int($offset)) { + throw new \RuntimeException('Cannot determine embedded archive offset'); + } + $input = fopen($inputPath, 'rb'); + if ($input === false) { + throw new \RuntimeException("Cannot read {$description}: {$inputPath}"); + } + try { + $length = stream_copy_to_stream($input, $output); + } finally { + fclose($input); + } + if (!is_int($length)) { + throw new \RuntimeException("Cannot copy {$description}: {$inputPath}"); + } + return [$offset, $length]; + } + + private function ensureDirectory(string $directory): void + { + if (!is_dir($directory) && !mkdir($directory, 0777, true) && !is_dir($directory)) { + throw new \RuntimeException("Cannot create embedded archive directory: {$directory}"); + } + } +} diff --git a/src/Build/EmbeddedTableRenderer.php b/src/Build/EmbeddedTableRenderer.php new file mode 100644 index 00000000..8b497697 --- /dev/null +++ b/src/Build/EmbeddedTableRenderer.php @@ -0,0 +1,58 @@ +' . PHP_EOL; + if (!$archive->isEmpty()) { + $code .= $windows + ? 'static const uint8_t *const typephp_embedded_archive_start = typephp_embedded_archive_data();' . PHP_EOL + : 'extern "C" const uint8_t typephp_embedded_archive_start[];' . PHP_EOL; + } + $code .= $this->renderTable( + 'typephp_opcode_entry', + 'typephp_opcodes', + 'typephp_project_opcode_table', + $archive->opcodeIndex, + ); + $code .= $this->renderTable( + 'typephp_embedded_file_entry', + 'typephp_embedded_files', + 'typephp_project_embedded_file_table', + $archive->fileIndex, + ); + $version = json_encode($phpVersion, JSON_THROW_ON_ERROR); + $code .= 'extern "C" const char *typephp_project_php_version(void) { return ' . $version . '; }' . PHP_EOL; + if ($archive->isEmpty()) { + $code .= 'extern "C" void typephp_opcode_table_install(void) {}' . PHP_EOL; + $code .= 'extern "C" void typephp_opcode_table_uninstall(void) {}' . PHP_EOL; + } + return $code; + } + + /** @param array $index */ + private function renderTable(string $type, string $name, string $accessor, array $index): string + { + $code = "static const {$type} {$name}[] = {" . PHP_EOL; + foreach ($index as $path => [$offset, $length]) { + $encodedPath = json_encode( + $path, + JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR, + ); + $code .= " {{$encodedPath}, typephp_embedded_archive_start + {$offset}, {$length}}," . PHP_EOL; + } + $code .= ' {nullptr, nullptr, 0},' . PHP_EOL . '};' . PHP_EOL; + $code .= "extern \"C\" const {$type} *{$accessor}(size_t *count) {" . PHP_EOL; + $code .= ' *count = ' . count($index) . "; return {$name};" . PHP_EOL . '}' . PHP_EOL; + return $code; + } +} diff --git a/src/Build/SourcePipelineTrait.php b/src/Build/SourcePipelineTrait.php index be54833c..2957a9c1 100644 --- a/src/Build/SourcePipelineTrait.php +++ b/src/Build/SourcePipelineTrait.php @@ -24,10 +24,10 @@ trait SourcePipelineTrait use PreparedProjectCacheTrait; /** @var list PHP files selected by embedded-files. */ - private array $bundledPhpFiles = []; + private array $embeddedPhpFiles = []; /** @var list All regular files selected by embedded-files. */ - private array $bundledFiles = []; + private array $embeddedFiles = []; /** @var list Selected PHP files not translated to native code. */ private array $embeddedOpcodeFiles = []; @@ -595,52 +595,15 @@ PHP $this->getBuildDir() . '/embedded-files-' . $this->targetName . '.bin', $archive, ); - $rawIndex = []; - $opcodeIndex = []; $sources = [$output]; - if ($this->bundledFiles !== [] || $files !== []) { - $temporaryArchive = $archive . '.tmp'; - if (!is_dir(dirname($archive)) - && !mkdir(dirname($archive), 0777, true) && !is_dir(dirname($archive))) { - throw new \RuntimeException('Cannot create embedded archive directory: ' . dirname($archive)); - } - $stream = fopen($temporaryArchive, 'wb'); - if ($stream === false) { - throw new \RuntimeException("Cannot create embedded file archive: {$temporaryArchive}"); - } - foreach ($this->bundledFiles as $file) { - $offset = ftell($stream); - $input = fopen($file, 'rb'); - if ($input === false) { - throw new \RuntimeException("Cannot read embedded file: {$file}"); - } - $length = stream_copy_to_stream($input, $stream); - fclose($input); - if ($length === false) { - throw new \RuntimeException("Cannot copy embedded file: {$file}"); - } - $rawIndex[$file] = [$offset, $length]; - } - foreach ($blobs as $file => $blobFile) { - $offset = ftell($stream); - $input = fopen($blobFile, 'rb'); - if ($input === false) { - throw new \RuntimeException("Cannot read opcode blob: {$blobFile}"); - } - $length = stream_copy_to_stream($input, $stream); - fclose($input); - if ($length === false) { - throw new \RuntimeException("Cannot copy opcode blob: {$blobFile}"); - } - $opcodeIndex[$this->anonymousOpcodeKeys[$file] ?? $file] = [$offset, $length]; - } - fclose($stream); - if (!is_file($archive) || hash_file('sha256', $archive) !== hash_file('sha256', $temporaryArchive)) { - rename($temporaryArchive, $archive); - } else { - unlink($temporaryArchive); - } - $archiveHash = hash_file('sha256', $archive); + $embeddedArchive = EmbeddedArchive::empty($archive); + if ($this->embeddedFiles !== [] || $files !== []) { + $embeddedArchive = (new EmbeddedArchiveBuilder())->build( + $archive, + $this->embeddedFiles, + $blobs, + $this->anonymousOpcodeKeys, + ); if ($this->isWindows()) { $this->embeddedArchiveFile = $archive; } else { @@ -649,47 +612,22 @@ PHP $section = $this->isMacos() ? '__TEXT,__const' : '.rodata'; $symbol = $this->isMacos() ? '_typephp_embedded_archive_start' : 'typephp_embedded_archive_start'; - $asmCode = "# archive-sha256: {$archiveHash}\n.section {$section}\n.globl {$symbol}\n.p2align 4\n{$symbol}:\n.incbin {$quotedArchive}\n"; + $asmCode = "# archive-sha256: {$embeddedArchive->hash}\n.section {$section}\n.globl {$symbol}\n.p2align 4\n{$symbol}:\n.incbin {$quotedArchive}\n"; $this->writeFile($assembly, $asmCode); $this->generatedProjectSources[$assembly] = true; $sources[] = $assembly; } $this->output( - 'Packed ' . count($rawIndex) . ' files and ' . count($opcodeIndex) . ' opcode blobs', + 'Packed ' . count($embeddedArchive->fileIndex) . ' files and ' + . count($embeddedArchive->opcodeIndex) . ' opcode blobs', 'green', ); } - $code = '#include ' . PHP_EOL; - $hasArchive = $this->bundledFiles !== [] || $files !== []; - if ($hasArchive && $this->isWindows()) { - $code .= 'static const uint8_t *const typephp_embedded_archive_start = typephp_embedded_archive_data();' . PHP_EOL; - } elseif ($hasArchive) { - $code .= 'extern "C" const uint8_t typephp_embedded_archive_start[];' . PHP_EOL; - } - $code .= 'static const typephp_opcode_entry typephp_opcodes[] = {' . PHP_EOL; - foreach ($opcodeIndex as $file => [$offset, $length]) { - $path = json_encode($file, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR); - $code .= " {{$path}, typephp_embedded_archive_start + {$offset}, {$length}}," . PHP_EOL; - } - $code .= ' {nullptr, nullptr, 0},' . PHP_EOL . '};' . PHP_EOL; - $code .= 'extern "C" const typephp_opcode_entry *typephp_project_opcode_table(size_t *count) {' . PHP_EOL; - $code .= ' *count = ' . count($opcodeIndex) . '; return typephp_opcodes;' . PHP_EOL . '}' . PHP_EOL; - $code .= 'static const typephp_embedded_file_entry typephp_embedded_files[] = {' . PHP_EOL; - foreach ($rawIndex as $file => [$offset, $length]) { - $path = json_encode($file, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR); - $code .= " {{$path}, typephp_embedded_archive_start + {$offset}, {$length}}," . PHP_EOL; - } - $code .= ' {nullptr, nullptr, 0},' . PHP_EOL . '};' . PHP_EOL; - $code .= 'extern "C" const typephp_embedded_file_entry *typephp_project_embedded_file_table(size_t *count) {' . PHP_EOL; - $code .= ' *count = ' . count($rawIndex) . '; return typephp_embedded_files;' . PHP_EOL . '}' . PHP_EOL; - $version = json_encode($phpVersion, JSON_THROW_ON_ERROR); - $code .= 'extern "C" const char *typephp_project_php_version(void) { return ' . $version . '; }' . PHP_EOL; - if ($rawIndex === [] && $opcodeIndex === []) { - $code .= 'extern "C" void typephp_opcode_table_install(void) {}' . PHP_EOL; - $code .= 'extern "C" void typephp_opcode_table_uninstall(void) {}' . PHP_EOL; - } - $this->writeFile($output, $code); + $this->writeFile( + $output, + (new EmbeddedTableRenderer())->render($embeddedArchive, $phpVersion, $this->isWindows()), + ); $this->generatedProjectSources[$output] = true; foreach (glob($this->getBuildDir() . '/anonymous-*.json') ?: [] as $legacyManifest) { unlink($legacyManifest); @@ -734,10 +672,10 @@ PHP // Apply command-line arguments after all configuration is loaded (so they // take the highest precedence) $this->applyCommandLineArguments(); - if ($this->bundledFiles !== [] && !$this->isBuildModeBin()) { + if ($this->embeddedFiles !== [] && !$this->isBuildModeBin()) { $this->error('`embedded-files` requires `mode: bin`'); } - if ($this->bundledFiles !== []) { + if ($this->embeddedFiles !== []) { $this->getOpcodeBuildExtensionArgs(); } if ($this->climate->arguments->defined('force')) { @@ -833,7 +771,7 @@ PHP $files = $this->filterIgnoredFiles($files); $preparedKey = $this->preparedProjectKey($files); if ($this->restorePreparedProject($preparedKey)) { - $this->embeddedOpcodeFiles = array_values(array_diff($this->bundledPhpFiles, $files)); + $this->embeddedOpcodeFiles = array_values(array_diff($this->embeddedPhpFiles, $files)); $files = $this->getSortedFiles($files); $this->initializeIncrementalCompilation($files); return $files; @@ -867,7 +805,7 @@ PHP $this->storePreparedProject($preparedKey); } $files = $this->getSortedFiles($files); - $this->embeddedOpcodeFiles = array_values(array_diff($this->bundledPhpFiles, $files)); + $this->embeddedOpcodeFiles = array_values(array_diff($this->embeddedPhpFiles, $files)); $this->initializeIncrementalCompilation($files); return $files; } @@ -1089,7 +1027,7 @@ PHP } catch (Unsupported $e) { echo ' unsupported syntax: ' . $e->getMessage() . "\n"; echo ' skip: ' . $file . "\n"; - if (in_array($file, $this->bundledPhpFiles, true) + if (in_array($file, $this->embeddedPhpFiles, true) && !in_array($file, $this->embeddedOpcodeFiles, true)) { $this->embeddedOpcodeFiles[] = $file; } @@ -1126,7 +1064,7 @@ PHP // metadata, then adds a direct native process entry beside it. $sourceFiles[] = $this->genExtension(); if ($this->isBuildModeEmbed() && !$this->isNanoMode() - && ($this->bundledFiles !== [] || $this->embeddedOpcodeFiles !== [])) { + && ($this->embeddedFiles !== [] || $this->embeddedOpcodeFiles !== [])) { array_push($sourceFiles, ...$this->genEmbeddedOpcodeTable()); } if ($this->isNanoMode()) { diff --git a/src/Translator.php b/src/Translator.php index 417baad9..76cac356 100644 --- a/src/Translator.php +++ b/src/Translator.php @@ -1235,7 +1235,7 @@ class Translator extends Preprocessor if ($this->isBuildModeEmbed() && !$this->isNanoMode()) { $code .= '#include ' . PHP_EOL; - if ($this->bundledFiles === [] && $this->embeddedOpcodeFiles === []) { + if ($this->embeddedFiles === [] && $this->embeddedOpcodeFiles === []) { // The runtime still calls these hooks; keep empty builds in this translation unit. $code .= 'extern "C" void typephp_opcode_table_install(void) {}' . PHP_EOL; $code .= 'extern "C" void typephp_opcode_table_uninstall(void) {}' . PHP_EOL; @@ -2143,7 +2143,7 @@ CODE; $ext = strtolower(pathinfo($filePath, PATHINFO_EXTENSION)); return match ($ext) { 'c' => 'c', - 's', 'S' => 'assembler', + 's' => 'assembler', 'm' => 'objective-c', 'mm' => 'objective-c++', 'cc', 'cpp', 'cxx' => null, @@ -2237,10 +2237,7 @@ CODE; /** @param array{cacheable_misc: bool, nano_runtime: bool, generated_project: bool} $task */ private function finalizeCompileFileTask(string $cppFile, string $objectFile, array $task): void { - if ($task['cacheable_misc']) { - $this->writeMiscObjectCacheMetadata($cppFile, $objectFile); - } - if ($task['nano_runtime']) { + if ($task['cacheable_misc'] || $task['nano_runtime']) { $this->writeMiscObjectCacheMetadata($cppFile, $objectFile); } if ($task['generated_project']) { @@ -2309,7 +2306,7 @@ CODE; $sourceFiles[] = $runtimeSource; } $sourceFiles[] = $this->getPhpxDir() . '/src/misc/typephp_main.cc'; - if ($this->bundledFiles !== [] || $this->embeddedOpcodeFiles !== []) { + if ($this->embeddedFiles !== [] || $this->embeddedOpcodeFiles !== []) { $sourceFiles[] = $this->getPhpxDir() . '/src/misc/typephp_opcode_table.cc'; // Select the decoder for the PHP CLI that generated the blobs. // Distribution PHP headers need not live under the PHP prefix. @@ -4065,7 +4062,7 @@ CODE; $this->error('Embedded file or directory does not exist: `' . $src . '`'); } if (is_file($resolved)) { - $this->bundledFiles[] = $resolved; + $this->embeddedFiles[] = $resolved; continue; } $iterator = new \RecursiveIteratorIterator( @@ -4073,23 +4070,23 @@ CODE; ); foreach ($iterator as $file) { if ($file->isFile()) { - $this->bundledFiles[] = $file->getPathname(); + $this->embeddedFiles[] = $file->getPathname(); } } } - $this->bundledFiles = array_values(array_unique($this->bundledFiles)); - sort($this->bundledFiles, SORT_STRING); - $this->bundledPhpFiles = array_values(array_filter( - $this->bundledFiles, + $this->embeddedFiles = array_values(array_unique($this->embeddedFiles)); + sort($this->embeddedFiles, SORT_STRING); + $this->embeddedPhpFiles = array_values(array_filter( + $this->embeddedFiles, // PHP extension stubs are API declarations for code generators. // They remain in the raw archive but must never be executed. static fn(string $file): bool => FileScanner::isPhpFile($file) && !str_ends_with($file, '.stub.php'), )); - if ($this->bundledFiles !== []) { + if ($this->embeddedFiles !== []) { $this->output( - 'embedded-files: found ' . count($this->bundledFiles) - . ' files (' . count($this->bundledPhpFiles) . ' PHP)', + 'embedded-files: found ' . count($this->embeddedFiles) + . ' files (' . count($this->embeddedPhpFiles) . ' PHP)', 'lightBlue', ); }