From 3b3711c13966d3d757d97b87b0a00a8e03ef130a Mon Sep 17 00:00:00 2001 From: tianfenghan Date: Fri, 7 Aug 2026 14:55:49 +0800 Subject: [PATCH] feat(compiler): implement std container default initialization and mutation guards - Add initialization of standard containers with default values when not boxed - Implement iteration guards for safe foreach structural mutations in std containers - Add support for compound operations on bigint, bigfloat, and decimal types - Introduce lazy offset retrieval for map containers with default values - Enhance unset operations to use proper type conversions and default values - Add parallel testing support with worker isolation and temporary directories - Implement proper container size reporting using runtime values instead of compile-time - Add comprehensive test coverage for container operations and edge cases --- run-tests.php | 115 ++++++++++++--- src/CompilerBase.php | 7 + src/Parser/PropertyAccessTrait.php | 11 +- src/Parser/StdContainerTrait.php | 136 ++++++++++++++---- .../container-default-compound.phpt | 37 +++++ .../std-map/foreach-mutation-guard.phpt | 29 ++++ tests/compiler/std-map/missing-read.phpt | 27 ++++ .../std-map/unset-key-conversion.phpt | 19 +++ .../std-vector/count-runtime-size.phpt | 17 +++ .../std-vector/foreach-alias-mutation.phpt | 28 ++++ 10 files changed, 378 insertions(+), 48 deletions(-) create mode 100644 tests/compiler/std-bigint/container-default-compound.phpt create mode 100644 tests/compiler/std-map/foreach-mutation-guard.phpt create mode 100644 tests/compiler/std-map/missing-read.phpt create mode 100644 tests/compiler/std-map/unset-key-conversion.phpt create mode 100644 tests/compiler/std-vector/count-runtime-size.phpt create mode 100644 tests/compiler/std-vector/foreach-alias-mutation.phpt diff --git a/run-tests.php b/run-tests.php index fbcc1ca3..f74b8316 100755 --- a/run-tests.php +++ b/run-tests.php @@ -36,9 +36,9 @@ Synopsis: php run-tests.php [options] [files] [directories] Options: - -j Run up to simultaneous testing processes in parallel for - quicker testing on systems with multiple logical processors. - Note that this is experimental feature. + -j, -j , --job + Run up to simultaneous testing processes in parallel + for quicker testing on systems with multiple logical processors. -l Read the testfiles to be executed from . After the test has finished all failed tests are written to the same . @@ -161,7 +161,7 @@ function main(): void $temp_source, $temp_target, $test_cnt, $test_files, $test_idx, $test_results, $testfile, $valgrind, $sum_results, $shuffle, $file_cache, $num_repeats, - $show_progress; + $show_progress, $aot_parallel_root; // Parallel testing global $workers, $workerID; global $context_line_count; @@ -357,6 +357,7 @@ function main(): void $shuffle = false; $bless = false; $workers = null; + $aot_parallel_root = null; $context_line_count = 3; $num_repeats = 1; $show_progress = true; @@ -418,15 +419,14 @@ function main(): void switch ($switch) { case 'j': - $workers = substr($argv[$i], 2); - if ($workers == 0 || !preg_match('/^\d+$/', $workers)) { - error("'$workers' is not a valid number of workers, try e.g. -j16 for 16 workers"); - } - $workers = intval($workers, 10); - // Don't use parallel testing infrastructure if there is only one worker. - if ($workers === 1) { - $workers = null; + $worker_count = substr($argv[$i], 2); + if ($worker_count === '') { + $worker_count = $argv[++$i] ?? ''; } + $workers = parse_worker_count($worker_count); + break; + case '--job': + $workers = parse_worker_count($argv[++$i] ?? ''); break; case 'r': case 'l': @@ -617,6 +617,10 @@ function main(): void exit(1); default: + if (str_starts_with($switch, '--job=')) { + $workers = parse_worker_count(substr($switch, strlen('--job='))); + break; + } echo "Illegal switch '$switch' specified!\n"; // no break case 'h': @@ -819,6 +823,17 @@ function verify_config(string $php): void } } +function parse_worker_count(string $value): ?int +{ + if (!preg_match('/^[1-9]\d*$/', $value)) { + error("'$value' is not a valid number of workers, try e.g. -j16 or --job 16"); + } + + $workers = intval($value, 10); + // Don't use parallel testing infrastructure if there is only one worker. + return $workers === 1 ? null : $workers; +} + /** * @param string[] $user_tests */ @@ -1323,6 +1338,7 @@ function run_all_tests(array $test_files, array $env, ?string $redir_tested = nu function run_all_tests_parallel(array $test_files, array $env, ?string $redir_tested): void { global $workers, $test_idx, $test_results, $failed_tests_file, $result_tests_file, $PHP_FAILED_TESTS, $shuffle, $valgrind, $show_progress; + global $aot_parallel_root; global $junit; @@ -1378,6 +1394,14 @@ function run_all_tests_parallel(array $test_files, array $env, ?string $redir_te // Don't start more workers than test files. $workers = max(1, min($workers, count($test_files))); + $aot_parallel_root = create_aot_parallel_root(); + register_shutdown_function(static function () use (&$aot_parallel_root): void { + if ($aot_parallel_root !== null) { + remove_directory($aot_parallel_root); + $aot_parallel_root = null; + } + }); + echo "Spawning $workers workers... "; // We use sockets rather than STDIN/STDOUT for comms because on Windows, @@ -1633,6 +1657,9 @@ escape: if ($testsInProgress < 0) { error("$testsInProgress test batches “in progress”, which is less than zero. THIS SHOULD NOT HAPPEN."); } + + remove_directory($aot_parallel_root); + $aot_parallel_root = null; } /** @@ -2462,7 +2489,10 @@ $message return 'FAILED'; } $args = substr($args, strlen(' -- ')); - $cmd = (IS_WINDOWS ? '.\\' : './') . $bin_file . ' ' . $args . $cmdRedirect; + $executable = str_contains($bin_file, DIRECTORY_SEPARATOR) + ? $bin_file + : (IS_WINDOWS ? '.\\' : './') . $bin_file; + $cmd = escapeshellarg($executable) . ' ' . $args . $cmdRedirect; } else { $content = file_get_contents($test_file); if (preg_match('/function main\(\)/', $content)) { @@ -4243,15 +4273,30 @@ function debug() function compile_php_file(string $file, string $compiler_args = ''): string { - global $compiler_path; + global $compiler_path, $workerID, $aot_parallel_root; $data = trim(file_get_contents($file)); if (!str_starts_with($data, '')) { throw new CompilationFailureException('Invalid PHP file'); } - $binary_file = str_replace('-', '_', basename($file, '.php')); + $binary_name = str_replace('-', '_', basename($file, '.php')); if (IS_WINDOWS) { - $binary_file .= '.exe'; + $binary_name .= '.exe'; + } + + $compiler_output_args = ''; + if ($workerID && $aot_parallel_root !== null) { + $worker_dir = $aot_parallel_root . DIRECTORY_SEPARATOR . 'worker-' . $workerID; + $build_dir = $worker_dir . DIRECTORY_SEPARATOR . 'build'; + $output_dir = $worker_dir . DIRECTORY_SEPARATOR . 'output'; + ensure_directory_exists($build_dir); + ensure_directory_exists($output_dir); + + $binary_file = $output_dir . DIRECTORY_SEPARATOR . $binary_name; + $compiler_output_args = ' --build-dir ' . escapeshellarg($build_dir) + . ' --output ' . escapeshellarg($binary_file); + } else { + $binary_file = $binary_name; } if (!str_contains($data, 'function main()')) { @@ -4270,10 +4315,11 @@ function compile_php_file(string $file, string $compiler_args = ''): string if (IS_WINDOWS && str_ends_with($cmd, '.php')) { $cmd = escapeshellarg(PHP_BINARY) . ' ' . $cmd; } + $cmd .= ' ' . escapeshellarg($file); if ($compiler_args !== '') { $cmd .= ' ' . $compiler_args; } - exec($cmd . ' ' . escapeshellarg($file) . ' 2>&1', $output, $exitCode); + exec($cmd . $compiler_output_args . ' 2>&1', $output, $exitCode); clearstatcache(true, $binary_file); if ($exitCode !== 0 || !file_exists($binary_file)) { @@ -4283,4 +4329,39 @@ function compile_php_file(string $file, string $compiler_args = ''): string return $binary_file; } +function create_aot_parallel_root(): string +{ + $suffix = getmypid() . '-' . bin2hex(random_bytes(6)); + $directory = rtrim(sys_get_temp_dir(), '/\\') . DIRECTORY_SEPARATOR . 'typephp-run-tests-' . $suffix; + ensure_directory_exists($directory); + return $directory; +} + +function ensure_directory_exists(string $directory): void +{ + if (!is_dir($directory) && !mkdir($directory, 0777, true) && !is_dir($directory)) { + throw new RuntimeException('Cannot create directory: ' . $directory); + } +} + +function remove_directory(string $directory): void +{ + if (!is_dir($directory)) { + return; + } + + $iterator = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator($directory, FilesystemIterator::SKIP_DOTS), + RecursiveIteratorIterator::CHILD_FIRST + ); + foreach ($iterator as $entry) { + if ($entry->isDir() && !$entry->isLink()) { + @rmdir($entry->getPathname()); + } else { + @unlink($entry->getPathname()); + } + } + @rmdir($directory); +} + main(); diff --git a/src/CompilerBase.php b/src/CompilerBase.php index 41464fe7..2aa34da0 100644 --- a/src/CompilerBase.php +++ b/src/CompilerBase.php @@ -4072,6 +4072,9 @@ class CompilerBase implements PropertyAccessContext $code .= 'php::Var ' . $name . ' = php::Var(new ' . $containerType . '(' . $info['typeId'] . '));' . PHP_EOL; $code .= $this->getIndent() . 'auto &' . $name . '_ref = ' . $name . '.toBox<' . $containerType . '>()->container;'; } + if (!isset($info['boxExpr']) && ($defaultValue = $this->getStdContainerDefaultValueExpr($info['type'])) !== null) { + $code .= PHP_EOL . $this->getIndent() . 'php::initializeStdContainer(' . $name . '_ref, ' . $defaultValue . ');'; + } } elseif ($type === Type::STD_VECTOR) { $info = $this->context->stdContainers[$name]; if (isset($info['boxExpr'])) { @@ -4086,6 +4089,10 @@ class CompilerBase implements PropertyAccessContext $code .= 'php::Var ' . $name . ' = php::Var(' . $boxCtor . ');' . PHP_EOL; $code .= $this->getIndent() . 'auto &' . $name . '_ref = ' . $name . '.toBox<' . $containerType . '>()->container;'; } + if (!isset($info['boxExpr']) && $info['size'] !== null + && ($defaultValue = $this->getStdContainerDefaultValueExpr($info['type'])) !== null) { + $code .= PHP_EOL . $this->getIndent() . 'php::initializeStdContainer(' . $name . '_ref, ' . $defaultValue . ');'; + } } elseif ($type === Type::STD_MAP || $type === Type::STD_ORDERED_MAP) { $info = $this->context->stdContainers[$name]; if (isset($info['boxExpr'])) { diff --git a/src/Parser/PropertyAccessTrait.php b/src/Parser/PropertyAccessTrait.php index 8f9791da..b590573a 100644 --- a/src/Parser/PropertyAccessTrait.php +++ b/src/Parser/PropertyAccessTrait.php @@ -717,14 +717,11 @@ trait PropertyAccessTrait if ($var->dim === null) { $this->fatalError($var, 'Cannot use [] for array unset'); } - $array = $this->parseIdentifier($var->var); - if ($this->isStdVector($array) or $this->isStdMap($array) or $this->isStdOrderedMap($array)) { - $this->assertStdContainerStructureMutable($var, $array); - } - $dim = $this->parseIdentifier($var->dim); - if ($this->isStdContainer($array)) { - $lines[] = $array . '_ref.offsetUnset(' . $dim . ');'; + if ($this->isStdContainerExpr($var)) { + $lines[] = $this->parseStdContainerOffsetUnset($var) . ';'; } else { + $array = $this->parseIdentifier($var->var); + $dim = $this->parseIdentifier($var->dim); $lines[] = $array . '.offsetUnset(' . $dim . ');'; } } elseif ($this->isPropertyFetch($var)) { diff --git a/src/Parser/StdContainerTrait.php b/src/Parser/StdContainerTrait.php index 7aefe7c8..e70c8445 100644 --- a/src/Parser/StdContainerTrait.php +++ b/src/Parser/StdContainerTrait.php @@ -284,7 +284,6 @@ trait StdContainerTrait $this->fatalError($left, 'std map expects a key'); } - $this->assertStdContainerStructureMutable($left, $container); return $this->parseStdContainerOffsetSet($left, $this->convertStdValueExpr($info, $right)); } @@ -301,7 +300,23 @@ trait StdContainerTrait if ($attr['accessLevel'] < $attr['totalLevel']) { $this->fatalError($expr, 'Cannot use assign operator on nested std::array'); } - return $arrayDimFetch . ' ' . $binaryOp . '= ' . $this->convertExprFromType($info['type'], $this->parseExpr($expr->expr)); + $rightExpr = $this->parseExpr($expr->expr); + if (in_array($info['type'], [Type::BIGINT, Type::BIGFLOAT, Type::DECIMAL], true)) { + $rightType = $this->detectTypeOfExpr($expr->expr); + $item = $this->genTmpVarName(); + $bigExpr = $this->parseBigAssignOpExpr( + $item, + $info['type'], + $rightExpr, + $rightType, + $binaryOp, + $expr->var, + $expr->expr + ); + return '([&](php::Var &' . $item . ') -> php::Var & { return ' . $item . ' = ' . $bigExpr . '; })(' + . $arrayDimFetch . ')'; + } + return $arrayDimFetch . ' ' . $binaryOp . '= ' . $this->convertExprFromType($info['type'], $rightExpr); } protected function parseStdContainerAssignOp(Expr\AssignOp $expr, string $op): string @@ -316,8 +331,24 @@ trait StdContainerTrait } $info = $this->getStdContainerInfo($expr->var); - $containerDimFetch = $this->parseStdContainerDimFetch($expr->var); - return $containerDimFetch . ' ' . $binaryOp . '= ' . $this->convertExprFromType($info['type'], $this->parseExpr($expr->expr)); + $containerDimFetch = $this->parseStdContainerDimFetch($expr->var, true); + $rightExpr = $this->parseExpr($expr->expr); + if (in_array($info['type'], [Type::BIGINT, Type::BIGFLOAT, Type::DECIMAL], true)) { + $rightType = $this->detectTypeOfExpr($expr->expr); + $item = $this->genTmpVarName(); + $bigExpr = $this->parseBigAssignOpExpr( + $item, + $info['type'], + $rightExpr, + $rightType, + $binaryOp, + $expr->var, + $expr->expr + ); + return '([&](php::Var &' . $item . ') -> php::Var & { return ' . $item . ' = ' . $bigExpr . '; })(' + . $containerDimFetch . ')'; + } + return $containerDimFetch . ' ' . $binaryOp . '= ' . $this->convertExprFromType($info['type'], $rightExpr); } protected function parseStdArrayDimFetch(Expr\ArrayDimFetch $expr): string @@ -373,7 +404,15 @@ trait StdContainerTrait ($this->context->stdContainers[$container]['iterationDepth'] ?? 0) + 1; } $iterator = $this->genTmpVarName(); - $code = "for (auto $iterator = {$container}_ref.begin(); $iterator != {$container}_ref.end(); ++$iterator) {" . PHP_EOL; + $code = ''; + if ($mutableContainer) { + $guard = $this->genTmpVarName(); + $code .= '{' . PHP_EOL; + $this->indentLevel++; + $code .= $this->getIndent() . "auto $guard = {$container}_ref.iterationGuard();" . PHP_EOL; + $code .= $this->getIndent(); + } + $code .= "for (auto $iterator = {$container}_ref.begin(); $iterator != {$container}_ref.end(); ++$iterator) {" . PHP_EOL; $this->indentLevel++; if ($node->keyVar) { $keyVar = $this->parseIdentifier($node->keyVar); @@ -415,11 +454,15 @@ trait StdContainerTrait $code .= $body . PHP_EOL; $code .= $this->getIndent() . '}'; + if ($mutableContainer) { + $this->indentLevel--; + $code .= PHP_EOL . $this->getIndent() . '}'; + } unset($this->context->objects[$valueVar]); return $code; } - protected function parseStdContainerDimFetch(Expr\ArrayDimFetch $expr): string + protected function parseStdContainerDimFetch(Expr\ArrayDimFetch $expr, bool $forUpdate = false): string { if ($this->isStdArrayExpr($expr)) { return $this->parseStdArrayDimFetch($expr); @@ -450,7 +493,18 @@ trait StdContainerTrait $container = $this->parseVariable($tmp); $index = $this->parseExpr($dim); $key = $info['kind'] === 'vector' ? $this->convertIntExpr($index) : $this->convertStdContainerKey($info, $index); - $access = $container . '_ref.offsetGet(' . $key . ')'; + $method = $forUpdate && ($info['kind'] === 'map' || $info['kind'] === 'ordered_map') + ? 'offsetGetForUpdate' + : 'offsetGet'; + $args = $key; + if ($method === 'offsetGetForUpdate') { + $defaultValue = $this->getStdContainerDefaultValueExpr($info['type']); + if ($defaultValue !== null) { + $method = 'offsetGetForUpdateLazy'; + $args .= ', []() { return ' . $defaultValue . '; }'; + } + } + $access = $container . '_ref.' . $method . '(' . $args . ')'; $expr->setAttribute('stdContainerDimFetch', ['var' => $container, 'accessLevel' => 1, 'totalLevel' => 1]); return $access; @@ -479,6 +533,57 @@ trait StdContainerTrait return $this->convertIntExpr($index); } + protected function getStdContainerDefaultValueExpr(string $type): ?string + { + return match ($type) { + Type::BIGINT => 'php::BigInt::newInstance(0)', + Type::BIGFLOAT => 'php::BigFloat::newInstance(0)', + Type::DECIMAL => 'php::Decimal::newInstance(0)', + default => null, + }; + } + + protected function parseStdContainerOffsetUnset(Expr\ArrayDimFetch $expr): string + { + if ($expr->dim === null) { + $this->fatalError($expr, 'std container expects an index'); + } + + if ($this->isStdArrayExpr($expr)) { + $info = $this->getStdArrayInfo($expr); + $target = $this->parseStdArrayDimFetch($expr); + $defaultValue = $this->getStdContainerDefaultValueExpr($info['type']); + if ($defaultValue !== null) { + return $target . ' = ' . $defaultValue; + } + + if ($this->isVarExpr($expr->var)) { + $parent = $this->parseVariable($expr->var) . '_ref'; + } elseif ($this->isArrayDimFetch($expr->var)) { + $parent = $this->parseStdArrayDimFetch($expr->var); + } else { + $this->fatalError($expr, 'std::array expects a variable'); + } + $index = $this->convertIntExpr($this->parseExpr($expr->dim)); + return $parent . '.offsetUnset(' . $index . ')'; + } + + $info = $this->getStdContainerInfo($expr); + if ($info === null || !$this->isVarExpr($expr->var)) { + $this->fatalError($expr, 'std container expects a variable'); + } + $container = $this->parseVariable($expr->var); + $indexExpr = $this->parseExpr($expr->dim); + $index = $info['kind'] === 'vector' + ? $this->convertIntExpr($indexExpr) + : $this->convertStdContainerKey($info, $indexExpr); + $defaultValue = $this->getStdContainerDefaultValueExpr($info['type']); + if ($defaultValue !== null && $info['kind'] === 'vector') { + return $container . '_ref.offsetSet(' . $index . ', ' . $defaultValue . ')'; + } + return $container . '_ref.offsetUnset(' . $index . ')'; + } + protected function getStdContainerElementType(string $type): string { return match ($type) { @@ -783,10 +888,6 @@ trait StdContainerTrait return $sizes[0] . $this->getPlatform()->getIntegerLiteralSuffix(); } if ($this->isStdVector($var)) { - $info = $this->context->stdContainers[$var]; - if ($info['size'] !== null) { - return $info['size'] . $this->getPlatform()->getIntegerLiteralSuffix(); - } return $var . '_ref.size()'; } if ($this->isStdContainer($var)) { @@ -814,19 +915,6 @@ trait StdContainerTrait } } - if ($this->isVarExpr($tmp)) { - $var = $this->parseVariable($tmp); - if ($this->isStdVector($var)) { - $info = $this->context->stdContainers[$var]; - if ($info['size'] !== null) { - return $info['size'] . $this->getPlatform()->getIntegerLiteralSuffix(); - } - return $var . '_ref.size()'; - } - if ($this->isStdMap($var) || $this->isStdOrderedMap($var)) { - return $var . '_ref.size()'; - } - } } return false; diff --git a/tests/compiler/std-bigint/container-default-compound.phpt b/tests/compiler/std-bigint/container-default-compound.phpt new file mode 100644 index 00000000..5a9fe01e --- /dev/null +++ b/tests/compiler/std-bigint/container-default-compound.phpt @@ -0,0 +1,37 @@ +--TEST-- +std high precision containers: defaults, compound writes and unset +--FILE-- +toStdArray(Type::BigInt, 2); + var_dump($alias[0]->toString()); +} + +function main() { + $integers = std::array(Type::BigInt, 2); + var_dump($integers[0]->toString()); + $integers[0] += 5; + var_dump($integers[0]->toString()); + inspect_bigint_array($integers); + unset($integers[0]); + var_dump($integers[0]->toString()); + + $floats = std::vector(Type::BigFloat, 1); + $floats[0] += 2; + var_dump($floats[0]->toString()); + unset($floats[0]); + var_dump($floats[0]->toString()); + + $decimals = std::map(Type::String, Type::Decimal); + $decimals['total'] += 3; + var_dump($decimals['total']->toString()); +} +?> +--EXPECT-- +string(1) "0" +string(1) "5" +string(1) "5" +string(1) "0" +string(1) "2" +string(1) "0" +string(1) "3" diff --git a/tests/compiler/std-map/foreach-mutation-guard.phpt b/tests/compiler/std-map/foreach-mutation-guard.phpt new file mode 100644 index 00000000..50b871c5 --- /dev/null +++ b/tests/compiler/std-map/foreach-mutation-guard.phpt @@ -0,0 +1,29 @@ +--TEST-- +std map: foreach allows value updates and blocks structural writes +--FILE-- + $value) { + $map[$key] += 10; + try { + $map['new'] = 3; + } catch (Throwable $e) { + echo "blocked\n"; + } + } + + var_dump(count($map)); + var_dump($map['a']); + var_dump($map['b']); +} +?> +--EXPECT-- +blocked +blocked +int(2) +int(11) +int(12) diff --git a/tests/compiler/std-map/missing-read.phpt b/tests/compiler/std-map/missing-read.phpt new file mode 100644 index 00000000..2b697f74 --- /dev/null +++ b/tests/compiler/std-map/missing-read.phpt @@ -0,0 +1,27 @@ +--TEST-- +std map: missing reads do not insert keys +--FILE-- + +--EXPECT-- +missing +int(0) +ordered missing +int(0) diff --git a/tests/compiler/std-map/unset-key-conversion.phpt b/tests/compiler/std-map/unset-key-conversion.phpt new file mode 100644 index 00000000..c11ae5a2 --- /dev/null +++ b/tests/compiler/std-map/unset-key-conversion.phpt @@ -0,0 +1,19 @@ +--TEST-- +std containers: unset uses normal key conversion +--FILE-- + +--EXPECT-- +int(0) +int(0) diff --git a/tests/compiler/std-vector/count-runtime-size.phpt b/tests/compiler/std-vector/count-runtime-size.phpt new file mode 100644 index 00000000..c75f2e27 --- /dev/null +++ b/tests/compiler/std-vector/count-runtime-size.phpt @@ -0,0 +1,17 @@ +--TEST-- +std vector: count uses current runtime size +--FILE-- + +--EXPECT-- +int(3) +int(4) diff --git a/tests/compiler/std-vector/foreach-alias-mutation.phpt b/tests/compiler/std-vector/foreach-alias-mutation.phpt new file mode 100644 index 00000000..b3107a23 --- /dev/null +++ b/tests/compiler/std-vector/foreach-alias-mutation.phpt @@ -0,0 +1,28 @@ +--TEST-- +std containers: foreach structural mutation through aliases is rejected safely +--FILE-- +toStdVector(Type::Int); + $alias[] = 3; +} + +function main() { + $vector = std::vector(Type::Int); + $vector[] = 1; + $vector[] = 2; + + foreach ($vector as $value) { + try { + append_to_std_vector($vector); + } catch (Throwable $e) { + echo "blocked\n"; + } + } + var_dump(count($vector)); +} +?> +--EXPECT-- +blocked +blocked +int(2)