diff --git a/docs/YIELD_GENERATOR.md b/docs/YIELD_GENERATOR.md index dedd723c..14c2d9ac 100644 --- a/docs/YIELD_GENERATOR.md +++ b/docs/YIELD_GENERATOR.md @@ -1,8 +1,8 @@ # yield / Generator -TypePHP 将包含 `yield` 或 `yield from` 的函数和方法编译为 Fiber generator。调用这类函数时不会立即执行函数体,而是返回一个 `TypePHP\FiberGenerator` 对象;第一次迭代、调用 `current()`、`valid()`、`send()` 或 `throw()` 时才创建并启动 Fiber。 +TypePHP 将包含 `yield` 或 `yield from` 的函数、方法、匿名函数和箭头函数编译为 Fiber generator。调用时不会立即执行 generator 函数体,而是返回一个全局 `\FiberGenerator` 对象;第一次迭代、调用 `current()`、`valid()`、`send()` 或 `throw()` 时才创建并启动 Fiber。 -生成的 C++ 函数体运行在 Fiber 中。每次 `yield` 都通过 `Fiber::suspend()` 将 key/value 交给迭代驱动方,并在 `next()`、`send()` 或 `throw()` 时恢复原来的 C++ 调用栈。运行时使用 `NEW`、`SUSPENDED`、`CLOSED_RETURNED` 和 `CLOSED_FAILED` 四种状态区分未启动、挂起、正常返回和异常关闭。 +生成的 C++ 函数体运行在 Fiber 中。每次 `yield` 都通过 `Fiber::suspend()` 将 key/value 交给迭代驱动方,并在 `next()`、`send()` 或 `throw()` 时恢复原来的 C++ 调用栈。运行时使用 `NEW`、`RUNNING`、`SUSPENDED`、`CLOSED_RETURNED` 和 `CLOSED_FAILED` 区分未启动、执行中、挂起、正常返回和异常关闭。 ## 迭代互操作 @@ -12,10 +12,10 @@ TypePHP 将包含 `yield` 或 `yield from` 的函数和方法编译为 Fiber gen |---|---| | TypePHP Native `foreach` | TypePHP Native generator | | TypePHP Native `foreach` | 动态 PHP 返回的 Zend `Generator` | -| ZendVM `foreach` | TypePHP Native generator 返回的 `TypePHP\FiberGenerator` | -| TypePHP `yield from` | 数组、`Iterator`、`IteratorAggregate`、Zend `Generator`、`TypePHP\FiberGenerator` | +| ZendVM `foreach` | TypePHP Native generator 返回的 `\FiberGenerator` | +| TypePHP `yield from` | 数组、`Iterator`、`IteratorAggregate`、Zend `Generator`、`\FiberGenerator` | -`TypePHP\FiberGenerator` 实现了 `Iterator`,其接口签名与 PHP 一致: +`\FiberGenerator` 实现了 `Iterator`,其接口签名与 PHP 一致: ```php rewind(): void @@ -34,7 +34,7 @@ getReturn(): mixed ### 不是 Zend Generator -TypePHP generator 返回的是 `TypePHP\FiberGenerator`,不是 PHP 内置的 `Generator`: +TypePHP generator 返回的是全局 `\FiberGenerator`,不是 PHP 内置的 `Generator`: ```php $generator instanceof Iterator; // true @@ -45,10 +45,10 @@ $generator instanceof Generator; // false - generator 函数不能声明精确返回类型 `Generator`。 - 可以使用 `Iterator`、`Traversable`、`iterable`、`object`、`mixed`,或包含这些兼容类型的联合类型。 -- `ReflectionGenerator` 只接受 Zend `Generator`,不能用于 `TypePHP\FiberGenerator`。 +- `ReflectionGenerator` 只接受 Zend `Generator`,不能用于 `\FiberGenerator`。 - `get_class()`、Reflection class 信息和异常栈中的类名与 Zend `Generator` 不同。 -- 不保证 `var_dump()`、调试属性、序列化错误、clone 行为及内部对象布局与 Zend `Generator` 一致。 -- `TypePHP\FiberGenerator` 是运行时内部实现类型,不应由业务代码直接实例化、继承、clone 或序列化。 +- 不保证 `var_dump()`、调试属性及内部对象布局与 Zend `Generator` 一致。 +- `\FiberGenerator` 是 final 运行时内部类型,禁止业务代码直接实例化、继承、clone 或序列化。 ### 不支持引用 Generator @@ -112,7 +112,7 @@ Zend `Generator` 是 ZendVM 的专用执行对象;TypePHP generator 使用 PHP - 委托通过 `rewind()`、`valid()`、`key()`、`current()`、`next()`、`send()`、`throw()` 和 `getReturn()` 方法完成。 - 自定义 Iterator 方法产生的副作用、异常栈和调用次数应避免依赖 Zend Generator 的内部实现细节。 -- 非 generator Iterator 的 `yield from` 结果为 `null`;只有 Zend `Generator` 和 `TypePHP\FiberGenerator` 会读取 `getReturn()`。 +- 非 generator Iterator 的 `yield from` 结果为 `null`;只有 Zend `Generator` 和 `\FiberGenerator` 会读取 `getReturn()`。 ## 性能差异 @@ -138,6 +138,6 @@ Fiber generator 不会改变普通数组或普通容器 `foreach` 的生成代 - `ReflectionGenerator`。 - generator 按引用返回、by-reference yield 或按引用 foreach。 - generator 的按引用参数或可变参数。 -- clone、序列化、调试输出及内部属性与 Zend Generator 相同。 +- 调试输出及内部属性与 Zend Generator 相同。 - 所有内部扩展 `Traversable` 都能通过 Zend iterator handlers 迭代。 - Fiber 关闭、复杂析构和进程退出时与 Zend Generator 完全相同的栈及析构顺序。 diff --git a/src/Generator/ClosureGenerator.php b/src/Generator/ClosureGenerator.php index 1ea60f53..781781ac 100644 --- a/src/Generator/ClosureGenerator.php +++ b/src/Generator/ClosureGenerator.php @@ -77,6 +77,25 @@ trait ClosureGenerator protected function genClosure(Expr\ArrowFunction|Expr\Closure $expr, array $params, array $uses = []): string { + $entryContext = $this->context; + $entryIndent = $this->indentLevel; + $entryInGeneratorBody = $this->inGeneratorBody; + + try { + return $this->doGenClosure($expr, $params, $uses); + } finally { + $this->context = $entryContext; + $this->indentLevel = $entryIndent; + $this->inGeneratorBody = $entryInGeneratorBody; + } + } + + private function doGenClosure(Expr\ArrowFunction|Expr\Closure $expr, array $params, array $uses = []): string + { + $isGenerator = $this->closureContainsYield($expr); + if ($isGenerator) { + $this->validateGeneratorClosure($expr, $params); + } $tmpVar = $this->genTmpVarName(); $code = $this->getIndent() . @@ -90,7 +109,10 @@ trait ClosureGenerator $this->context = new FunctionContext(); $this->context->inClosure = true; - if ($expr->returnType instanceof NullableType || $expr->returnType instanceof UnionType || $expr->returnType instanceof IntersectionType) { + if (!$isGenerator + && ($expr->returnType instanceof NullableType + || $expr->returnType instanceof UnionType + || $expr->returnType instanceof IntersectionType)) { $returnTypeInfo = $this->buildTypeCheckFromNode($expr->returnType); if (!empty($returnTypeInfo['check'])) { $this->context->closureReturnTypeCheck = $returnTypeInfo['check']; @@ -156,7 +178,9 @@ trait ClosureGenerator $this->addArgument('this_', Type::OBJECT); } - $body = $this->genClosureBody($expr); + $body = $isGenerator + ? $this->genGeneratorClosureFactoryBody($expr, $params, $uses) + : $this->genClosureBody($expr); $code .= $this->genScopeVarDecl() . $body; $this->indentLevel--; @@ -193,6 +217,120 @@ trait ClosureGenerator } } + protected function closureContainsYield(Expr\ArrowFunction|Expr\Closure $expr): bool + { + if ($expr instanceof Expr\ArrowFunction) { + return $this->containsYieldInNode($expr->expr); + } + return $this->containsYieldInNodes($expr->stmts); + } + + protected function validateGeneratorClosure(Expr\ArrowFunction|Expr\Closure $expr, array $params): void + { + if ($expr->byRef) { + $this->fatalError($expr, 'Generator closures returning by reference are not supported yet'); + } + foreach ($params as $param) { + if ($param->byRef || $param->variadic) { + $this->fatalError($param, 'Generator closures with by-reference or variadic parameters are not supported yet'); + } + } + if (!$this->generatorReturnTypeAcceptsFiber($expr->returnType)) { + $this->fatalError( + $expr, + 'Generator closure return type must accept \\FiberGenerator; use Iterator, Traversable, iterable, object, mixed, or omit the return type' + ); + } + } + + protected function genGeneratorClosureFactoryBody( + Expr\ArrowFunction|Expr\Closure $expr, + array $params, + array $uses + ): string { + $capturedNames = []; + $capturedArgs = []; + foreach ($params as $param) { + $name = $this->parseIdentifier($param->var); + $capturedNames[] = $name; + $capturedArgs[] = $name; + } + foreach ($uses as $useItem) { + $name = $this->parseIdentifier($useItem->var); + $capturedNames[] = $name; + // Building an initializer_list copies Variants by value. Re-wrap + // reference captures so the delayed Fiber callback keeps identity. + $capturedArgs[] = $useItem->byRef ? $name . '.toReference()' : $name; + } + + $outerContext = $this->context; + $outerIndent = $this->indentLevel; + $outerInGeneratorBody = $this->inGeneratorBody; + $callbackVar = $this->genTmpVarName(); + + $code = $this->getIndent() . 'php::ClosureFn ' . $callbackVar . ' = [](' + . 'INTERNAL_FUNCTION_PARAMETERS, ' + . Type::OBJECT . ' &this_, ' + . Type::ARGS . ' &vars_) -> ' . Type::VAR . ' {' . PHP_EOL; + + $this->context = new FunctionContext(); + $this->context->inClosure = true; + $this->inGeneratorBody = true; + $this->indentLevel++; + + try { + foreach ($capturedNames as $i => $name) { + $code .= $this->getIndent() . Type::VAR . ' ' . $name . ' = vars_.get(' . $i . ');' . PHP_EOL; + $this->addArgument($name, Type::VAR); + } + if ($this->methodDef) { + $this->addArgument('this_', Type::OBJECT); + } + + $this->indentLevel++; + $body = ''; + if ($this->methodDef && $this->methodDef->hasDynamicCall) { + $body .= $this->genScopeSwitchCode(); + } + if ($expr instanceof Expr\ArrowFunction) { + [$value, $beforeStmts, $afterStmts] = $this->parseExprWithCapturedStmts($expr->expr); + $body .= $this->formatCapturedStmtLines($beforeStmts); + if ($afterStmts) { + $resultVar = $this->addTmpVar(Type::VAR); + $body .= $this->getIndent() . $resultVar . ' = ' . $value . ';' . PHP_EOL; + $body .= $this->formatCapturedStmtLines($afterStmts); + $value = $resultVar; + } + $body .= $this->getIndent() . 'return ' . $value . ';' . PHP_EOL; + } else { + $body .= $this->parseStmts($expr->stmts); + if (!$this->isReturnStmtInLastLine($expr->stmts)) { + $body .= $this->getIndent() . 'return php::null;' . PHP_EOL; + } + } + $this->indentLevel--; + + $code .= $this->genScopeVarDecl(); + $code .= $this->getIndent() . 'try {' . PHP_EOL; + $code .= $body; + $code .= $this->getIndent() . '} catch (zend_object *) {' . PHP_EOL; + $code .= $this->getIndent() . ' return php::null;' . PHP_EOL; + $code .= $this->getIndent() . '}' . PHP_EOL; + } finally { + $this->context = $outerContext; + $this->indentLevel = $outerIndent; + $this->inGeneratorBody = $outerInGeneratorBody; + } + + $code .= $this->getIndent() . '};' . PHP_EOL; + $args = $capturedArgs ? '{ ' . implode(', ', $capturedArgs) . ' }' : '{}'; + $callback = $this->methodDef + ? 'php::newClosure(' . $callbackVar . ', ' . $args . ', this_)' + : 'php::newClosure(' . $callbackVar . ', ' . $args . ')'; + $code .= $this->getIndent() . 'return typephp_new_fiber_generator(' . $callback . ');' . PHP_EOL; + return $code; + } + protected function genClosureBody(NodeAbstract $expr): string { if ($expr instanceof Node\Expr\ArrowFunction) { diff --git a/src/Generator/FiberGenerator.php b/src/Generator/FiberGenerator.php index 213ccc66..92a68e47 100644 --- a/src/Generator/FiberGenerator.php +++ b/src/Generator/FiberGenerator.php @@ -70,7 +70,7 @@ trait FiberGenerator } } if (!$this->generatorReturnTypeAcceptsFiber($v->returnType)) { - $this->fatalError($v, 'Generator return type must accept TypePHP\\FiberGenerator; use Iterator, Traversable, iterable, object, mixed, or omit the return type'); + $this->fatalError($v, 'Generator return type must accept \\FiberGenerator; use Iterator, Traversable, iterable, object, mixed, or omit the return type'); } $functionDef->generator = true; $functionDef->returnType = Type::VAR; @@ -112,7 +112,7 @@ trait FiberGenerator [, $class] = $this->resolveTypeDecl($type, self::DECL_TYPE_OF_RETURN); $class = strtolower(ltrim($class, '\\')); - return in_array($class, ['iterator', 'traversable', 'typephp\\fibergenerator'], true); + return in_array($class, ['iterator', 'traversable', 'fibergenerator'], true); } protected function parseYieldExpr(Yield_ $expr): string @@ -140,7 +140,7 @@ trait FiberGenerator $closed = $this->genTmpVarName(); $this->addLocalVar($closed, Type::BOOL); return $closed . ' = false;' . PHP_EOL - . $this->getIndent() . 'typephp_fiber_yield_from(' . $this->parseExprAsValue($expr->expr) . ', &' . $closed . ');' . PHP_EOL + . $this->getIndent() . 'typephp_fiber_yield_from(' . $this->genYieldFromIterable($expr) . ', &' . $closed . ');' . PHP_EOL . $this->getIndent() . 'if (' . $closed . ') {' . PHP_EOL . $this->getIndent() . ' return ' . self::VALUE_NULL . ';' . PHP_EOL . $this->getIndent() . '}'; @@ -148,20 +148,51 @@ trait FiberGenerator protected function genYieldPayload(Yield_ $expr): string { - $value = $expr->value ? $this->parseExprAsValue($expr->value) : self::VALUE_NULL; if ($expr->key) { - $key = $this->parseExprAsValue($expr->key); + // PHP evaluates an explicit yield key before its value. Materialize + // both operands so lowering helpers cannot hoist value side effects + // ahead of the key or defer postfix side effects until after resume. + $key = $this->materializeYieldOperand($expr->key, true); + $value = $expr->value + ? $this->materializeYieldOperand($expr->value, true) + : self::VALUE_NULL; return 'php::Array(php::StdStrKeyMap{{"key", ' . $key . '}, {"value", ' . $value . '}, {"has_key", true}})'; } + $value = $expr->value + ? $this->materializeYieldOperand($expr->value) + : self::VALUE_NULL; return 'php::Array(php::StdStrKeyMap{{"value", ' . $value . '}, {"has_key", false}})'; } + private function materializeYieldOperand(Node $expr, bool $force = false): string + { + [$value, $beforeStmts, $afterStmts] = $this->parseExprWithCapturedStmts($expr); + foreach ($beforeStmts as $stmt) { + $this->context->beforeStmtLines[] = $stmt; + } + if (!$force && !$afterStmts) { + return $value; + } + + $tmpVar = $this->addTmpVar(Type::VAR); + $this->context->beforeStmtLines[] = $tmpVar . ' = ' . $value . ';'; + foreach ($afterStmts as $stmt) { + $this->context->beforeStmtLines[] = $stmt; + } + return $tmpVar; + } + + private function genYieldFromIterable(YieldFrom $expr): string + { + return $this->materializeYieldOperand($expr->expr); + } + protected function parseYieldFromExpr(YieldFrom $expr): string { if (!$this->inGeneratorBody) { $this->fatalError($expr, 'The `Expr_YieldFrom` is not supported outside generator functions'); } - return 'typephp_fiber_yield_from(' . $this->parseExprAsValue($expr->expr) . ', nullptr)'; + return 'typephp_fiber_yield_from(' . $this->genYieldFromIterable($expr) . ', nullptr)'; } protected function genFiberGeneratorFunction(Function_|ClassMethod $v, FunctionDef $functionDef, string $nativeName): string @@ -256,7 +287,7 @@ trait FiberGenerator $closureExpr = $this->class ? 'php::newClosure(' . $closureVar . ', ' . $args . ', this_)' : 'php::newClosure(' . $closureVar . ', ' . $args . ')'; - $code .= $this->getIndent() . 'return php::newObject(typephp_fiber_generator_ce, {' . $closureExpr . '});' . PHP_EOL; + $code .= $this->getIndent() . 'return typephp_new_fiber_generator(' . $closureExpr . ');' . PHP_EOL; $this->indentLevel--; $code .= '}' . PHP_EOL; diff --git a/tests/compiler/generator/closed-value-release.phpt b/tests/compiler/generator/closed-value-release.phpt new file mode 100644 index 00000000..710062c3 --- /dev/null +++ b/tests/compiler/generator/closed-value-release.phpt @@ -0,0 +1,35 @@ +--TEST-- +closed generator releases captured values while preserving its return value +--FILE-- +current(); + $generator->next(); + gc_collect_cycles(); + var_dump($generator->getReturn()); + var_dump(GeneratorRetentionState::$weak?->get() === null); +} +?> +--EXPECT-- +int(2) +bool(true) diff --git a/tests/compiler/generator/closure-generator.phpt b/tests/compiler/generator/closure-generator.phpt new file mode 100644 index 00000000..d4cb94a8 --- /dev/null +++ b/tests/compiler/generator/closure-generator.phpt @@ -0,0 +1,53 @@ +--TEST-- +anonymous and arrow generator closures preserve lazy execution captures and return values +--FILE-- + $this->value + $base + $offset; + return 9; + }; + } + + public int $value = 10; +} + +function main(): void +{ + $state = 1; + $factory = function (int $add) use (&$state): iterable { + ++$state; + $sent = yield 'closure' => $state + $add; + return $sent; + }; + + $generator = $factory(3); + var_dump($state); + var_dump($generator->key(), $generator->current()); + var_dump($state); + var_dump($generator->send(7)); + var_dump($generator->getReturn()); + + $arrow = fn (): iterable => yield 'arrow' => $state; + var_dump($arrow()->key(), $arrow()->current()); + + $method = (new GeneratorClosureBox())->make(2)(3); + var_dump($method->current()); + $method->next(); + var_dump($method->getReturn()); +} +?> +--EXPECT-- +int(1) +string(7) "closure" +int(5) +int(2) +NULL +int(7) +string(5) "arrow" +int(2) +int(15) +int(9) diff --git a/tests/compiler/generator/failed-value-release.phpt b/tests/compiler/generator/failed-value-release.phpt new file mode 100644 index 00000000..aa34bc1c --- /dev/null +++ b/tests/compiler/generator/failed-value-release.phpt @@ -0,0 +1,41 @@ +--TEST-- +failed generator releases captured values while retaining failed state +--FILE-- +current(); + } catch (Throwable $e) { + echo $e->getMessage(), "\n"; + } + gc_collect_cycles(); + var_dump($generator->valid()); + var_dump(FailedGeneratorRetentionState::$weak?->get() === null); +} +?> +--EXPECT-- +failed +bool(false) +bool(true) diff --git a/tests/compiler/generator/fiber-generator-arginfo.phpt b/tests/compiler/generator/fiber-generator-arginfo.phpt index e1a7d477..ac863b41 100644 --- a/tests/compiler/generator/fiber-generator-arginfo.phpt +++ b/tests/compiler/generator/fiber-generator-arginfo.phpt @@ -1,5 +1,5 @@ --TEST-- -TypePHP FiberGenerator methods expose Iterator-compatible return types +FiberGenerator methods expose Iterator-compatible return types --INI-- error_reporting=E_ALL --FILE-- diff --git a/tests/compiler/generator/global-fiber-generator-class.phpt b/tests/compiler/generator/global-fiber-generator-class.phpt new file mode 100644 index 00000000..80bac677 --- /dev/null +++ b/tests/compiler/generator/global-fiber-generator-class.phpt @@ -0,0 +1,27 @@ +--TEST-- +FiberGenerator is registered globally and accepted as an exact generator return type +--FILE-- +current()); + $generator->next(); + var_dump($generator->getReturn()); +} +?> +--EXPECT-- +string(14) "FiberGenerator" +bool(true) +bool(false) +int(1) +int(2) diff --git a/tests/compiler/generator/internal-class-guards.phpt b/tests/compiler/generator/internal-class-guards.phpt new file mode 100644 index 00000000..55b2392f --- /dev/null +++ b/tests/compiler/generator/internal-class-guards.phpt @@ -0,0 +1,34 @@ +--TEST-- +FiberGenerator cannot be directly constructed cloned or serialized +--FILE-- +current(); + + foreach (['clone', 'serialize', 'construct'] as $operation) { + try { + if ($operation === 'clone') { + $copy = clone $generator; + } elseif ($operation === 'serialize') { + serialize($generator); + } else { + new FiberGenerator(null); + } + echo $operation, ":allowed\n"; + } catch (Throwable $e) { + echo $operation, ':', get_class($e), "\n"; + } + } +} +?> +--EXPECT-- +clone:Error +serialize:Exception +construct:Error diff --git a/tests/compiler/generator/new-empty-throw.phpt b/tests/compiler/generator/new-empty-throw.phpt new file mode 100644 index 00000000..0abfa241 --- /dev/null +++ b/tests/compiler/generator/new-empty-throw.phpt @@ -0,0 +1,26 @@ +--TEST-- +throw on a new generator that returns before yielding rethrows the supplied exception +--FILE-- +throw(new RuntimeException('new-empty')); + } catch (Throwable $e) { + echo get_class($e), ':', $e->getMessage(), "\n"; + } + var_dump($generator->getReturn()); +} +?> +--EXPECT-- +RuntimeException:new-empty +int(7) diff --git a/tests/compiler/generator/reentrant-generator.phpt b/tests/compiler/generator/reentrant-generator.phpt new file mode 100644 index 00000000..26ca5ace --- /dev/null +++ b/tests/compiler/generator/reentrant-generator.phpt @@ -0,0 +1,39 @@ +--TEST-- +reentrant generator advance throws without corrupting outer generator state +--FILE-- +next(); + } catch (Throwable $e) { + echo get_class($e), ':', $e->getMessage(), "\n"; + } + yield 2; + return 3; +} + +function main(): void +{ + $generator = reentrant_generator(); + ReentrantGeneratorState::$generator = $generator; + var_dump($generator->current()); + $generator->next(); + var_dump($generator->current(), $generator->valid()); + $generator->next(); + var_dump($generator->valid(), $generator->getReturn()); +} +?> +--EXPECT-- +int(1) +Error:Cannot resume an already running generator +int(2) +bool(true) +bool(false) +int(3) diff --git a/tests/compiler/generator/yield-evaluation-order.phpt b/tests/compiler/generator/yield-evaluation-order.phpt new file mode 100644 index 00000000..ee1ea18a --- /dev/null +++ b/tests/compiler/generator/yield-evaluation-order.phpt @@ -0,0 +1,42 @@ +--TEST-- +yield evaluates key and value side effects before suspension in PHP order +--FILE-- + match (mark('selector')) { + 'selector' => mark('value'), + }; + yield $box->value++; +} + +function main(): void +{ + $box = new YieldOrderBox(); + $generator = ordered_yield($box); + var_dump($generator->current()); + var_dump($box->value); + $generator->next(); + var_dump($generator->current()); + var_dump($box->value); +} +?> +--EXPECT-- +key +selector +value +string(5) "value" +int(0) +int(0) +int(1) diff --git a/tests/compiler/generator/yield-from-iterator-order.phpt b/tests/compiler/generator/yield-from-iterator-order.phpt new file mode 100644 index 00000000..3019ac5c --- /dev/null +++ b/tests/compiler/generator/yield-from-iterator-order.phpt @@ -0,0 +1,35 @@ +--TEST-- +yield from calls Iterator current before key +--FILE-- +index < 1; } + public function current(): mixed { echo "current\n"; return 1; } + public function key(): mixed { echo "key\n"; return 0; } + public function next(): void { echo "next\n"; ++$this->index; } +} + +function ordered_yield_from(): iterable +{ + yield from new YieldFromOrderIterator(); +} + +function main(): void +{ + foreach (ordered_yield_from() as $key => $value) { + echo "body\n"; + } +} +?> +--EXPECT-- +rewind +valid +current +key +body +next +valid