Page MenuHomeWickedGov Phorge

TokenStreamPatcher.php
No OneTemporary

Size
15 KB
Referenced Files
None
Subscribers
None

TokenStreamPatcher.php

<?php
declare( strict_types = 1 );
namespace Wikimedia\Parsoid\Wt2Html\TT;
use Wikimedia\Parsoid\NodeData\DataParsoid;
use Wikimedia\Parsoid\Tokens\EndTagTk;
use Wikimedia\Parsoid\Tokens\EOFTk;
use Wikimedia\Parsoid\Tokens\KV;
use Wikimedia\Parsoid\Tokens\NlTk;
use Wikimedia\Parsoid\Tokens\SelfclosingTagTk;
use Wikimedia\Parsoid\Tokens\TagTk;
use Wikimedia\Parsoid\Tokens\Token;
use Wikimedia\Parsoid\Utils\PHPUtils;
use Wikimedia\Parsoid\Utils\PipelineUtils;
use Wikimedia\Parsoid\Utils\TokenUtils;
use Wikimedia\Parsoid\Wt2Html\PegTokenizer;
use Wikimedia\Parsoid\Wt2Html\TokenTransformManager;
/**
* This class is an attempt to fixup the token stream to reparse strings
* as tokens that failed to parse in the tokenizer because of SOL or
* other constraints OR because tags were being constructed in pieces
* or whatever.
*
* This is a pure hack to improve compatibility with the core parser
* given that we dont have a preprocessor. This will be a grab-bag of
* heuristics and tricks to handle different scenarios.
*/
class TokenStreamPatcher extends TokenHandler {
private PegTokenizer $tokenizer;
/** @var int|null */
private $srcOffset;
private bool $sol;
private array $tokenBuf;
private int $wikiTableNesting;
/** True only for top-level & attribute value pipelines */
private bool $inIndependentParse;
/** @var Token|null */
private $lastConvertedTableCellToken;
/** @var SelfclosingTagTk|null */
private $tplStartToken = null;
/** @var NlTk|null */
private $discardableNlTk = null;
public function __construct( TokenTransformManager $manager, array $options ) {
$newOptions = [ 'tsp' => true ] + $options;
parent::__construct( $manager, $newOptions );
$this->tokenizer = new PegTokenizer( $this->env );
$this->reset();
}
/**
* Resets any internal state for this token handler.
*
* @param array $parseOpts
*/
public function resetState( array $parseOpts ): void {
parent::resetState( $parseOpts );
$this->inIndependentParse = $this->atTopLevel || isset( $this->options['attrExpansion'] );
}
private function reset() {
$this->srcOffset = 0;
$this->sol = true;
$this->tokenBuf = [];
$this->wikiTableNesting = 0;
// This marker tries to track the most recent table-cell token (td/th)
// that was converted to string. For those, we want to get rid
// of their corresponding mw:TSRMarker meta tag.
//
// This marker is set when we convert a td/th token to string
//
// This marker is cleared in one of the following scenarios:
// 1. When we clear a mw:TSRMarker corresponding to the token set earlier
// 2. When we change table nesting
// 3. When we hit a tr/td/th/caption token that wasn't converted to string
$this->lastConvertedTableCellToken = null;
}
/**
* @inheritDoc
*/
public function onNewline( NlTk $token ): ?TokenHandlerResult {
$self = $this;
$this->env->log( 'trace/tsp', $this->pipelineId,
static function () use ( $self, $token ) {
return "(indep=" . ( $self->inIndependentParse ? "yes" : "no " ) .
";sol=" . ( $self->sol ? "yes" : "no " ) . ') ' .
PHPUtils::jsonEncode( $token );
}
);
$this->srcOffset = $token->dataParsoid->tsr->end ?? null;
if ( $this->sol && $this->tplStartToken ) {
// When using core preprocessor, start-of-line start is forced by
// inserting a newline in certain cases (the "T2529 hack"). In the
// legacy parser, the T2529 hack is never applied if the template was
// already at the start of the line (the `!$piece['lineStart']`
// check in Parser::braceSubstitution where T2529 is handled), but
// that context (`$this->sol`) isn't passed through when Parsoid
// invokes the core preprocessor. Thus, when $this->sol is true,
// prepare to (if the following tokens warrant it) remove an unnecessary
// T2529 newline added by the legacy preprocessor.
$this->discardableNlTk = $token;
}
$this->tokenBuf[] = $token;
$this->sol = true;
return new TokenHandlerResult( [] );
}
/**
* @inheritDoc
*/
public function onEnd( EOFTk $token ): ?TokenHandlerResult {
$res = $this->onAny( $token );
$this->reset();
return $res;
}
/**
* Clear start of line info
*/
private function clearSOL() {
// clear tsr and sol flag
$this->srcOffset = null;
$this->sol = false;
}
/**
* Fully reprocess the output tokens from the tokenizer through
* all the other handlers in stage 2.
*
* @param int|false $srcOffset See TokenUtils::shiftTokenTSR, which has b/c for null
* @param array $toks
* @param bool $popEOF
* @return array
*/
private function reprocessTokens( $srcOffset, array $toks, bool $popEOF = false ): array {
// Update tsr
TokenUtils::shiftTokenTSR( $toks, $srcOffset );
$toks = (array)PipelineUtils::processContentInPipeline(
$this->env,
$this->manager->getFrame(),
$toks,
[
'pipelineType' => 'peg-tokens-to-expanded-tokens',
'pipelineOpts' => [],
'sol' => true,
'toplevel' => $this->atTopLevel,
]
);
if ( $popEOF ) {
array_pop( $toks ); // pop EOFTk
}
return $toks;
}
private function convertTokenToString( Token $token ): array {
$da = $token->dataParsoid;
$tsr = $da->tsr ?? null;
if ( $tsr && $tsr->end > $tsr->start ) {
// > will only hold if these are valid numbers
$str = $tsr->substr( $this->manager->getFrame()->getSrcText() );
// sol === false ensures that the pipe will not be parsed as a <td>/listItem again
$toks = $this->tokenizer->tokenizeSync( $str, [ 'sol' => false ] );
return $this->reprocessTokens( $tsr->start, $toks, true );
} elseif ( !empty( $da->autoInsertedStart ) && !empty( $da->autoInsertedEnd ) ) {
return [ '' ];
} else {
// SSS FIXME: What about "!!" and "||"??
switch ( $token->getName() ) {
case 'td':
return [ '|' ];
case 'th':
return [ '!' ];
case 'tr':
return [ '|-' ];
case 'caption':
return [ $token instanceof TagTk ? '|+' : '' ];
case 'table':
return [ $token instanceof EndTagTk ? '|}' : $token ];
case 'listItem':
return [ implode( '', $token->getAttributeV( 'bullets' ) ) ];
}
// No conversion if we get here
return [ $token ];
}
}
/**
* @inheritDoc
*/
public function onAny( $token ): ?TokenHandlerResult {
try {
return $this->onAnyInternal( $token );
} finally {
// Ensure we always clean up discardableNlTk and tplStartToken even
// in the presence of exceptions.
$this->discardableNlTk = null;
if ( $this->tplStartToken !== $token ) {
$this->tplStartToken = null;
}
}
}
/**
* The legacy parser's "T2529 hack" attempts to ensure templates are
* always evaluated in start-of-line context by prepending a newline
* if necessary. However, it is inconsistent: in particular it
* only treats }| : ; # * as SOL-sensitive tokens, neglecting ==
* (headings) and ! | |} (in table context).
*
* If we're using the core preprocessor for template expansion:
* - The core preprocessor as invoked by Parsoid will always insert the
* newline in the "T2529 cases" (even though it's not necessary; Parsoid
* is already in SOL mode) *HOWEVER*
* - As described in ::onNewline() above, the newline insertion is
* /supposed/ to be suppressed if the template was *already*
* at the start of the line. So we need to strip the unnecessarily
* added NlTk to avoid "extra" whitespace in Parsoid's expansion.
* Ex: "{{my-tpl}}" in sol-context which will get expanded to "\n*foo"
* but the "\n" wasn't necessary
*
* If we're in native preprocessor mode:
* - If we are in SOL state, we don't need to add a newline.
* - If we are not in SOL state, we need to insert a newline in 'T2529' cases.
* Ex: "{{my-tpl}}" in sol-context which expands to "*foo" but in
* non-sol context expands to "\n*foo"
*
* @param string $tokenName
*/
private function handleT2529Hack( string $tokenName ): void {
// Core's
if ( $tokenName === 'table' || $tokenName === 'listItem' ) {
// We're in a context when the core preprocessor would apply
// the "T2529 hack" to ensure start-of-line context.
if ( $this->discardableNlTk ) {
// We're using core preprocessor and were already at
// the start of the line, so the core preprocessor wouldn't
// actually have inserted a newline here. Swallow up ours.
array_pop( $this->tokenBuf );
} elseif ( !$this->sol &&
$this->tplStartToken &&
$this->env->nativeTemplateExpansionEnabled()
) {
// Native preprocessor; add a newline in "T2529 cases"
// for correct whitespace. (Remember that this only happens
// if we weren't already at the start of the line.)
// Add a newline & force SOL
$this->tokenBuf[] = new NlTk( null );
$this->sol = true;
}
}
}
/**
* @param mixed $token
* @return ?TokenHandlerResult
*/
public function onAnyInternal( $token ): ?TokenHandlerResult {
$self = $this;
$this->env->log( 'trace/tsp', $this->pipelineId,
static function () use ( $self, $token ) {
return "(indep=" . ( $self->inIndependentParse ? "yes" : "no " ) .
";sol=" . ( $self->sol ? "yes" : "no " ) . ') ' .
PHPUtils::jsonEncode( $token );
}
);
$tokens = [ $token ];
$tc = TokenUtils::getTokenType( $token );
switch ( $tc ) {
case 'string':
// While we are buffering newlines to suppress them
// in case we see a category, buffer all intervening
// white-space as well.
if ( count( $this->tokenBuf ) > 0 && preg_match( '/^\s*$/D', $token ) ) {
$this->tokenBuf[] = $token;
return new TokenHandlerResult( [] );
}
// This is only applicable where we use Parsoid's (broken) native preprocessor.
// This supports scenarios like "{{1x|*bar}}". When "{{{1}}}" is tokenized
// "*bar" isn't available and so won't become a list.
// FIXME: {{1x|1===foo==}} will still be broken. So, this fix below is somewhat
// independent of T2529 for our broken preprocessor but we are restricting the
// fix to T2529.
$T2529hack = false;
if ( $this->env->nativeTemplateExpansionEnabled() &&
$this->tplStartToken &&
preg_match( '/^(?:{\\||[:;#*])/', $token )
) {
// Add a newline & force SOL
$T2529hack = true;
// Remove newline insertion in the core preprocessor
// only occurs if we weren't already at the start of
// the line (see discussion in ::onNewline() above).
if ( !$this->sol ) {
$this->tokenBuf[] = new NlTk( null );
$this->sol = true;
}
}
if ( $this->sol ) {
// Attempt to match "{|" after a newline and convert
// it to a table token.
if ( $this->inIndependentParse && str_starts_with( $token, '{|' ) ) {
// Reparse string with the 'table_start_tag' rule
// and fully reprocess them.
$retoks = $this->tokenizer->tokenizeAs( $token, 'table_start_tag', /* sol */true );
if ( $retoks === false ) {
// XXX: The string begins with table start syntax,
// we really shouldn't be here. Anything else on the
// line would get swallowed up as attributes.
$this->env->log( 'error', 'Failed to tokenize table start tag.' );
$this->clearSOL();
} else {
$tokens = $this->reprocessTokens( $this->srcOffset, $retoks );
$this->wikiTableNesting++;
$this->lastConvertedTableCellToken = null;
}
} elseif ( $this->inIndependentParse && $T2529hack ) { // {| has been handled above
$retoks = $this->tokenizer->tokenizeAs( $token, 'list_item', /* sol */true );
if ( $retoks === false ) {
$this->env->log( 'error', 'Failed to tokenize list item.' );
$this->clearSOL();
} else {
$tokens = $this->reprocessTokens( $this->srcOffset, $retoks );
}
} elseif ( preg_match( '/^\s*$/D', $token ) ) {
// White-space doesn't change SOL state
// Update srcOffset
$this->srcOffset += strlen( $token );
} else {
$this->clearSOL();
}
} else {
$this->clearSOL();
}
break;
case 'CommentTk':
// Comments don't change SOL state
// Update srcOffset
$this->srcOffset = $token->dataParsoid->tsr->end ?? null;
break;
case 'SelfclosingTagTk':
if ( $token->getName() === 'meta' && ( $token->dataParsoid->stx ?? '' ) !== 'html' ) {
if ( TokenUtils::hasTypeOf( $token, 'mw:Transclusion' ) ) {
$this->tplStartToken = $token;
}
$this->srcOffset = $token->dataParsoid->tsr->end ?? null;
if ( count( $this->tokenBuf ) > 0 &&
TokenUtils::hasTypeOf( $token, 'mw:Transclusion' )
) {
// If we have buffered newlines, we might very well encounter
// a category link, so continue buffering.
$this->tokenBuf[] = $token;
return new TokenHandlerResult( [] );
}
} elseif ( TokenUtils::isSolTransparentLinkTag( $token ) ) {
// Replace buffered newline & whitespace tokens with mw:EmptyLine
// meta-tokens. This tunnels them through the rest of the transformations
// without affecting them. During HTML building, they are expanded
// back to newlines / whitespace.
$n = count( $this->tokenBuf );
if ( $n > 0 ) {
$i = 0;
while ( $i < $n &&
!( $this->tokenBuf[$i] instanceof SelfclosingTagTk )
) {
$i++;
}
$dp = new DataParsoid;
$dp->tokens = array_slice( $this->tokenBuf, 0, $i );
$toks = [
new SelfclosingTagTk( 'meta',
[ new KV( 'typeof', 'mw:EmptyLine' ) ],
$dp
)
];
if ( $i < $n ) {
$toks[] = $this->tokenBuf[$i];
if ( $i + 1 < $n ) {
$dp = new DataParsoid;
$dp->tokens = array_slice( $this->tokenBuf, $i + 1 );
$toks[] = new SelfclosingTagTk( 'meta',
[ new KV( 'typeof', 'mw:EmptyLine' ) ],
$dp
);
}
}
$tokens = array_merge( $toks, $tokens );
$this->tokenBuf = [];
}
$this->clearSOL();
} else {
$this->clearSOL();
}
break;
case 'TagTk':
if ( $this->inIndependentParse && !TokenUtils::isHTMLTag( $token ) ) {
$tokenName = $token->getName();
$this->handleT2529Hack( $tokenName );
if ( $tokenName === 'listItem' && isset( $this->options['attrExpansion'] ) ) {
// Convert list items back to bullet wikitext in attribute context
$tokens = $this->convertTokenToString( $token );
} elseif ( $tokenName === 'table' ) {
$this->lastConvertedTableCellToken = null;
$this->wikiTableNesting++;
} elseif ( in_array( $tokenName, [ 'td', 'th', 'tr', 'caption' ], true ) ) {
if ( $this->wikiTableNesting === 0 ) {
if ( $token->getName() === 'td' || $token->getName() === 'th' ) {
$this->lastConvertedTableCellToken = $token;
}
$tokens = $this->convertTokenToString( $token );
} else {
$this->lastConvertedTableCellToken = null;
}
}
}
$this->clearSOL();
break;
case 'EndTagTk':
if ( $this->inIndependentParse && !TokenUtils::isHTMLTag( $token ) ) {
if ( $this->wikiTableNesting > 0 ) {
if ( $token->getName() === 'table' ) {
$this->lastConvertedTableCellToken = null;
$this->wikiTableNesting--;
}
} elseif ( $token->getName() === 'table' || $token->getName() === 'caption' ) {
// Convert this to "|}"
$tokens = $this->convertTokenToString( $token );
}
}
$this->clearSOL();
break;
default:
break;
}
// Emit buffered newlines (and a transclusion meta-token, if any)
if ( count( $this->tokenBuf ) > 0 ) {
$tokens = array_merge( $this->tokenBuf, $tokens );
$this->tokenBuf = [];
}
return new TokenHandlerResult( $tokens );
}
}

File Metadata

Mime Type
text/x-php
Expires
Sat, Nov 21, 22:56 (1 d, 23 h)
Storage Engine
local-disk
Storage Format
Raw Data
Storage Handle
90/a5/92762b3c517e3c32d1ec23912169
Default Alt Text
TokenStreamPatcher.php (15 KB)

Event Timeline