Page Menu
Home
WickedGov Phorge
Search
Configure Global Search
Log In
Files
F7410375
TokenStreamPatcher.php
No One
Temporary
Actions
Download File
Edit File
Delete File
View Transforms
Subscribe
Flag For Later
Award Token
Size
15 KB
Referenced Files
None
Subscribers
None
TokenStreamPatcher.php
View Options
<?php
declare
(
strict_types
=
1
);
namespace
Wikimedia\Parsoid\Wt2Html\TT
;
use
Wikimedia\Parsoid\NodeData\DataParsoid
;
use
Wikimedia\Parsoid\Tokens\EndTagTk
;
use
Wikimedia\Parsoid\Tokens\EOFTk
;
use
Wikimedia\Parsoid\Tokens\KV
;
use
Wikimedia\Parsoid\Tokens\NlTk
;
use
Wikimedia\Parsoid\Tokens\SelfclosingTagTk
;
use
Wikimedia\Parsoid\Tokens\TagTk
;
use
Wikimedia\Parsoid\Tokens\Token
;
use
Wikimedia\Parsoid\Utils\PHPUtils
;
use
Wikimedia\Parsoid\Utils\PipelineUtils
;
use
Wikimedia\Parsoid\Utils\TokenUtils
;
use
Wikimedia\Parsoid\Wt2Html\PegTokenizer
;
use
Wikimedia\Parsoid\Wt2Html\TokenTransformManager
;
/**
* This class is an attempt to fixup the token stream to reparse strings
* as tokens that failed to parse in the tokenizer because of SOL or
* other constraints OR because tags were being constructed in pieces
* or whatever.
*
* This is a pure hack to improve compatibility with the core parser
* given that we dont have a preprocessor. This will be a grab-bag of
* heuristics and tricks to handle different scenarios.
*/
class
TokenStreamPatcher
extends
TokenHandler
{
private
PegTokenizer
$tokenizer
;
/** @var int|null */
private
$srcOffset
;
private
bool
$sol
;
private
array
$tokenBuf
;
private
int
$wikiTableNesting
;
/** True only for top-level & attribute value pipelines */
private
bool
$inIndependentParse
;
/** @var Token|null */
private
$lastConvertedTableCellToken
;
/** @var SelfclosingTagTk|null */
private
$tplStartToken
=
null
;
/** @var NlTk|null */
private
$discardableNlTk
=
null
;
public
function
__construct
(
TokenTransformManager
$manager
,
array
$options
)
{
$newOptions
=
[
'tsp'
=>
true
]
+
$options
;
parent
::
__construct
(
$manager
,
$newOptions
);
$this
->
tokenizer
=
new
PegTokenizer
(
$this
->
env
);
$this
->
reset
();
}
/**
* Resets any internal state for this token handler.
*
* @param array $parseOpts
*/
public
function
resetState
(
array
$parseOpts
):
void
{
parent
::
resetState
(
$parseOpts
);
$this
->
inIndependentParse
=
$this
->
atTopLevel
||
isset
(
$this
->
options
[
'attrExpansion'
]
);
}
private
function
reset
()
{
$this
->
srcOffset
=
0
;
$this
->
sol
=
true
;
$this
->
tokenBuf
=
[];
$this
->
wikiTableNesting
=
0
;
// This marker tries to track the most recent table-cell token (td/th)
// that was converted to string. For those, we want to get rid
// of their corresponding mw:TSRMarker meta tag.
//
// This marker is set when we convert a td/th token to string
//
// This marker is cleared in one of the following scenarios:
// 1. When we clear a mw:TSRMarker corresponding to the token set earlier
// 2. When we change table nesting
// 3. When we hit a tr/td/th/caption token that wasn't converted to string
$this
->
lastConvertedTableCellToken
=
null
;
}
/**
* @inheritDoc
*/
public
function
onNewline
(
NlTk
$token
):
?
TokenHandlerResult
{
$self
=
$this
;
$this
->
env
->
log
(
'trace/tsp'
,
$this
->
pipelineId
,
static
function
()
use
(
$self
,
$token
)
{
return
"(indep="
.
(
$self
->
inIndependentParse
?
"yes"
:
"no "
)
.
";sol="
.
(
$self
->
sol
?
"yes"
:
"no "
)
.
') '
.
PHPUtils
::
jsonEncode
(
$token
);
}
);
$this
->
srcOffset
=
$token
->
dataParsoid
->
tsr
->
end
??
null
;
if
(
$this
->
sol
&&
$this
->
tplStartToken
)
{
// When using core preprocessor, start-of-line start is forced by
// inserting a newline in certain cases (the "T2529 hack"). In the
// legacy parser, the T2529 hack is never applied if the template was
// already at the start of the line (the `!$piece['lineStart']`
// check in Parser::braceSubstitution where T2529 is handled), but
// that context (`$this->sol`) isn't passed through when Parsoid
// invokes the core preprocessor. Thus, when $this->sol is true,
// prepare to (if the following tokens warrant it) remove an unnecessary
// T2529 newline added by the legacy preprocessor.
$this
->
discardableNlTk
=
$token
;
}
$this
->
tokenBuf
[]
=
$token
;
$this
->
sol
=
true
;
return
new
TokenHandlerResult
(
[]
);
}
/**
* @inheritDoc
*/
public
function
onEnd
(
EOFTk
$token
):
?
TokenHandlerResult
{
$res
=
$this
->
onAny
(
$token
);
$this
->
reset
();
return
$res
;
}
/**
* Clear start of line info
*/
private
function
clearSOL
()
{
// clear tsr and sol flag
$this
->
srcOffset
=
null
;
$this
->
sol
=
false
;
}
/**
* Fully reprocess the output tokens from the tokenizer through
* all the other handlers in stage 2.
*
* @param int|false $srcOffset See TokenUtils::shiftTokenTSR, which has b/c for null
* @param array $toks
* @param bool $popEOF
* @return array
*/
private
function
reprocessTokens
(
$srcOffset
,
array
$toks
,
bool
$popEOF
=
false
):
array
{
// Update tsr
TokenUtils
::
shiftTokenTSR
(
$toks
,
$srcOffset
);
$toks
=
(
array
)
PipelineUtils
::
processContentInPipeline
(
$this
->
env
,
$this
->
manager
->
getFrame
(),
$toks
,
[
'pipelineType'
=>
'peg-tokens-to-expanded-tokens'
,
'pipelineOpts'
=>
[],
'sol'
=>
true
,
'toplevel'
=>
$this
->
atTopLevel
,
]
);
if
(
$popEOF
)
{
array_pop
(
$toks
);
// pop EOFTk
}
return
$toks
;
}
private
function
convertTokenToString
(
Token
$token
):
array
{
$da
=
$token
->
dataParsoid
;
$tsr
=
$da
->
tsr
??
null
;
if
(
$tsr
&&
$tsr
->
end
>
$tsr
->
start
)
{
// > will only hold if these are valid numbers
$str
=
$tsr
->
substr
(
$this
->
manager
->
getFrame
()->
getSrcText
()
);
// sol === false ensures that the pipe will not be parsed as a <td>/listItem again
$toks
=
$this
->
tokenizer
->
tokenizeSync
(
$str
,
[
'sol'
=>
false
]
);
return
$this
->
reprocessTokens
(
$tsr
->
start
,
$toks
,
true
);
}
elseif
(
!
empty
(
$da
->
autoInsertedStart
)
&&
!
empty
(
$da
->
autoInsertedEnd
)
)
{
return
[
''
];
}
else
{
// SSS FIXME: What about "!!" and "||"??
switch
(
$token
->
getName
()
)
{
case
'td'
:
return
[
'|'
];
case
'th'
:
return
[
'!'
];
case
'tr'
:
return
[
'|-'
];
case
'caption'
:
return
[
$token
instanceof
TagTk
?
'|+'
:
''
];
case
'table'
:
return
[
$token
instanceof
EndTagTk
?
'|}'
:
$token
];
case
'listItem'
:
return
[
implode
(
''
,
$token
->
getAttributeV
(
'bullets'
)
)
];
}
// No conversion if we get here
return
[
$token
];
}
}
/**
* @inheritDoc
*/
public
function
onAny
(
$token
):
?
TokenHandlerResult
{
try
{
return
$this
->
onAnyInternal
(
$token
);
}
finally
{
// Ensure we always clean up discardableNlTk and tplStartToken even
// in the presence of exceptions.
$this
->
discardableNlTk
=
null
;
if
(
$this
->
tplStartToken
!==
$token
)
{
$this
->
tplStartToken
=
null
;
}
}
}
/**
* The legacy parser's "T2529 hack" attempts to ensure templates are
* always evaluated in start-of-line context by prepending a newline
* if necessary. However, it is inconsistent: in particular it
* only treats }| : ; # * as SOL-sensitive tokens, neglecting ==
* (headings) and ! | |} (in table context).
*
* If we're using the core preprocessor for template expansion:
* - The core preprocessor as invoked by Parsoid will always insert the
* newline in the "T2529 cases" (even though it's not necessary; Parsoid
* is already in SOL mode) *HOWEVER*
* - As described in ::onNewline() above, the newline insertion is
* /supposed/ to be suppressed if the template was *already*
* at the start of the line. So we need to strip the unnecessarily
* added NlTk to avoid "extra" whitespace in Parsoid's expansion.
* Ex: "{{my-tpl}}" in sol-context which will get expanded to "\n*foo"
* but the "\n" wasn't necessary
*
* If we're in native preprocessor mode:
* - If we are in SOL state, we don't need to add a newline.
* - If we are not in SOL state, we need to insert a newline in 'T2529' cases.
* Ex: "{{my-tpl}}" in sol-context which expands to "*foo" but in
* non-sol context expands to "\n*foo"
*
* @param string $tokenName
*/
private
function
handleT2529Hack
(
string
$tokenName
):
void
{
// Core's
if
(
$tokenName
===
'table'
||
$tokenName
===
'listItem'
)
{
// We're in a context when the core preprocessor would apply
// the "T2529 hack" to ensure start-of-line context.
if
(
$this
->
discardableNlTk
)
{
// We're using core preprocessor and were already at
// the start of the line, so the core preprocessor wouldn't
// actually have inserted a newline here. Swallow up ours.
array_pop
(
$this
->
tokenBuf
);
}
elseif
(
!
$this
->
sol
&&
$this
->
tplStartToken
&&
$this
->
env
->
nativeTemplateExpansionEnabled
()
)
{
// Native preprocessor; add a newline in "T2529 cases"
// for correct whitespace. (Remember that this only happens
// if we weren't already at the start of the line.)
// Add a newline & force SOL
$this
->
tokenBuf
[]
=
new
NlTk
(
null
);
$this
->
sol
=
true
;
}
}
}
/**
* @param mixed $token
* @return ?TokenHandlerResult
*/
public
function
onAnyInternal
(
$token
):
?
TokenHandlerResult
{
$self
=
$this
;
$this
->
env
->
log
(
'trace/tsp'
,
$this
->
pipelineId
,
static
function
()
use
(
$self
,
$token
)
{
return
"(indep="
.
(
$self
->
inIndependentParse
?
"yes"
:
"no "
)
.
";sol="
.
(
$self
->
sol
?
"yes"
:
"no "
)
.
') '
.
PHPUtils
::
jsonEncode
(
$token
);
}
);
$tokens
=
[
$token
];
$tc
=
TokenUtils
::
getTokenType
(
$token
);
switch
(
$tc
)
{
case
'string'
:
// While we are buffering newlines to suppress them
// in case we see a category, buffer all intervening
// white-space as well.
if
(
count
(
$this
->
tokenBuf
)
>
0
&&
preg_match
(
'/^
\s
*$/D'
,
$token
)
)
{
$this
->
tokenBuf
[]
=
$token
;
return
new
TokenHandlerResult
(
[]
);
}
// This is only applicable where we use Parsoid's (broken) native preprocessor.
// This supports scenarios like "{{1x|*bar}}". When "{{{1}}}" is tokenized
// "*bar" isn't available and so won't become a list.
// FIXME: {{1x|1===foo==}} will still be broken. So, this fix below is somewhat
// independent of T2529 for our broken preprocessor but we are restricting the
// fix to T2529.
$T2529hack
=
false
;
if
(
$this
->
env
->
nativeTemplateExpansionEnabled
()
&&
$this
->
tplStartToken
&&
preg_match
(
'/^(?:{
\\
||[:;#*])/'
,
$token
)
)
{
// Add a newline & force SOL
$T2529hack
=
true
;
// Remove newline insertion in the core preprocessor
// only occurs if we weren't already at the start of
// the line (see discussion in ::onNewline() above).
if
(
!
$this
->
sol
)
{
$this
->
tokenBuf
[]
=
new
NlTk
(
null
);
$this
->
sol
=
true
;
}
}
if
(
$this
->
sol
)
{
// Attempt to match "{|" after a newline and convert
// it to a table token.
if
(
$this
->
inIndependentParse
&&
str_starts_with
(
$token
,
'{|'
)
)
{
// Reparse string with the 'table_start_tag' rule
// and fully reprocess them.
$retoks
=
$this
->
tokenizer
->
tokenizeAs
(
$token
,
'table_start_tag'
,
/* sol */
true
);
if
(
$retoks
===
false
)
{
// XXX: The string begins with table start syntax,
// we really shouldn't be here. Anything else on the
// line would get swallowed up as attributes.
$this
->
env
->
log
(
'error'
,
'Failed to tokenize table start tag.'
);
$this
->
clearSOL
();
}
else
{
$tokens
=
$this
->
reprocessTokens
(
$this
->
srcOffset
,
$retoks
);
$this
->
wikiTableNesting
++;
$this
->
lastConvertedTableCellToken
=
null
;
}
}
elseif
(
$this
->
inIndependentParse
&&
$T2529hack
)
{
// {| has been handled above
$retoks
=
$this
->
tokenizer
->
tokenizeAs
(
$token
,
'list_item'
,
/* sol */
true
);
if
(
$retoks
===
false
)
{
$this
->
env
->
log
(
'error'
,
'Failed to tokenize list item.'
);
$this
->
clearSOL
();
}
else
{
$tokens
=
$this
->
reprocessTokens
(
$this
->
srcOffset
,
$retoks
);
}
}
elseif
(
preg_match
(
'/^
\s
*$/D'
,
$token
)
)
{
// White-space doesn't change SOL state
// Update srcOffset
$this
->
srcOffset
+=
strlen
(
$token
);
}
else
{
$this
->
clearSOL
();
}
}
else
{
$this
->
clearSOL
();
}
break
;
case
'CommentTk'
:
// Comments don't change SOL state
// Update srcOffset
$this
->
srcOffset
=
$token
->
dataParsoid
->
tsr
->
end
??
null
;
break
;
case
'SelfclosingTagTk'
:
if
(
$token
->
getName
()
===
'meta'
&&
(
$token
->
dataParsoid
->
stx
??
''
)
!==
'html'
)
{
if
(
TokenUtils
::
hasTypeOf
(
$token
,
'mw:Transclusion'
)
)
{
$this
->
tplStartToken
=
$token
;
}
$this
->
srcOffset
=
$token
->
dataParsoid
->
tsr
->
end
??
null
;
if
(
count
(
$this
->
tokenBuf
)
>
0
&&
TokenUtils
::
hasTypeOf
(
$token
,
'mw:Transclusion'
)
)
{
// If we have buffered newlines, we might very well encounter
// a category link, so continue buffering.
$this
->
tokenBuf
[]
=
$token
;
return
new
TokenHandlerResult
(
[]
);
}
}
elseif
(
TokenUtils
::
isSolTransparentLinkTag
(
$token
)
)
{
// Replace buffered newline & whitespace tokens with mw:EmptyLine
// meta-tokens. This tunnels them through the rest of the transformations
// without affecting them. During HTML building, they are expanded
// back to newlines / whitespace.
$n
=
count
(
$this
->
tokenBuf
);
if
(
$n
>
0
)
{
$i
=
0
;
while
(
$i
<
$n
&&
!(
$this
->
tokenBuf
[
$i
]
instanceof
SelfclosingTagTk
)
)
{
$i
++;
}
$dp
=
new
DataParsoid
;
$dp
->
tokens
=
array_slice
(
$this
->
tokenBuf
,
0
,
$i
);
$toks
=
[
new
SelfclosingTagTk
(
'meta'
,
[
new
KV
(
'typeof'
,
'mw:EmptyLine'
)
],
$dp
)
];
if
(
$i
<
$n
)
{
$toks
[]
=
$this
->
tokenBuf
[
$i
];
if
(
$i
+
1
<
$n
)
{
$dp
=
new
DataParsoid
;
$dp
->
tokens
=
array_slice
(
$this
->
tokenBuf
,
$i
+
1
);
$toks
[]
=
new
SelfclosingTagTk
(
'meta'
,
[
new
KV
(
'typeof'
,
'mw:EmptyLine'
)
],
$dp
);
}
}
$tokens
=
array_merge
(
$toks
,
$tokens
);
$this
->
tokenBuf
=
[];
}
$this
->
clearSOL
();
}
else
{
$this
->
clearSOL
();
}
break
;
case
'TagTk'
:
if
(
$this
->
inIndependentParse
&&
!
TokenUtils
::
isHTMLTag
(
$token
)
)
{
$tokenName
=
$token
->
getName
();
$this
->
handleT2529Hack
(
$tokenName
);
if
(
$tokenName
===
'listItem'
&&
isset
(
$this
->
options
[
'attrExpansion'
]
)
)
{
// Convert list items back to bullet wikitext in attribute context
$tokens
=
$this
->
convertTokenToString
(
$token
);
}
elseif
(
$tokenName
===
'table'
)
{
$this
->
lastConvertedTableCellToken
=
null
;
$this
->
wikiTableNesting
++;
}
elseif
(
in_array
(
$tokenName
,
[
'td'
,
'th'
,
'tr'
,
'caption'
],
true
)
)
{
if
(
$this
->
wikiTableNesting
===
0
)
{
if
(
$token
->
getName
()
===
'td'
||
$token
->
getName
()
===
'th'
)
{
$this
->
lastConvertedTableCellToken
=
$token
;
}
$tokens
=
$this
->
convertTokenToString
(
$token
);
}
else
{
$this
->
lastConvertedTableCellToken
=
null
;
}
}
}
$this
->
clearSOL
();
break
;
case
'EndTagTk'
:
if
(
$this
->
inIndependentParse
&&
!
TokenUtils
::
isHTMLTag
(
$token
)
)
{
if
(
$this
->
wikiTableNesting
>
0
)
{
if
(
$token
->
getName
()
===
'table'
)
{
$this
->
lastConvertedTableCellToken
=
null
;
$this
->
wikiTableNesting
--;
}
}
elseif
(
$token
->
getName
()
===
'table'
||
$token
->
getName
()
===
'caption'
)
{
// Convert this to "|}"
$tokens
=
$this
->
convertTokenToString
(
$token
);
}
}
$this
->
clearSOL
();
break
;
default
:
break
;
}
// Emit buffered newlines (and a transclusion meta-token, if any)
if
(
count
(
$this
->
tokenBuf
)
>
0
)
{
$tokens
=
array_merge
(
$this
->
tokenBuf
,
$tokens
);
$this
->
tokenBuf
=
[];
}
return
new
TokenHandlerResult
(
$tokens
);
}
}
File Metadata
Details
Attached
Mime Type
text/x-php
Expires
Sat, Nov 21, 22:56 (1 d, 23 h)
Storage Engine
local-disk
Storage Format
Raw Data
Storage Handle
90/a5/92762b3c517e3c32d1ec23912169
Default Alt Text
TokenStreamPatcher.php (15 KB)
Attached To
Mode
rMWPROD MediaWiki Production
Attached
Detach File
Event Timeline
Log In to Comment