]+about="[^"]*"[^>]*>/u', '', $out );
$out = preg_replace( '#(\s)\s*\s*#u', '$1', $out );
$out = preg_replace( '#\s*#u', '', $out );
$out = preg_replace( '#(href=")(?:\.?\./)+#u', '$1', $out );
// replace unnecessary URL escaping
$out = preg_replace_callback( '/ href="[^"]*"/u', static function ( $m ) {
return Utils::decodeURI( $m[0] );
}, $out );
// strip thumbnail size prefixes
return preg_replace(
'#(src="[^"]*?)/thumb(/[0-9a-f]/[0-9a-f]{2}/[^/]+)/[0-9]+px-[^"/]+(?=")#u', '$1$2',
$out
);
}
/**
* Strip Parsoid ID attributes (id="mwXX", used to associate NodeData) from an HTML string
* @param string $s
* @return string
*/
public static function stripParsoidIds( string $s ): string {
return preg_replace( '/ id="mw([-\w]{2,})"/u', '', $s );
}
private static function cleanSpans(
Node $node, ?string $stripSpanTypeof
): void {
if ( !$stripSpanTypeof ) {
return;
}
$child = null;
$next = null;
for ( $child = $node->firstChild; $child; $child = $next ) {
$next = $child->nextSibling;
if ( $child instanceof Element && DOMCompat::nodeName( $child ) === 'span' &&
preg_match( $stripSpanTypeof, DOMCompat::getAttribute( $child, 'typeof' ) ?? '' )
) {
self::unwrapSpan( $node, $child, $stripSpanTypeof );
}
}
}
private static function unwrapSpan(
Node $parent, Node $node, ?string $stripSpanTypeof
): void {
// first recurse to unwrap any spans in the immediate children.
self::cleanSpans( $node, $stripSpanTypeof );
// now unwrap this span.
DOMUtils::migrateChildren( $node, $parent, $node );
$parent->removeChild( $node );
}
private static function newlineAround( ?Node $node ): bool {
return $node && preg_match(
'/^(body|caption|div|dd|dt|li|p|table|tr|td|th|tbody|dl|ol|ul|h[1-6])$/D',
DOMCompat::nodeName( $node )
);
}
private static function normalizeIEWVisitor(
Node $node, array $opts
): Node {
$child = null;
$next = null;
$prev = null;
if ( DOMCompat::nodeName( $node ) === 'pre' ) {
// Preserve newlines in tags
$opts['inPRE'] = true;
}
if ( !$opts['preserveIEW'] && $node instanceof Text ) {
if ( !$opts['inPRE'] ) {
$node->data = preg_replace( '/\s+/u', ' ', $node->data );
}
if ( $opts['stripLeadingWS'] ) {
$node->data = preg_replace( '/^\s+/u', '', $node->data, 1 );
}
if ( $opts['stripTrailingWS'] ) {
$node->data = preg_replace( '/\s+$/uD', '', $node->data, 1 );
}
}
// unwrap certain SPAN nodes
self::cleanSpans( $node, $opts['stripSpanTypeof'] );
// now remove comment nodes
if ( !$opts['parsoidOnly'] ) {
for ( $child = $node->firstChild; $child; $child = $next ) {
$next = $child->nextSibling;
if ( $child instanceof Comment ) {
$node->removeChild( $child );
}
}
}
// reassemble text nodes split by a comment or span, if necessary
if ( $node instanceof Element ) {
DOMCompat::normalize( $node );
}
// now recurse.
if ( DOMCompat::nodeName( $node ) === 'pre' ) {
// hack, since PHP adds a newline before
$opts['stripLeadingWS'] = false;
$opts['stripTrailingWS'] = true;
} elseif (
DOMCompat::nodeName( $node ) === 'span' &&
DOMUtils::matchTypeOf( $node, '/^mw:/' )
) {
// SPAN is transparent; pass the strip parameters down to kids
} else {
$opts['stripLeadingWS'] = $opts['stripTrailingWS'] = self::newlineAround( $node );
}
$child = $node->firstChild;
// Skip over the empty mw:FallbackId and strip leading WS
// on the other side of it.
if ( $child && DOMUtils::isHeading( $node ) && WTUtils::isFallbackIdSpan( $child ) ) {
$child = $child->nextSibling;
}
for ( ; $child; $child = $next ) {
$next = $child->nextSibling;
$newOpts = $opts;
$newOpts['stripTrailingWS'] = $opts['stripTrailingWS'] && !$child->nextSibling;
self::normalizeIEWVisitor( $child, $newOpts );
$opts['stripLeadingWS'] = false;
}
if ( $opts['inPRE'] || $opts['preserveIEW'] ) {
return $node;
}
// now add newlines around appropriate nodes.
for ( $child = $node->firstChild; $child; $child = $next ) {
$prev = $child->previousSibling;
$next = $child->nextSibling;
if ( self::newlineAround( $child ) ) {
if ( $prev instanceof Text ) {
$prev->data = preg_replace( '/\s*$/uD', "\n", $prev->data, 1 );
} else {
$prev = $node->ownerDocument->createTextNode( "\n" );
$node->insertBefore( $prev, $child );
}
if ( $next instanceof Text ) {
$next->data = preg_replace( '/^\s*/u', "\n", $next->data, 1 );
} else {
$next = $node->ownerDocument->createTextNode( "\n" );
$node->insertBefore( $next, $child->nextSibling );
}
}
}
return $node;
}
/**
* Normalize newlines in IEW to spaces instead.
*
* @param Element $body The document body node to normalize.
* @param ?string $stripSpanTypeof Regular expression to strip typeof attributes
* @param bool $parsoidOnly
* @param bool $preserveIEW
* @return Element
*/
public static function unwrapSpansAndNormalizeIEW(
Element $body, ?string $stripSpanTypeof = null, bool $parsoidOnly = false, bool $preserveIEW = false
): Element {
$opts = [
'preserveIEW' => $preserveIEW,
'parsoidOnly' => $parsoidOnly,
'stripSpanTypeof' => $stripSpanTypeof,
'stripLeadingWS' => true,
'stripTrailingWS' => true,
'inPRE' => false
];
// clone body first, since we're going to destructively mutate it.
// @phan-suppress-next-line PhanTypeMismatchReturnSuperType
return self::normalizeIEWVisitor( $body->cloneNode( true ), $opts );
}
/**
* Strip some php output we aren't generating.
*
* @param string $html
* @return string
*/
public static function normalizePhpOutput( string $html ): string {
return preg_replace(
// do not expect section editing for now
'/]+class="mw-headline"[^>]*>(.*?)<\/span> '
. '*('
. '\[<\/span>.*?\]<\/span><\/span>)?/u',
'$1',
$html
);
}
/**
* Normalize the expected parser output by parsing it using a HTML5 parser and
* re-serializing it to HTML. Ideally, the parser would normalize inter-tag
* whitespace for us. For now, we fake that by simply stripping all newlines.
*
* @param string $source
* @return string
*/
public static function normalizeHTML( string $source ): string {
try {
$body = self::unwrapSpansAndNormalizeIEW( DOMCompat::getBody( DOMUtils::parseHTML( $source ) ) );
$html = ContentUtils::toXML( $body, [ 'innerXML' => true ] );
// a few things we ignore for now..
// .replace(/\/wiki\/Main_Page/g, 'Main Page')
// do not expect a toc for now
$html = preg_replace(
'/]+?id="toc"[^>]*>\s*]*>[\s\S]+?<\/div>[\s\S]+?<\/div>\s*/u',
'',
$html );
$html = self::normalizePhpOutput( $html );
// remove empty span tags
$html = preg_replace( '/(\s)
\s*<\/span>\s*/u', '$1', $html );
$html = preg_replace( '/\s*<\/span>/u', '', $html );
// general class and titles, typically on links
$html = preg_replace( '/ (class|rel|about|typeof)="[^"]*"/', '', $html );
// strip red link markup, we do not check if a page exists yet
$html = preg_replace(
"#/index.php\\?title=([^']+?)&action=edit&redlink=1#", '/wiki/$1', $html );
// strip red link title info
$html = preg_replace(
"/ \\((?:page does not exist|encara no existeix|bet ele jaratılmaǵan|lonkásá ezalí tɛ̂)\\)/",
'', $html );
// the expected html has some extra space in tags, strip it
$html = preg_replace( '//', '">', $html );
// parsoid always add a page name to lonely fragments
$html = preg_replace( '/href="#/', 'href="Main Page#', $html );
// replace unnecessary URL escaping
$html = preg_replace_callback( '/ href="[^"]*"/',
static function ( $m ) {
return Utils::decodeURI( $m[0] );
},
$html );
// strip empty spans
$html = preg_replace( '#(\s)\s*\s*#u', '$1', $html );
return preg_replace( '#\s*#u', '', $html );
} catch ( Exception $e ) {
error_log( 'normalizeHTML failed on' . $source . ' with the following error: ' . $e );
return $source;
}
}
/**
* @param string $string
* @param string $color
* @param bool $inverse
* @return string
* @suppress PhanUndeclaredClassMethod
* @suppress UnusedSuppression
*/
public static function colorString(
string $string, string $color, bool $inverse = false
): string {
if ( $inverse ) {
$color = [ $color, 'reverse' ];
}
if ( !self::$consoleColor ) {
// Attempt to instantiate this class to determine if the
// (optional) php-console-color library is installed.
try {
self::$consoleColor = new \PHP_Parallel_Lint\PhpConsoleColor\ConsoleColor();
} catch ( Error $e ) {
/* fall back to no-color mode */
}
}
if ( self::$consoleColor && self::$consoleColor->isSupported() ) {
return self::$consoleColor->apply( $color, $string );
} else {
return $string;
}
}
}