PhpTokenizer.php
No OneTemporary
Actions

Size

19 KB

Referenced Files

None

Subscribers

None

PhpTokenizer.php
View Options

	<?php
	/*---------------------------------------------------------------------------------------------
	* Copyright (c) Microsoft Corporation. All rights reserved.
	* Licensed under the MIT License. See License.txt in the project root for license information.
	--------------------------------------------------------------------------------------------/

	namespace Microsoft\PhpParser;

	// If this predates PHP 7.4, T_COALESCE_EQUAL and T_FN are unavailable.
	// The replacement value is arbitrary - it just has to be different from other values of token constants.
	define(__NAMESPACE__ . '\T_COALESCE_EQUAL', defined('T_COALESCE_EQUAL') ? constant('T_COALESCE_EQUAL') : 'T_COALESCE_EQUAL');
	define(__NAMESPACE__ . '\T_FN', defined('T_FN') ? constant('T_FN') : 'T_FN');
	// If this predates PHP 8.0, T_MATCH is unavailable. The replacement value is arbitrary - it just has to be different from other values of token constants.
	define(__NAMESPACE__ . '\T_MATCH', defined('T_MATCH') ? constant('T_MATCH') : 'T_MATCH');
	define(__NAMESPACE__ . '\T_NULLSAFE_OBJECT_OPERATOR', defined('T_NULLSAFE_OBJECT_OPERATOR') ? constant('T_NULLSAFE_OBJECT_OPERATOR') : 'T_NULLSAFE_OBJECT_OPERATOR');
	define(__NAMESPACE__ . '\T_ATTRIBUTE', defined('T_ATTRIBUTE') ? constant('T_ATTRIBUTE') : 'T_ATTRIBUTE');
	// If this predates PHP 8.1, T_ENUM is unavailable. The replacement value is arbitrary - it just has to be different from other values of token constants.
	define(__NAMESPACE__ . '\T_ENUM', defined('T_ENUM') ? constant('T_ENUM') : 'T_ENUM');
	define(__NAMESPACE__ . '\T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG', defined('T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG') ? constant('T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG') : 'T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG');
	define(__NAMESPACE__ . '\T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG', defined('T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG') ? constant('T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG') : 'T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG');
	define(__NAMESPACE__ . '\T_READONLY', defined('T_READONLY') ? constant('T_READONLY') : 'T_READONLY');

	/**
	* Tokenizes content using PHP's built-in `token_get_all`, and converts to "lightweight" Token representation.
	*
	* Initially we tried hand-spinning the lexer (see `experiments/Lexer.php`), but we had difficulties optimizing
	* performance (especially when working with Unicode characters.)
	*
	* Class PhpTokenizer
	* @package Microsoft\PhpParser
	*/
	class PhpTokenizer implements TokenStreamProviderInterface {
	public $pos;
	public $endOfFilePos;

	private $tokensArray;

	public function __construct($content) {
	$this->tokensArray = $this->getTokensArrayFromContent($content);
	$this->endOfFilePos = \count($this->tokensArray) - 1;
	$this->pos = 0;
	}

	public function scanNextToken() : Token {
	return $this->pos >= $this->endOfFilePos
	? $this->tokensArray[$this->endOfFilePos]
	: $this->tokensArray[$this->pos++];
	}

	public function getCurrentPosition() : int {
	return $this->pos;
	}

	public function setCurrentPosition(int $pos) {
	$this->pos = $pos;
	}

	public function getEndOfFilePosition() : int {
	return $this->endOfFilePos;
	}

	public function getTokensArray() : array {
	return $this->tokensArray;
	}

	/**
	* Return an array of Token object instances created from $content.
	*
	* This method is optimized heavily - this processes every single token being created.
	*
	* @param string $content the raw php code
	* @param ?int $parseContext can be SourceElements when extracting doc comments
	* @param int $initialPos
	* @param bool $treatCommentsAsTrivia
	* @return Token[]
	*/
	public static function getTokensArrayFromContent(
	$content, $parseContext = null, $initialPos = 0, $treatCommentsAsTrivia = true
	) : array {
	if ($parseContext !== null) {
	// If needed, add a prefix so that token_get_all will tokenize the remaining $contents
	$prefix = self::PARSE_CONTEXT_TO_PREFIX[$parseContext];
	$content = $prefix . $content;
	}

	$tokens = static::tokenGetAll($content, $parseContext);

	$arr = [];
	$fullStart = $start = $pos = $initialPos;
	if ($parseContext !== null) {
	// If needed, skip over the prefix we added for token_get_all and remove those tokens.
	// This was moved out of the below main loop as an optimization.
	// (the common case of parsing an entire file uses a null parseContext)
	foreach ($tokens as $i => $token) {
	unset($tokens[$i]);
	if (\is_array($token)) {
	$pos += \strlen($token[1]);
	} else {
	$pos += \strlen($token);
	}
	if (\strlen($prefix) < $pos) {
	$fullStart = $start = $pos = $initialPos;
	break;
	}
	}
	}

	// Convert tokens from token_get_all to Token instances,
	// skipping whitespace and (usually, when parseContext is null) comments.
	foreach ($tokens as $token) {
	if (\is_array($token)) {
	$tokenKind = $token[0];
	$strlen = \strlen($token[1]);
	} else {
	$pos += \strlen($token);
	$newTokenKind = self::TOKEN_MAP[$token] ?? TokenKind::Unknown;
	$arr[] = new Token($newTokenKind, $fullStart, $start, $pos - $fullStart);
	$start = $fullStart = $pos;
	continue;
	}

	$pos += $strlen;

	// Optimization note: In PHP < 7.2, the switch statement would check case by case,
	// so putting the most common cases first is slightly faster
	switch ($tokenKind) {
	case \T_WHITESPACE:
	$start += $strlen;
	break;
	case \T_STRING:
	$name = \strtolower($token[1]);
	$newTokenKind = TokenStringMaps::RESERVED_WORDS[$name] ?? TokenKind::Name;
	$arr[] = new Token($newTokenKind, $fullStart, $start, $pos - $fullStart);
	$start = $fullStart = $pos;
	break;
	case \T_OPEN_TAG:
	$arr[] = new Token(TokenKind::ScriptSectionStartTag, $fullStart, $start, $pos-$fullStart);
	$start = $fullStart = $pos;
	break;
	case \PHP_VERSION_ID >= 80000 ? \T_NAME_QUALIFIED : -1000:
	case \PHP_VERSION_ID >= 80000 ? \T_NAME_FULLY_QUALIFIED : -1001:
	// NOTE: This switch is called on every token of every file being parsed, so this traded performance for readability.
	//
	// PHP's Opcache is able to optimize switches that are exclusively known longs,
	// but not switches that mix strings and longs or have unknown longs.
	// Longs are only known if they're declared within the same class or an internal constant (tokenizer).
	//
	// For some reason, the SWITCH_LONG opcode was not generated when the expression was part of a class constant.
	// (seen with php -d opcache.opt_debug_level=0x20000)
	//
	// Use negative values because that's not expected to overlap with token kinds that token_get_all() will return.
	//
	// T_NAME_* was added in php 8.0 to forbid whitespace between parts of names.
	// Here, emulate the tokenization of php 7 by splitting it up into 1 or more tokens.
	foreach (\explode('\\', $token[1]) as $i => $name) {
	if ($i) {
	$arr[] = new Token(TokenKind::BackslashToken, $fullStart, $start, 1 + $start - $fullStart);
	$start++;
	$fullStart = $start;
	}
	if ($name === '') {
	continue;
	}
	// TODO: TokenStringMaps::RESERVED_WORDS[$name] ?? TokenKind::Name for compatibility?
	$len = \strlen($name);
	$arr[] = new Token(TokenKind::Name, $fullStart, $start, $len + $start - $fullStart);
	$start += $len;
	$fullStart = $start;
	}
	break;
	case \PHP_VERSION_ID >= 80000 ? \T_NAME_RELATIVE : -1002:
	// This is a namespace-relative name: namespace\...
	foreach (\explode('\\', $token[1]) as $i => $name) {
	$len = \strlen($name);
	if (!$i) {
	$arr[] = new Token(TokenKind::NamespaceKeyword, $fullStart, $start, $len + $start - $fullStart);
	$start += $len;
	$fullStart = $start;
	continue;
	}
	$arr[] = new Token(TokenKind::BackslashToken, $fullStart, $start, 1);
	$start++;

	// TODO: TokenStringMaps::RESERVED_WORDS[$name] ?? TokenKind::Name for compatibility?
	$arr[] = new Token(TokenKind::Name, $start, $start, $len);

	$start += $len;
	$fullStart = $start;
	}
	break;
	case \T_COMMENT:
	case \T_DOC_COMMENT:
	if ($treatCommentsAsTrivia) {
	$start += $strlen;
	break;
	}
	// fall through
	default:
	$newTokenKind = self::TOKEN_MAP[$tokenKind] ?? TokenKind::Unknown;
	$arr[] = new Token($newTokenKind, $fullStart, $start, $pos - $fullStart);
	$start = $fullStart = $pos;
	break;
	}
	}

	$arr[] = new Token(TokenKind::EndOfFileToken, $fullStart, $start, $pos - $fullStart);
	return $arr;
	}

	/**
	* @param string $content the raw php code
	* @param ?int $parseContext can be SourceElements when extracting doc comments.
	* Having this available may be useful for subclasses to decide whether or not to post-process results, cache results, etc.
	* @return array[]\|string[] an array of tokens. When concatenated, these tokens must equal $content.
	*
	* This exists so that it can be overridden in subclasses, e.g. to cache the result of tokenizing entire files.
	* Applications using tolerant-php-parser may often end up needing to use the token stream for other reasons that are hard to do in the resulting AST,
	* such as iterating over T_COMMENTS, checking for inline html,
	* looking up all tokens (including skipped tokens) on a given line, etc.
	*/
	protected static function tokenGetAll(string $content, $parseContext): array
	{
	return @\token_get_all($content);
	}

	const TOKEN_MAP = [
	T_CLASS_C => TokenKind::Name,
	T_DIR => TokenKind::Name,
	T_FILE => TokenKind::Name,
	T_FUNC_C => TokenKind::Name,
	T_METHOD_C => TokenKind::Name,
	T_NS_C => TokenKind::Name,
	T_TRAIT_C => TokenKind::Name,
	T_LINE => TokenKind::Name,

	T_STRING => TokenKind::Name,
	T_VARIABLE => TokenKind::VariableName,

	T_ABSTRACT => TokenKind::AbstractKeyword,
	T_LOGICAL_AND => TokenKind::AndKeyword,
	T_ARRAY => TokenKind::ArrayKeyword,
	T_AS => TokenKind::AsKeyword,
	T_BREAK => TokenKind::BreakKeyword,
	T_CALLABLE => TokenKind::CallableKeyword,
	T_CASE => TokenKind::CaseKeyword,
	T_CATCH => TokenKind::CatchKeyword,
	T_CLASS => TokenKind::ClassKeyword,
	T_CLONE => TokenKind::CloneKeyword,
	T_CONST => TokenKind::ConstKeyword,
	T_CONTINUE => TokenKind::ContinueKeyword,
	T_DECLARE => TokenKind::DeclareKeyword,
	T_DEFAULT => TokenKind::DefaultKeyword,
	T_DO => TokenKind::DoKeyword,
	T_ECHO => TokenKind::EchoKeyword,
	T_ELSE => TokenKind::ElseKeyword,
	T_ELSEIF => TokenKind::ElseIfKeyword,
	T_EMPTY => TokenKind::EmptyKeyword,
	T_ENDDECLARE => TokenKind::EndDeclareKeyword,
	T_ENDFOR => TokenKind::EndForKeyword,
	T_ENDFOREACH => TokenKind::EndForEachKeyword,
	T_ENDIF => TokenKind::EndIfKeyword,
	T_ENDSWITCH => TokenKind::EndSwitchKeyword,
	T_ENDWHILE => TokenKind::EndWhileKeyword,
	T_ENUM => TokenKind::EnumKeyword,
	T_EVAL => TokenKind::EvalKeyword,
	T_EXIT => TokenKind::ExitKeyword,
	T_EXTENDS => TokenKind::ExtendsKeyword,
	T_FINAL => TokenKind::FinalKeyword,
	T_FINALLY => TokenKind::FinallyKeyword,
	T_FOR => TokenKind::ForKeyword,
	T_FOREACH => TokenKind::ForeachKeyword,
	T_FN => TokenKind::FnKeyword,
	T_FUNCTION => TokenKind::FunctionKeyword,
	T_GLOBAL => TokenKind::GlobalKeyword,
	T_GOTO => TokenKind::GotoKeyword,
	T_HALT_COMPILER => TokenKind::HaltCompilerKeyword,
	T_IF => TokenKind::IfKeyword,
	T_IMPLEMENTS => TokenKind::ImplementsKeyword,
	T_INCLUDE => TokenKind::IncludeKeyword,
	T_INCLUDE_ONCE => TokenKind::IncludeOnceKeyword,
	T_INSTANCEOF => TokenKind::InstanceOfKeyword,
	T_INSTEADOF => TokenKind::InsteadOfKeyword,
	T_INTERFACE => TokenKind::InterfaceKeyword,
	T_ISSET => TokenKind::IsSetKeyword,
	T_LIST => TokenKind::ListKeyword,
	T_MATCH => TokenKind::MatchKeyword,
	T_NAMESPACE => TokenKind::NamespaceKeyword,
	T_NEW => TokenKind::NewKeyword,
	T_LOGICAL_OR => TokenKind::OrKeyword,
	T_PRINT => TokenKind::PrintKeyword,
	T_PRIVATE => TokenKind::PrivateKeyword,
	T_PROTECTED => TokenKind::ProtectedKeyword,
	T_PUBLIC => TokenKind::PublicKeyword,
	T_READONLY => TokenKind::ReadonlyKeyword,
	T_REQUIRE => TokenKind::RequireKeyword,
	T_REQUIRE_ONCE => TokenKind::RequireOnceKeyword,
	T_RETURN => TokenKind::ReturnKeyword,
	T_STATIC => TokenKind::StaticKeyword,
	T_SWITCH => TokenKind::SwitchKeyword,
	T_THROW => TokenKind::ThrowKeyword,
	T_TRAIT => TokenKind::TraitKeyword,
	T_TRY => TokenKind::TryKeyword,
	T_UNSET => TokenKind::UnsetKeyword,
	T_USE => TokenKind::UseKeyword,
	T_VAR => TokenKind::VarKeyword,
	T_WHILE => TokenKind::WhileKeyword,
	T_LOGICAL_XOR => TokenKind::XorKeyword,
	T_YIELD => TokenKind::YieldKeyword,
	T_YIELD_FROM => TokenKind::YieldFromKeyword,

	"[" => TokenKind::OpenBracketToken,
	"]" => TokenKind::CloseBracketToken,
	"(" => TokenKind::OpenParenToken,
	")" => TokenKind::CloseParenToken,
	"{" => TokenKind::OpenBraceToken,
	"}" => TokenKind::CloseBraceToken,
	"." => TokenKind::DotToken,
	T_OBJECT_OPERATOR => TokenKind::ArrowToken,
	T_NULLSAFE_OBJECT_OPERATOR => TokenKind::QuestionArrowToken,
	T_ATTRIBUTE => TokenKind::AttributeToken,
	T_INC => TokenKind::PlusPlusToken,
	T_DEC => TokenKind::MinusMinusToken,
	T_POW => TokenKind::AsteriskAsteriskToken,
	"*" => TokenKind::AsteriskToken,
	"+" => TokenKind::PlusToken,
	"-" => TokenKind::MinusToken,
	"~" => TokenKind::TildeToken,
	"!" => TokenKind::ExclamationToken,
	"$" => TokenKind::DollarToken,
	"/" => TokenKind::SlashToken,
	"%" => TokenKind::PercentToken,
	T_SL => TokenKind::LessThanLessThanToken,
	T_SR => TokenKind::GreaterThanGreaterThanToken,
	"<" => TokenKind::LessThanToken,
	">" => TokenKind::GreaterThanToken,
	T_IS_SMALLER_OR_EQUAL => TokenKind::LessThanEqualsToken,
	T_IS_GREATER_OR_EQUAL => TokenKind::GreaterThanEqualsToken,
	T_IS_EQUAL => TokenKind::EqualsEqualsToken,
	T_IS_IDENTICAL => TokenKind::EqualsEqualsEqualsToken,
	T_IS_NOT_EQUAL => TokenKind::ExclamationEqualsToken,
	T_IS_NOT_IDENTICAL => TokenKind::ExclamationEqualsEqualsToken,
	"^" => TokenKind::CaretToken,
	"\|" => TokenKind::BarToken,
	"&" => TokenKind::AmpersandToken,
	T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG => TokenKind::AmpersandToken,
	T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG => TokenKind::AmpersandToken,
	T_BOOLEAN_AND => TokenKind::AmpersandAmpersandToken,
	T_BOOLEAN_OR => TokenKind::BarBarToken,
	":" => TokenKind::ColonToken,
	";" => TokenKind::SemicolonToken,
	"=" => TokenKind::EqualsToken,
	T_POW_EQUAL => TokenKind::AsteriskAsteriskEqualsToken,
	T_MUL_EQUAL => TokenKind::AsteriskEqualsToken,
	T_DIV_EQUAL => TokenKind::SlashEqualsToken,
	T_MOD_EQUAL => TokenKind::PercentEqualsToken,
	T_PLUS_EQUAL => TokenKind::PlusEqualsToken,
	T_MINUS_EQUAL => TokenKind::MinusEqualsToken,
	T_CONCAT_EQUAL => TokenKind::DotEqualsToken,
	T_SL_EQUAL => TokenKind::LessThanLessThanEqualsToken,
	T_SR_EQUAL => TokenKind::GreaterThanGreaterThanEqualsToken,
	T_AND_EQUAL => TokenKind::AmpersandEqualsToken,
	T_XOR_EQUAL => TokenKind::CaretEqualsToken,
	T_OR_EQUAL => TokenKind::BarEqualsToken,
	"," => TokenKind::CommaToken,
	namespace\T_COALESCE_EQUAL => TokenKind::QuestionQuestionEqualsToken,
	T_COALESCE => TokenKind::QuestionQuestionToken,
	T_SPACESHIP => TokenKind::LessThanEqualsGreaterThanToken,
	T_ELLIPSIS => TokenKind::DotDotDotToken,
	T_NS_SEPARATOR => TokenKind::BackslashToken,
	T_PAAMAYIM_NEKUDOTAYIM => TokenKind::ColonColonToken,
	T_DOUBLE_ARROW => TokenKind::DoubleArrowToken, // TODO missing from spec

	"@" => TokenKind::AtSymbolToken,
	"`" => TokenKind::BacktickToken,
	"?" => TokenKind::QuestionToken,

	T_LNUMBER => TokenKind::IntegerLiteralToken,

	T_DNUMBER => TokenKind::FloatingLiteralToken,

	T_OPEN_TAG => TokenKind::ScriptSectionStartTag,
	T_OPEN_TAG_WITH_ECHO => TokenKind::ScriptSectionStartWithEchoTag,
	T_CLOSE_TAG => TokenKind::ScriptSectionEndTag,

	T_INLINE_HTML => TokenKind::InlineHtml,

	"\"" => TokenKind::DoubleQuoteToken,
	"'" => TokenKind::SingleQuoteToken,
	T_ENCAPSED_AND_WHITESPACE => TokenKind::EncapsedAndWhitespace,
	T_DOLLAR_OPEN_CURLY_BRACES => TokenKind::DollarOpenBraceToken,
	T_CURLY_OPEN => TokenKind::OpenBraceDollarToken,
	T_CONSTANT_ENCAPSED_STRING => TokenKind::StringLiteralToken,

	T_ARRAY_CAST => TokenKind::ArrayCastToken,
	T_BOOL_CAST => TokenKind::BoolCastToken,
	T_DOUBLE_CAST => TokenKind::DoubleCastToken,
	T_INT_CAST => TokenKind::IntCastToken,
	T_OBJECT_CAST => TokenKind::ObjectCastToken,
	T_STRING_CAST => TokenKind::StringCastToken,
	T_UNSET_CAST => TokenKind::UnsetCastToken,

	T_START_HEREDOC => TokenKind::HeredocStart,
	T_END_HEREDOC => TokenKind::HeredocEnd,
	T_STRING_VARNAME => TokenKind::StringVarname,
	T_COMMENT => TokenKind::CommentToken,
	T_DOC_COMMENT => TokenKind::DocCommentToken,
	T_NUM_STRING => TokenKind::IntegerLiteralToken
	];

	const PARSE_CONTEXT_TO_PREFIX = [
	ParseContext::SourceElements => "<?php "
	];
	}

File Metadata

Mime Type: text/x-php
Expires: Sat, Oct 3, 20:27 (4 d, 2 h ago)
Storage Engine: local-disk
Storage Format: Raw Data
Storage Handle: f4/e9/65d0fdba2e000daf7e52585faf62
Default Alt Text: PhpTokenizer.php (19 KB)

PhpTokenizer.phpNo OneTemporaryActions

PhpTokenizer.phpView Options

File Metadata

Event Timeline

PhpTokenizer.php
No OneTemporary
Actions

PhpTokenizer.php
View Options