Lua 55 support

README.rst

@@ -28,8 +28,20 @@ It will install the shell command 'luaparser'.
 Compatibility with Lua grammar
 ------------------------------------------------------------------------------
 
-3.2.1  =>  Lua 5.3 grammar
-3.2.1+ =>  Lua 5.4 grammar
+==================  =============================  ================================================
+Lua source version  Recommended ``luaparser``       Grammar support
+==================  =============================  ================================================
+Lua 5.1--5.3        ``luaparser>=3.2.1``            Supported
+Lua 5.4             ``luaparser>=3.3.0``            Supported
+Lua 5.5             Unreleased (after 4.2.0)        Supported
+==================  =============================  ================================================
+
+Use the newest ``luaparser`` release that satisfies your application's Python
+version requirements. Grammar support is backward compatible, so a newer
+release can parse source written for an older Lua version. ``luaparser`` does
+not enforce all compile-time semantic restrictions of a particular Lua
+version, so successful parsing is not a substitute for checking the source
+with that version's ``luac``.
 
 Options
 ------------------------------------------------------------------------------

luaparser/ast.py

@@ -72,7 +72,7 @@ def to_pretty_str(root: Node, indent=2) -> str:
 
 
 def to_lua_source(root: Node, indent=4) -> str:
-    return printers.LuaOutputVisitor(indent_size=indent).do_visit(root)
+    return printers.LuaOutputVisitor(indent_size=indent).to_source(root)
 
 
 def to_xml_str(tree):

luaparser/astnodes.py

@@ -65,7 +65,7 @@ def __eq__(self, other) -> bool:
             return _equal_dicts(
                 self.__dict__,
                 other.__dict__,
-                ["_first_token", "_last_token", "_index"],
+                ["_first_token", "_last_token", "_index", "_long_bracket_level"],
             )
         return False
 
@@ -285,9 +285,33 @@ class LocalAssign(Assign):
         values: List of values.
     """
 
-    def __init__(self, targets: List[Name], values: List[Node], **kwargs):
+    def __init__(
+            self,
+            targets: List[Name],
+            values: List[Node],
+            attribute: Optional[Attribute] = None,
+            **kwargs
+    ):
         super().__init__(targets, values, **kwargs)
         self._name: str = "LocalAssign"
+        self.attribute: Optional[Attribute] = attribute
+
+
+class GlobalAssign(Assign):
+    """Lua 5.5 global declaration statement."""
+
+    def __init__(
+            self,
+            targets: Optional[List[Name]] = None,
+            values: Optional[List[Node]] = None,
+            attribute: Optional[Attribute] = None,
+            wildcard: bool = False,
+            **kwargs
+    ):
+        super().__init__(targets or [], values or [], **kwargs)
+        self._name: str = "GlobalAssign"
+        self.attribute: Optional[Attribute] = attribute
+        self.wildcard: bool = wildcard
 
 
 class While(Statement):
@@ -541,6 +565,14 @@ def __init__(self, name: Expression, args: List[Expression], body: Block, **kwar
         self.body: Block = body
 
 
+class GlobalFunction(Function):
+    """Lua 5.5 global function declaration statement."""
+
+    def __init__(self, name: Name, args: List[Expression], body: Block, **kwargs):
+        super().__init__(name, args, body, **kwargs)
+        self._name: str = "GlobalFunction"
+
+
 class Method(Statement):
     """Define the Lua Object Oriented function statement.
 
@@ -614,10 +646,11 @@ def __init__(self, n: NumberType, **kwargs):
 
 
 class Varargs(Expression):
-    """Define the Lua Varargs expression (...)."""
+    """Define a variadic parameter, optionally named in Lua 5.5."""
 
-    def __init__(self, **kwargs):
+    def __init__(self, name: Optional[Name] = None, **kwargs):
         super(Varargs, self).__init__("Varargs", **kwargs)
+        self.name: Optional[Name] = name
 
 
 class StringDelimiter(Enum):
@@ -640,12 +673,18 @@ def __init__(
             s: bytes,
             raw: str,
             delimiter: StringDelimiter = StringDelimiter.SINGLE_QUOTE,
+            long_bracket_level: int = 0,
             **kwargs
     ):
         super(String, self).__init__("String", **kwargs)
         self.s: bytes = s
         self.raw: str = raw
         self.delimiter: StringDelimiter = delimiter
+        self._long_bracket_level: int = long_bracket_level
+
+    @property
+    def long_bracket_level(self) -> int:
+        return self._long_bracket_level
 
 
 class Field(Expression):

luaparser/builder.py

@@ -314,13 +314,40 @@ def visitStat_localfunction(self, ctx: LuaParser.Stat_localfunctionContext):
     # Visit a parse tree produced by LuaParser#stat_local.
     def visitStat_local(self, ctx: LuaParser.Stat_localContext):
         att_name_list = self.visitAttnamelist(ctx.attnamelist())
+        attribute = self.visit(ctx.attnamelist().attrib()) if ctx.attnamelist().attrib() else None
 
         if ctx.EQ():
             exp_list = self.visitExplist(ctx.explist())
         else:
             exp_list = []
 
-        return self.add_context(ctx, LocalAssign(targets=att_name_list, values=exp_list))
+        return self.add_context(ctx, LocalAssign(
+            targets=att_name_list,
+            values=exp_list,
+            attribute=attribute,
+        ))
+
+    def visitStat_global(self, ctx: LuaParser.Stat_globalContext):
+        return self.visit(ctx.globalstat())
+
+    def visitGlobalstat_function(self, ctx: LuaParser.Globalstat_functionContext):
+        func_name = self.visit(ctx.NAME(1))
+        param_list, block = self.visitFuncbody(ctx.funcbody())
+        return self.add_context(ctx, GlobalFunction(func_name, param_list, block))
+
+    def visitGlobalstat_names(self, ctx: LuaParser.Globalstat_namesContext):
+        names = self.visitAttnamelist(ctx.attnamelist())
+        values = self.visitExplist(ctx.explist()) if ctx.explist() else []
+        attribute = self.visit(ctx.attnamelist().attrib()) if ctx.attnamelist().attrib() else None
+        return self.add_context(ctx, GlobalAssign(
+            targets=names,
+            values=values,
+            attribute=attribute,
+        ))
+
+    def visitGlobalstat_wildcard(self, ctx: LuaParser.Globalstat_wildcardContext):
+        attribute = self.visit(ctx.attrib()) if ctx.attrib() else None
+        return self.add_context(ctx, GlobalAssign(attribute=attribute, wildcard=True))
 
     # Visit a parse tree produced by LuaParser#functiondef.
     def visitFunctiondef(self, ctx: LuaParser.FunctiondefContext) -> AnonymousFunction:
@@ -630,10 +657,14 @@ def visitParlist(self, ctx: LuaParser.ParlistContext) -> List[Expression]:
         else:
             name_list = []
 
-        if ctx.DDD():
-            name_list.append(Varargs())
+        if ctx.varargparam():
+            name_list.append(self.visit(ctx.varargparam()))
         return name_list
 
+    def visitVarargparam(self, ctx: LuaParser.VarargparamContext) -> Varargs:
+        name = self.visit(ctx.NAME()) if ctx.NAME() else None
+        return self.add_context(ctx, Varargs(name=name))
+
     # Visit a parse tree produced by LuaParser#tableconstructor.
     def visitTableconstructor(self, ctx: LuaParser.TableconstructorContext):
         if ctx.fieldlist():
@@ -686,15 +717,19 @@ def visitNumber(self, ctx: LuaParser.NumberContext):
         try:
             number = ast.literal_eval(number_text)
         except (ValueError, SyntaxError):
-            # exception occurs with leading zero number: 002
-            number = float(number_text)
+            if number_text.lower().startswith("0x"):
+                number = float.fromhex(number_text)
+            else:
+                # exception occurs with leading zero number: 002
+                number = float(number_text)
         return Number(
             number,
         )
 
     # Visit a parse tree produced by LuaParser#string.
     def visitString(self, ctx: LuaParser.StringContext):
         lua_str = ctx.getText()
+        long_bracket_level = 0
 
         delimiter: StringDelimiter = StringDelimiter.SINGLE_QUOTE
 
@@ -710,11 +745,15 @@ def visitString(self, ctx: LuaParser.StringContext):
         else:
             m = LUA_DOUBLE_SQUARE_RE.match(lua_str)
             if m:
+                long_bracket_level = len(m.group("eq"))
                 lua_str = m.group("body")
                 delimiter = StringDelimiter.DOUBLE_SQUARE
 
         if delimiter == StringDelimiter.DOUBLE_QUOTE or delimiter == StringDelimiter.SINGLE_QUOTE:
             unescaped_str = unescape_lua_string(lua_str)
         else:
-            unescaped_str = lua_str.encode("utf-8")
-        return String(unescaped_str, lua_str, delimiter)
+            normalized_str = re.sub(r"\r\n|\n\r|\r", "\n", lua_str)
+            if normalized_str.startswith("\n"):
+                normalized_str = normalized_str[1:]
+            unescaped_str = normalized_str.encode("utf-8")
+        return String(unescaped_str, lua_str, delimiter, long_bracket_level)

luaparser/parser/LuaLexer.g4

@@ -69,20 +69,11 @@ SQEQ     : '~=';
 
 NAME: [a-zA-Z_][a-zA-Z_0-9]*;
 
-NORMALSTRING: '"' ( EscapeSequence | ~('\\' | '"'))* '"';
+NORMALSTRING: '"' ( EscapeSequence | ~('\\' | '"' | '\r' | '\n'))* '"';
 
-CHARSTRING: '\'' ( EscapeSequence | ~('\'' | '\\'))* '\'';
+CHARSTRING: '\'' ( EscapeSequence | ~('\'' | '\\' | '\r' | '\n'))* '\'';
 
-LONGSTRING: '[' '=' '=' '=' '=' '=' '=' '=' '=' '[' .*? ']' '=' '=' '=' '=' '=' '=' '=' '=' ']'
-          | '[' '=' '=' '=' '=' '=' '=' '=' '[' .*? ']' '=' '=' '=' '=' '=' '=' '=' ']'
-          | '[' '=' '=' '=' '=' '=' '=' '[' .*? ']' '=' '=' '=' '=' '=' '=' ']'
-          | '[' '=' '=' '=' '=' '=' '[' .*? ']' '=' '=' '=' '=' '=' ']'
-          | '[' '=' '=' '=' '=' '[' .*? ']' '=' '=' '=' '=' ']'
-          | '[' '=' '=' '=' '[' .*? ']' '=' '=' '=' ']'
-          | '[' '=' '=' '[' .*? ']' '=' '=' ']'
-          | '[' '=' '[' .*? ']' '=' ']'
-          | '[' '[' .*? ']' ']'
-          ;
+LONGSTRING: '[' NESTED_STR ']';
 
 fragment NESTED_STR: '=' NESTED_STR '=' | '[' .*? ']';
 
@@ -103,19 +94,20 @@ fragment ExponentPart: [eE] [+-]? Digit+;
 fragment HexExponentPart: [pP] [+-]? Digit+;
 
 fragment EscapeSequence:
-    '\\' [abfnrtvz"'|$#\\] // World of Warcraft Lua additionally escapes |$# 
-    | '\\' '\r'? '\n'
+    '\\' [abfnrtv"'\\]
+    | '\\' 'z' [ \t\u000B\u000C\r\n]*
+    | '\\' ('\r' '\n'? | '\n' '\r'?)
     | DecimalEscape
     | HexEscape
     | UtfEscape
 ;
 
 fragment DecimalEscape:
     '\\'
-    ( Digit
-    | Digit Digit
-    | [0-1] Digit Digit
+    ( [0-1] Digit Digit
     | '2' ('5' [0-5] | [0-4] Digit)
+    | Digit Digit { self.IsDecimalEscapeTerminated() }?
+    | Digit { self.IsDecimalEscapeTerminated() }?
     )
 ;
 
@@ -155,8 +147,8 @@ LINE_COMMENT
     -> channel(2)
     ;
 
-WS: [ \t\u000C\r]+ -> channel(HIDDEN);
+WS: [ \t\u000B\u000C\r]+ -> channel(HIDDEN);
 
 NL: [\n] -> channel(1);
 
-SHEBANG: '#' { this.IsLine1Col0() }? '!'? SingleLineInputCharacter* -> channel(HIDDEN);
+SHEBANG: '#' { self.IsLine1Col0() }? '!'? SingleLineInputCharacter* -> channel(HIDDEN);