From 51644a6c584eb9833af71c40198fdc5d8a99c904 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Wed, 25 Apr 2018 19:06:33 +0300 Subject: [PATCH 1/9] Added examples/lark.g - Reference implementation of the Lark grammar (inspired by issue #116) --- examples/README.md | 1 + examples/lark.g | 49 ++++++++++++++++++++++++++++++++++++++++ examples/lark_grammar.py | 18 +++++++++++++++ lark/grammars/common.g | 1 + lark/load_grammar.py | 8 ++++++- 5 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 examples/lark.g create mode 100644 examples/lark_grammar.py diff --git a/examples/README.md b/examples/README.md index 3fbe3ea..25bf504 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,6 +7,7 @@ - [indented\_tree.py](indented\_tree.py) - A demonstration of parsing indentation ("whitespace significant" language) - [fruitflies.py](fruitflies.py) - A demonstration of ambiguity - [turtle\_dsl.py](turtle_dsl.py) - Implements a LOGO-like toy language for Python's turtle, with interpreter. +- [lark\_grammar.py](lark_grammar.py) - A reference implementation of the Lark grammar (using LALR(1) + standard lexer) ### Advanced diff --git a/examples/lark.g b/examples/lark.g new file mode 100644 index 0000000..1fbf592 --- /dev/null +++ b/examples/lark.g @@ -0,0 +1,49 @@ +start: (_item | _NL)* + +_item: rule + | token + | statement + +rule: RULE priority? ":" expansions _NL +token: TOKEN priority? ":" expansions _NL + +priority: "." NUMBER + +statement: "%ignore" expansions _NL -> ignore + | "%import" import_args ["->" TOKEN] _NL -> import + +import_args: name ("." name)* + +?expansions: alias (_VBAR alias)* + +?alias: expansion ["->" RULE] + +?expansion: expr* + +?expr: atom [OP | "~" NUMBER [".." NUMBER]] + +?atom: "(" expansions ")" + | "[" expansions "]" -> maybe + | STRING ".." STRING -> literal_range + | name + | (REGEXP | STRING) -> literal + +name: RULE + | TOKEN + +_VBAR: _NL? "|" +OP: /[+*][?]?|[?](?![a-z])/ +RULE: /!?[_?]?[a-z][_a-z0-9]*/ +TOKEN: /_?[A-Z][_A-Z0-9]*/ +STRING: _STRING "i"? +REGEXP: /\/(?!\/)(\\\/|\\\\|[^\/\n])*?\/[imslux]*/ +_NL: /(\r?\n)+\s*/ + +%import common.ESCAPED_STRING -> _STRING +%import common.INT -> NUMBER +%import common.WS_INLINE + +COMMENT: "//" /[^\n]/* + +%ignore WS_INLINE +%ignore COMMENT diff --git a/examples/lark_grammar.py b/examples/lark_grammar.py new file mode 100644 index 0000000..88fc4cf --- /dev/null +++ b/examples/lark_grammar.py @@ -0,0 +1,18 @@ +from lark import Lark + +parser = Lark(open('examples/lark.g'), parser="lalr") + +grammar_files = [ + 'examples/python2.g', + 'examples/python3.g', + 'examples/lark.g', + 'lark/grammars/common.g', +] + +def test(): + for grammar_file in grammar_files: + tree = parser.parse(open(grammar_file).read()) + print("All grammars parsed successfully") + +if __name__ == '__main__': + test() diff --git a/lark/grammars/common.g b/lark/grammars/common.g index 2bd02d0..8bc8079 100644 --- a/lark/grammars/common.g +++ b/lark/grammars/common.g @@ -20,6 +20,7 @@ SIGNED_NUMBER: ["+"|"-"] NUMBER // // Strings // +//STRING: /"(\\\"|\\\\|[^"\n])*?"i?/ STRING_INNER: ("\\\""|/[^"]/) ESCAPED_STRING: "\"" STRING_INNER* "\"" diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 43d1bf5..13aeff0 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -122,7 +122,7 @@ RULES = { 'statement': ['ignore', 'import'], 'ignore': ['_IGNORE expansions _NL'], 'import': ['_IMPORT import_args _NL', - '_IMPORT import_args _TO TOKEN'], + '_IMPORT import_args _TO TOKEN _NL'], 'import_args': ['_import_args'], '_import_args': ['name', '_import_args _DOT name'], @@ -375,6 +375,7 @@ class TokenTreeToPattern(Transformer): return p def expansion(self, items): + assert items if len(items) == 1: return items[0] if len({i.flags for i in items}) > 1: @@ -486,6 +487,11 @@ class Grammar: # Convert token-trees to strings/regexps transformer = PrepareLiterals() * TokenTreeToPattern() + for name, (token_tree, priority) in token_defs: + for t in token_tree.find_data('expansion'): + if not t.children: + raise GrammarError("Tokens cannot be empty (%s)" % name) + tokens = [TokenDef(name, transformer.transform(token_tree), priority) for name, (token_tree, priority) in token_defs] From 0a40137ac79d9bfbc477e2d4c443b8503d3e28da Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Wed, 25 Apr 2018 19:09:50 +0300 Subject: [PATCH 2/9] Update README.md --- examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index 25bf504..0951c86 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,7 +7,7 @@ - [indented\_tree.py](indented\_tree.py) - A demonstration of parsing indentation ("whitespace significant" language) - [fruitflies.py](fruitflies.py) - A demonstration of ambiguity - [turtle\_dsl.py](turtle_dsl.py) - Implements a LOGO-like toy language for Python's turtle, with interpreter. -- [lark\_grammar.py](lark_grammar.py) - A reference implementation of the Lark grammar (using LALR(1) + standard lexer) +- [lark\_grammar.py](lark_grammar.py) + [lark.g](lark.g) - A reference implementation of the Lark grammar (using LALR(1) + standard lexer) ### Advanced From 53b01b7a62cf8e47b665690e4a9a694ee289be3b Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Thu, 17 May 2018 12:57:04 +0300 Subject: [PATCH 3/9] Updated README - Build now refering to lark-parser --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 619be58..293ca1f 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Most importantly, Lark will save you time and prevent you from getting parsing h Lark has no dependencies. -[![Build Status](https://travis-ci.org/erezsh/lark.svg?branch=master)](https://travis-ci.org/erezsh/lark) +[![Build Status](https://travis-ci.org/lark-parser/lark.svg?branch=master)](https://travis-ci.org/lark-parser/lark) ### Hello World From ab2ef6d86ebf594702174eb43983f6fb34d28a35 Mon Sep 17 00:00:00 2001 From: AlexV Date: Thu, 17 May 2018 14:48:29 +0200 Subject: [PATCH 4/9] toxing for easy multiversion tests --- .travis.yml | 4 ++-- tox.ini | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 tox.ini diff --git a/.travis.yml b/.travis.yml index c49062a..6f323b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,6 @@ python: - "3.6" - "pypy" # PyPy2 5.8.0 - "pypy3" # Pypy3 5.8.0-beta0 +install: pip install tox-travis script: - - pip install -r nearley-requirements.txt - - python -m tests + - tox diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..d5820b9 --- /dev/null +++ b/tox.ini @@ -0,0 +1,30 @@ +[tox] +envlist = py27, py34, py35, py36, pypy, pypy3 +skip_missing_interpreters=true + +[travis] +2.7 = py27 +3.4 = py34 +3.5 = py35 +3.6 = py36 +pypy = pypy +pypy3 = pypy3 + +# Note : We can depend on travis matrix if needed +;[travis:env] +;DJANGO = +; 1.7: django17 +; 1.8: django18, docs + +[testenv] +whitelist_externals = git +deps = + -rnearley-requirements.txt + +# to always force recreation and avoid unexpected side effects +recreate=True + +commands= + git submodule sync -q + git submodule update --init + python -m tests \ No newline at end of file From 2dcccb016048d3452395e59014a4a80341673e85 Mon Sep 17 00:00:00 2001 From: AlexV Date: Thu, 17 May 2018 16:26:09 +0200 Subject: [PATCH 5/9] cleaning up tox.ini --- tox.ini | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tox.ini b/tox.ini index d5820b9..2c92629 100644 --- a/tox.ini +++ b/tox.ini @@ -10,12 +10,6 @@ skip_missing_interpreters=true pypy = pypy pypy3 = pypy3 -# Note : We can depend on travis matrix if needed -;[travis:env] -;DJANGO = -; 1.7: django17 -; 1.8: django18, docs - [testenv] whitelist_externals = git deps = From 47ff0e99fc6a84496c1ab616cfb84cc9944646b0 Mon Sep 17 00:00:00 2001 From: Bruno Pedrosa Date: Thu, 24 May 2018 12:26:49 +0200 Subject: [PATCH 6/9] New example showing how to use lark+qscintilla (Issue #150) --- examples/examples-requirements.txt | 2 + examples/qscintilla_json.py | 218 +++++++++++++++++++++++++++++ 2 files changed, 220 insertions(+) create mode 100644 examples/examples-requirements.txt create mode 100644 examples/qscintilla_json.py diff --git a/examples/examples-requirements.txt b/examples/examples-requirements.txt new file mode 100644 index 0000000..8f93a87 --- /dev/null +++ b/examples/examples-requirements.txt @@ -0,0 +1,2 @@ +PyQt5==5.10.1 +QScintilla==2.10.4 \ No newline at end of file diff --git a/examples/qscintilla_json.py b/examples/qscintilla_json.py new file mode 100644 index 0000000..01ebf8b --- /dev/null +++ b/examples/qscintilla_json.py @@ -0,0 +1,218 @@ +import sys +import textwrap +import re + +from PyQt5.Qt import * # noqa + +from PyQt5.Qsci import QsciScintilla +from PyQt5.Qsci import QsciLexerCustom + +from editor import Editor + +from lark import Lark, inline_args, Transformer + + +class LexerJson(QsciLexerCustom): + + def __init__(self, parent=None): + super().__init__(parent) + self.create_grammar() + self.create_styles() + + def create_styles(self): + deeppink = QColor(249, 38, 114) + khaki = QColor(230, 219, 116) + mediumpurple = QColor(174, 129, 255) + mediumturquoise = QColor(81, 217, 205) + yellowgreen = QColor(166, 226, 46) + lightcyan = QColor(213, 248, 232) + darkslategrey = QColor(39, 40, 34) + + styles = { + 0: mediumturquoise, + 1: mediumpurple, + 2: yellowgreen, + 3: deeppink, + 4: khaki, + 5: lightcyan + } + + for style, color in styles.items(): + self.setColor(color, style) + self.setPaper(darkslategrey, style) + self.setFont(self.parent().font(), style) + + self.token_styles = { + "__COLON": 5, + "__COMMA": 5, + "__FALSE1": 0, + "__LBRACE": 5, + "__LSQB": 5, + "__NULL2": 0, + "__RBRACE": 5, + "__RSQB": 5, + "__TRUE0": 0, + "ESCAPED_STRING": 4, + "SIGNED_NUMBER": 1, + } + + def create_grammar(self): + grammar = ''' + ?start: value + ?value: object + | array + | string + | SIGNED_NUMBER -> number + | "true" -> true + | "false" -> false + | "null" -> null + array : "[" [value ("," value)*] "]" + object : "{" [pair ("," pair)*] "}" + pair : string ":" value + string : ESCAPED_STRING + %import common.ESCAPED_STRING + %import common.SIGNED_NUMBER + %import common.WS + %ignore WS + ''' + + class TreeToJson(Transformer): + @inline_args + def string(self, s): + return s[1:-1].replace('\\"', '"') + + array = list + pair = tuple + object = dict + number = inline_args(float) + + def null(self, _): return None + + def true(self, _): return True + + def false(self, _): return False + + self.lark = Lark(grammar, parser='lalr', transformer=TreeToJson()) + # All tokens: print([t.name for t in self.lark.parser.lexer.tokens]) + + def defaultPaper(self, style): + return QColor(39, 40, 34) + + def language(self): + return "Json" + + def description(self, style): + return {v: k for k, v in self.token_styles.items()}.get(style, "") + + def styleText(self, start, end): + self.startStyling(start) + text = self.parent().text()[start:end] + last_pos = 0 + + try: + for token in self.lark.lex(text): + ws_len = token.pos_in_stream - last_pos + if ws_len: + self.setStyling(ws_len, 0) # whitespace + + token_len = len(bytearray(token, "utf-8")) + self.setStyling( + token_len, self.token_styles.get(token.type, 0)) + + last_pos = token.pos_in_stream + token_len + except Exception as e: + print(e) + + +class EditorAll(QsciScintilla): + + def __init__(self, parent=None): + super().__init__(parent) + + # Set font defaults + font = QFont() + font.setFamily('Consolas') + font.setFixedPitch(True) + font.setPointSize(8) + font.setBold(True) + self.setFont(font) + + # Set margin defaults + fontmetrics = QFontMetrics(font) + self.setMarginsFont(font) + self.setMarginWidth(0, fontmetrics.width("000") + 6) + self.setMarginLineNumbers(0, True) + self.setMarginsForegroundColor(QColor(128, 128, 128)) + self.setMarginsBackgroundColor(QColor(39, 40, 34)) + self.setMarginType(1, self.SymbolMargin) + self.setMarginWidth(1, 12) + + # Set indentation defaults + self.setIndentationsUseTabs(False) + self.setIndentationWidth(4) + self.setBackspaceUnindents(True) + self.setIndentationGuides(True) + + # self.setFolding(QsciScintilla.CircledFoldStyle) + + # Set caret defaults + self.setCaretForegroundColor(QColor(247, 247, 241)) + self.setCaretWidth(2) + + # Set selection color defaults + self.setSelectionBackgroundColor(QColor(61, 61, 52)) + self.resetSelectionForegroundColor() + + # Set multiselection defaults + self.SendScintilla(QsciScintilla.SCI_SETMULTIPLESELECTION, True) + self.SendScintilla(QsciScintilla.SCI_SETMULTIPASTE, 1) + self.SendScintilla( + QsciScintilla.SCI_SETADDITIONALSELECTIONTYPING, True) + + lexer = LexerJson(self) + self.setLexer(lexer) + + +def main(): + app = QApplication(sys.argv) + ex = EditorAll() + ex.setWindowTitle(__file__) + ex.setText(textwrap.dedent("""\ + { + "_id": "5b05ffcbcf8e597939b3f5ca", + "about": "Excepteur consequat commodo esse voluptate aute aliquip ad sint deserunt commodo eiusmod irure. Sint aliquip sit magna duis eu est culpa aliqua excepteur ut tempor nulla. Aliqua ex pariatur id labore sit. Quis sit ex aliqua veniam exercitation laboris anim adipisicing. Lorem nisi reprehenderit ullamco labore qui sit ut aliqua tempor consequat pariatur proident.", + "address": "665 Malbone Street, Thornport, Louisiana, 243", + "age": 23, + "balance": "$3,216.91", + "company": "BULLJUICE", + "email": "elisekelley@bulljuice.com", + "eyeColor": "brown", + "gender": "female", + "guid": "d3a6d865-0f64-4042-8a78-4f53de9b0707", + "index": 0, + "isActive": false, + "isActive2": true, + "latitude": -18.660714, + "longitude": -85.378048, + "name": "Elise Kelley", + "phone": "+1 (808) 543-3966", + "picture": "http://placehold.it/32x32", + "registered": "2017-09-30T03:47:40 -02:00", + "tags": [ + "et", + "nostrud", + "in", + "fugiat", + "incididunt", + "labore", + "nostrud" + ] + }\ + """)) + ex.resize(800, 600) + ex.show() + sys.exit(app.exec_()) + + +if __name__ == "__main__": + main() From 895e056e74c02413d71bafc150a2c1e8d2ced3a7 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Thu, 24 May 2018 15:20:55 +0300 Subject: [PATCH 7/9] Fixes --- examples/qscintilla_json.py | 64 ++++++++++++------------------------- 1 file changed, 20 insertions(+), 44 deletions(-) diff --git a/examples/qscintilla_json.py b/examples/qscintilla_json.py index 01ebf8b..afbd488 100644 --- a/examples/qscintilla_json.py +++ b/examples/qscintilla_json.py @@ -7,8 +7,6 @@ from PyQt5.Qt import * # noqa from PyQt5.Qsci import QsciScintilla from PyQt5.Qsci import QsciLexerCustom -from editor import Editor - from lark import Lark, inline_args, Transformer @@ -45,54 +43,30 @@ class LexerJson(QsciLexerCustom): self.token_styles = { "__COLON": 5, "__COMMA": 5, - "__FALSE1": 0, "__LBRACE": 5, "__LSQB": 5, - "__NULL2": 0, "__RBRACE": 5, "__RSQB": 5, - "__TRUE0": 0, - "ESCAPED_STRING": 4, - "SIGNED_NUMBER": 1, + "FALSE": 0, + "NULL": 0, + "TRUE": 0, + "STRING": 4, + "NUMBER": 1, } def create_grammar(self): grammar = ''' - ?start: value - ?value: object - | array - | string - | SIGNED_NUMBER -> number - | "true" -> true - | "false" -> false - | "null" -> null - array : "[" [value ("," value)*] "]" - object : "{" [pair ("," pair)*] "}" - pair : string ":" value - string : ESCAPED_STRING - %import common.ESCAPED_STRING - %import common.SIGNED_NUMBER + anons: ":" "{" "}" "," "[" "]" + TRUE: "true" + FALSE: "false" + NULL: "NULL" + %import common.ESCAPED_STRING -> STRING + %import common.SIGNED_NUMBER -> NUMBER %import common.WS %ignore WS ''' - class TreeToJson(Transformer): - @inline_args - def string(self, s): - return s[1:-1].replace('\\"', '"') - - array = list - pair = tuple - object = dict - number = inline_args(float) - - def null(self, _): return None - - def true(self, _): return True - - def false(self, _): return False - - self.lark = Lark(grammar, parser='lalr', transformer=TreeToJson()) + self.lark = Lark(grammar, parser=None, lexer='standard') # All tokens: print([t.name for t in self.lark.parser.lexer.tokens]) def defaultPaper(self, style): @@ -173,11 +147,7 @@ class EditorAll(QsciScintilla): self.setLexer(lexer) -def main(): - app = QApplication(sys.argv) - ex = EditorAll() - ex.setWindowTitle(__file__) - ex.setText(textwrap.dedent("""\ +EXAMPLE_TEXT = textwrap.dedent("""\ { "_id": "5b05ffcbcf8e597939b3f5ca", "about": "Excepteur consequat commodo esse voluptate aute aliquip ad sint deserunt commodo eiusmod irure. Sint aliquip sit magna duis eu est culpa aliqua excepteur ut tempor nulla. Aliqua ex pariatur id labore sit. Quis sit ex aliqua veniam exercitation laboris anim adipisicing. Lorem nisi reprehenderit ullamco labore qui sit ut aliqua tempor consequat pariatur proident.", @@ -208,7 +178,13 @@ def main(): "nostrud" ] }\ - """)) + """) + +def main(): + app = QApplication(sys.argv) + ex = EditorAll() + ex.setWindowTitle(__file__) + ex.setText(EXAMPLE_TEXT) ex.resize(800, 600) ex.show() sys.exit(app.exec_()) From ac219555f94d654f83c6e629131a7148b46921c8 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Thu, 24 May 2018 15:24:32 +0300 Subject: [PATCH 8/9] Improvements --- examples/qscintilla_json.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/examples/qscintilla_json.py b/examples/qscintilla_json.py index afbd488..e7c4b7f 100644 --- a/examples/qscintilla_json.py +++ b/examples/qscintilla_json.py @@ -1,20 +1,27 @@ +# +# This example shows how to write a syntax-highlighted editor with Qt and Lark +# +# Requirements: +# +# PyQt5==5.10.1 +# QScintilla==2.10.4 + import sys import textwrap -import re from PyQt5.Qt import * # noqa from PyQt5.Qsci import QsciScintilla from PyQt5.Qsci import QsciLexerCustom -from lark import Lark, inline_args, Transformer +from lark import Lark class LexerJson(QsciLexerCustom): def __init__(self, parent=None): super().__init__(parent) - self.create_grammar() + self.create_parser() self.create_styles() def create_styles(self): @@ -54,7 +61,7 @@ class LexerJson(QsciLexerCustom): "NUMBER": 1, } - def create_grammar(self): + def create_parser(self): grammar = ''' anons: ":" "{" "}" "," "[" "]" TRUE: "true" From 9b8b383510b932776b04a08de8416bb09f59849e Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Thu, 24 May 2018 15:25:10 +0300 Subject: [PATCH 9/9] A requirements file for the examples is an overkill --- examples/examples-requirements.txt | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 examples/examples-requirements.txt diff --git a/examples/examples-requirements.txt b/examples/examples-requirements.txt deleted file mode 100644 index 8f93a87..0000000 --- a/examples/examples-requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -PyQt5==5.10.1 -QScintilla==2.10.4 \ No newline at end of file