transformer overhaul draft

hcl2/api.py

@@ -3,7 +3,7 @@
 
 from lark.tree import Tree
 from hcl2.parser import parser, reconstruction_parser
-from hcl2.transformer import DictTransformer
+from hcl2.dict_transformer import DictTransformer
 from hcl2.reconstructor import HCLReconstructor, HCLReverseTransformer
 
 

hcl2/transformer.py → hcl2/dict_transformer.py

@@ -277,6 +277,10 @@ def heredoc_template_trim(self, args: List) -> str:
     def new_line_or_comment(self, args: List) -> _DiscardType:
         return Discard
 
+    # def EQ(self, args: List):
+    #     print("EQ", args)
+    #     return args
+
     def for_tuple_expr(self, args: List) -> str:
         args = self.strip_new_line_tokens(args)
         for_expr = " ".join([self.to_tf_inline(arg) for arg in args[1:-1]])

hcl2/parser.py

@@ -12,7 +12,7 @@
 def parser() -> Lark:
     """Build standard parser for transforming HCL2 text into python structures"""
     return Lark.open(
-        "hcl2.lark",
+        "rule_transformer/hcl2.lark",
         parser="lalr",
         cache=str(PARSER_FILE),  # Disable/Delete file to effect changes to the grammar
         rel_to=__file__,
@@ -29,7 +29,7 @@ def reconstruction_parser() -> Lark:
     if necessary.
     """
     return Lark.open(
-        "hcl2.lark",
+        "rule_transformer/hcl2.lark",
         parser="lalr",
         # Caching must be disabled to allow for reconstruction until lark-parser/lark#1472 is fixed:
         #

hcl2/reconstructor.py

@@ -167,12 +167,17 @@ def _should_add_space(self, rule, current_terminal, is_block_label: bool = False
         if self._is_equals_sign(current_terminal):
             return True
 
+        if is_block_label:
+            pass
+            # print(rule, self._last_rule, current_terminal, self._last_terminal)
+
         if is_block_label and isinstance(rule, Token) and rule.value == "string":
             if (
                 current_terminal == self._last_terminal == Terminal("DBLQUOTE")
                 or current_terminal == Terminal("DBLQUOTE")
-                and self._last_terminal == Terminal("NAME")
+                and self._last_terminal == Terminal("IDENTIFIER")
             ):
+                # print("true")
                 return True
 
         # if we're in a ternary or binary operator, add space around the operator

hcl2/rule_transformer/deserializer.py

@@ -0,0 +1,31 @@
+import json
+from typing import Any, TextIO, List
+
+from hcl2.rule_transformer.rules.abstract import LarkElement, LarkRule
+from hcl2.rule_transformer.utils import DeserializationOptions
+
+
+class Deserializer:
+    def __init__(self, options=DeserializationOptions()):
+        self.options = options
+
+    def load_python(self, value: Any) -> LarkElement:
+        pass
+
+    def loads(self, value: str) -> LarkElement:
+        return self.load_python(json.loads(value))
+
+    def load(self, file: TextIO) -> LarkElement:
+        return self.loads(file.read())
+
+    def _deserialize(self, value: Any) -> LarkElement:
+        pass
+
+    def _deserialize_dict(self, value: dict) -> LarkRule:
+        pass
+
+    def _deserialize_list(self, value: List) -> LarkRule:
+        pass
+
+    def _deserialize_expression(self, value: str) -> LarkRule:
+        pass

hcl2/rule_transformer/editor.py

@@ -0,0 +1,77 @@
+import dataclasses
+from copy import copy, deepcopy
+from typing import List, Optional, Set, Tuple
+
+from hcl2.rule_transformer.rules.abstract import LarkRule
+from hcl2.rule_transformer.rules.base import BlockRule, StartRule
+
+
+@dataclasses.dataclass
+class TreePathElement:
+
+    name: str
+    index: int = 0
+
+
+@dataclasses.dataclass
+class TreePath:
+
+    elements: List[TreePathElement] = dataclasses.field(default_factory=list)
+
+    @classmethod
+    def build(cls, elements: List[Tuple[str, Optional[int]] | str]):
+        results = []
+        for element in elements:
+            if isinstance(element, tuple):
+                if len(element) == 1:
+                    result = TreePathElement(element[0], 0)
+                else:
+                    result = TreePathElement(*element)
+            else:
+                result = TreePathElement(element, 0)
+
+            results.append(result)
+
+        return cls(results)
+
+    def __iter__(self):
+        return self.elements.__iter__()
+
+    def __len__(self):
+        return self.elements.__len__()
+
+
+class Editor:
+    def __init__(self, rules_tree: LarkRule):
+        self.rules_tree = rules_tree
+
+    @classmethod
+    def _find_one(cls, rules_tree: LarkRule, path_element: TreePathElement) -> LarkRule:
+        return cls._find_all(rules_tree, path_element.name)[path_element.index]
+
+    @classmethod
+    def _find_all(cls, rules_tree: LarkRule, rule_name: str) -> List[LarkRule]:
+        children = []
+        print("rule", rules_tree)
+        print("rule children", rules_tree.children)
+        for child in rules_tree.children:
+            if isinstance(child, LarkRule) and child.lark_name() == rule_name:
+                children.append(child)
+
+        return children
+
+    def find_by_path(self, path: TreePath, rule_name: str) -> List[LarkRule]:
+        path = deepcopy(path.elements)
+
+        current_rule = self.rules_tree
+        while len(path) > 0:
+            current_path, *path = path
+            print(current_path, path)
+            current_rule = self._find_one(current_rule, current_path)
+
+        return self._find_all(current_rule, rule_name)
+
+    # def visit(self, path: TreePath) -> "Editor":
+    #
+    #     while len(path) > 1:
+    #         current =

hcl2/rule_transformer/hcl2.lark

@@ -0,0 +1,163 @@
+// ============================================================================
+// Terminals
+// ============================================================================
+
+// Whitespace and Comments
+NL_OR_COMMENT: /\n[ \t]*/ | /#.*\n/ | /\/\/.*\n/ | /\/\*(.|\n)*?(\*\/)/
+
+// Keywords
+IF : "if"
+IN : "in"
+FOR : "for"
+FOR_EACH : "for_each"
+
+
+// Literals
+NAME : /[a-zA-Z_][a-zA-Z0-9_-]*/
+ESCAPED_INTERPOLATION.2: /\$\$\{[^}]*\}/
+STRING_CHARS.1: /(?:(?!\$\$\{)(?!\$\{)[^"\\]|\\.|(?:\$(?!\$?\{)))+/
+DECIMAL : "0".."9"
+NEGATIVE_DECIMAL : "-" DECIMAL
+EXP_MARK : ("e" | "E") ("+" | "-")? DECIMAL+
+INT_LITERAL: NEGATIVE_DECIMAL? DECIMAL+
+FLOAT_LITERAL: (NEGATIVE_DECIMAL? DECIMAL+ | NEGATIVE_DECIMAL+) "." DECIMAL+ (EXP_MARK)?
+               | (NEGATIVE_DECIMAL? DECIMAL+ | NEGATIVE_DECIMAL+) (EXP_MARK)
+
+// Operators
+BINARY_OP : DOUBLE_EQ | NEQ | LT | GT | LEQ | GEQ | MINUS | ASTERISK | SLASH | PERCENT | DOUBLE_AMP | DOUBLE_PIPE | PLUS
+DOUBLE_EQ : "=="
+NEQ : "!="
+LT : "<"
+GT : ">"
+LEQ : "<="
+GEQ : ">="
+MINUS : "-"
+ASTERISK : "*"
+SLASH : "/"
+PERCENT : "%"
+DOUBLE_AMP : "&&"
+DOUBLE_PIPE : "||"
+PLUS : "+"
+NOT : "!"
+QMARK : "?"
+
+// Punctuation
+LPAR : "("
+RPAR : ")"
+LBRACE : "{"
+RBRACE : "}"
+LSQB : "["
+RSQB : "]"
+COMMA : ","
+DOT : "."
+EQ : /[ \t]*=(?!=|>)/
+COLON : ":"
+DBLQUOTE : "\""
+
+// Interpolation
+INTERP_START : "${"
+
+// Splat Operators
+ATTR_SPLAT : ".*"
+FULL_SPLAT_START : "[*]"
+
+// Special Operators
+FOR_OBJECT_ARROW : "=>"
+ELLIPSIS : "..."
+COLONS: "::"
+
+// Heredocs
+HEREDOC_TEMPLATE : /<<(?P<heredoc>[a-zA-Z][a-zA-Z0-9._-]+)\n?(?:.|\n)*?\n\s*(?P=heredoc)\n/
+HEREDOC_TEMPLATE_TRIM : /<<-(?P<heredoc_trim>[a-zA-Z][a-zA-Z0-9._-]+)\n?(?:.|\n)*?\n\s*(?P=heredoc_trim)\n/
+
+// Ignore whitespace (but not newlines, as they're significant in HCL)
+%ignore /[ \t]+/
+
+// ============================================================================
+// Rules
+// ============================================================================
+
+// Top-level structure
+start : body
+
+// Body and basic constructs
+body : (new_line_or_comment? (attribute | block))* new_line_or_comment?
+attribute : identifier EQ expression
+block : identifier (identifier | string)* new_line_or_comment? LBRACE body RBRACE
+
+// Whitespace and comments
+new_line_or_comment: ( NL_OR_COMMENT )+
+
+// Basic literals and identifiers
+identifier : NAME
+keyword: IN | FOR | IF | FOR_EACH
+int_lit: INT_LITERAL
+float_lit: FLOAT_LITERAL
+string: DBLQUOTE string_part* DBLQUOTE
+string_part: STRING_CHARS
+           | ESCAPED_INTERPOLATION
+           | interpolation
+
+// Expressions
+?expression : expr_term | operation | conditional
+interpolation: INTERP_START expression RBRACE
+conditional : expression QMARK new_line_or_comment? expression new_line_or_comment? COLON new_line_or_comment? expression
+
+// Operations
+?operation : unary_op | binary_op
+!unary_op : (MINUS | NOT) expr_term
+binary_op : expression binary_term new_line_or_comment?
+binary_term : binary_operator new_line_or_comment? expression
+!binary_operator : BINARY_OP
+
+// Expression terms
+expr_term : LPAR new_line_or_comment? expression new_line_or_comment? RPAR
+            | float_lit
+            | int_lit
+            | string
+            | tuple
+            | object
+            | identifier
+            | function_call
+            | heredoc_template
+            | heredoc_template_trim
+            | index_expr_term
+            | get_attr_expr_term
+            | attr_splat_expr_term
+            | full_splat_expr_term
+            | for_tuple_expr
+            | for_object_expr
+
+// Collections
+tuple : LSQB new_line_or_comment? (expression new_line_or_comment? COMMA new_line_or_comment?)* (expression new_line_or_comment? COMMA? new_line_or_comment?)?  RSQB
+object : LBRACE new_line_or_comment? ((object_elem | (object_elem new_line_or_comment? COMMA)) new_line_or_comment?)* RBRACE
+object_elem : object_elem_key ( EQ | COLON ) expression
+object_elem_key : float_lit | int_lit | identifier | string | object_elem_key_dot_accessor | object_elem_key_expression
+object_elem_key_expression : LPAR expression RPAR
+object_elem_key_dot_accessor : identifier (DOT identifier)+
+
+// Heredocs
+heredoc_template : HEREDOC_TEMPLATE
+heredoc_template_trim : HEREDOC_TEMPLATE_TRIM
+
+// Functions
+function_call : identifier (COLONS identifier COLONS identifier)? LPAR new_line_or_comment? arguments? new_line_or_comment? RPAR
+arguments : (expression (new_line_or_comment? COMMA new_line_or_comment? expression)* (COMMA | ELLIPSIS)? new_line_or_comment?)
+
+// Indexing and attribute access
+index_expr_term : expr_term index
+get_attr_expr_term : expr_term get_attr
+attr_splat_expr_term : expr_term attr_splat
+full_splat_expr_term : expr_term full_splat
+?index : braces_index | short_index
+braces_index : LSQB new_line_or_comment? expression new_line_or_comment? RSQB
+short_index : DOT INT_LITERAL
+get_attr : DOT identifier
+attr_splat : ATTR_SPLAT (get_attr | index)*
+full_splat : FULL_SPLAT_START (get_attr | index)*
+
+// For expressions
+!for_tuple_expr : LSQB new_line_or_comment? for_intro new_line_or_comment? expression new_line_or_comment? for_cond? new_line_or_comment? RSQB
+!for_object_expr : LBRACE new_line_or_comment? for_intro new_line_or_comment? expression FOR_OBJECT_ARROW new_line_or_comment? expression new_line_or_comment? ELLIPSIS? new_line_or_comment? for_cond? new_line_or_comment? RBRACE
+!for_intro : FOR new_line_or_comment? identifier (COMMA identifier new_line_or_comment?)? new_line_or_comment? IN new_line_or_comment? expression new_line_or_comment? COLON new_line_or_comment?
+!for_cond : IF new_line_or_comment? expression

hcl2/rule_transformer/json.py

@@ -0,0 +1,12 @@
+from json import JSONEncoder
+from typing import Any
+
+from hcl2.rule_transformer.rules.abstract import LarkRule
+
+
+class LarkEncoder(JSONEncoder):
+    def default(self, obj: Any):
+        if isinstance(obj, LarkRule):
+            return obj.serialize()
+        else:
+            return super().default(obj)