Macro imporvements

cocas/assembler/assembler.py

@@ -17,7 +17,8 @@
 def assemble_module(input_stream: antlr4.InputStream,
                     target_instructions: TargetInstructions,
                     macros_library: dict[str, dict[int, MacroDefinition]],
-                    filepath: Path) -> ObjectModule:
+                    filepath: Path,
+                    include_paths: list[Path]) -> ObjectModule:
     """
     Convert lines of an assembler file to object code
 
@@ -26,7 +27,7 @@ def assemble_module(input_stream: antlr4.InputStream,
     :param macros_library: standard macros of assembler
     :param filepath: path of the file to use in error handling
     """
-    macro_expanded_input_stream = process_macros(input_stream, macros_library, filepath)
+    macro_expanded_input_stream = process_macros(input_stream, macros_library, filepath, include_paths, False)
     r = build_ast(macro_expanded_input_stream, filepath)
     return generate_object_module(r, target_instructions)
 
@@ -52,6 +53,7 @@ def assemble_files(target: str,
                    relative_path: Optional[Path],
                    absolute_path: Optional[Path],
                    realpath: bool,
+                   include_paths: list[Path],
                    macro_libraries: list[dict[str, dict[int, MacroDefinition]]] = None
                    ) -> list[tuple[Path, ObjectModule]]:
     """
@@ -63,6 +65,7 @@ def assemble_files(target: str,
     :param relative_path: if debug paths should be relative to some path
     :param absolute_path: if relative paths should be converted to absolute
     :param realpath: if paths should be converted to canonical
+    :param include_paths: list of include search paths
     :param macro_libraries: user's .mlb files, different from standard .mlb
     :return: list of pairs [source file path, object module]
     """
@@ -88,7 +91,7 @@ def assemble_files(target: str,
         if not data.endswith('\n'):
             data += '\n'
         input_stream = antlr4.InputStream(data)
-        obj = assemble_module(input_stream, target_instructions, macros, filepath)
+        obj = assemble_module(input_stream, target_instructions, macros, filepath, include_paths)
 
         debug_info_path = get_debug_info_path(filepath, debug, relative_path, realpath)
         if debug_info_path:

cocas/assembler/macro_processor.py

@@ -1,4 +1,5 @@
 import codecs
+import antlr4
 import itertools
 import re
 from base64 import b64encode
@@ -15,7 +16,7 @@
 from .generated import MacroLexer, MacroParser, MacroVisitor
 
 
-def unique(params: list[str]):
+def unique(params: list[str], macro_stack: list[str]):
     register_available = [True] * 4
     var_params = []
     for param in params:
@@ -39,9 +40,27 @@ def unique(params: list[str]):
         defined_vars[param] = f'r{i}'
     return defined_vars
 
+def mpush(params: list[str], macro_stack: list[str]):
+    for param in params:
+        macro_stack.append(param)
+
+    return dict()
+
+def mpop(params: list[str], macro_stack: list[str]):
+    defined_vars = dict()
+    for param in params:
+        if not macro_stack:
+            raise CdmTempException('mpop: macro stack is empty')
+
+        defined_vars[param] = macro_stack.pop()
+
+    return defined_vars	
+
 
 macro_instructions = {
     'unique': unique,
+    'mpush': mpush,
+    'mpop': mpop,
 }
 
 
@@ -127,13 +146,16 @@ def sub_all(ps):
 
 # noinspection PyPep8Naming
 class ExpandMacrosVisitor(MacroVisitor):
-    def __init__(self, rewriter: Optional[TokenStreamRewriter], mlb_macros, filepath: str):
+    def __init__(self, rewriter: Optional[TokenStreamRewriter], mlb_macros, filepath: str, include_paths: list[Path], nested):
         # rewriter should be None if then will be called .visit(MlbContext)
         # rewriter should be valid if then will be called .visit(ProgramContext)
+        self.nested = nested
         self.nonce = 0
+        self.macro_stack = []
         self.macros = {name: mlb_macros[name].copy() for name in mlb_macros}
         self.rewriter = rewriter
         self.filepath = filepath
+        self.include_paths = include_paths
 
     @staticmethod
     def _generate_location_line(filepath: str, line: int, info: str = None) -> str:
@@ -151,6 +173,34 @@ def add_macro(self, macro: MacroDefinition):
                 raise CdmTempException(f"Redefinition of macro {macro.name}/{macro.arity} with different body")
         self.macros[macro.name][macro.arity] = macro
 
+    def include_file(self, include_filename: str):
+        # Firstly check in directoru where file is
+        with Path(self.filepath).parent as path:
+            include_filepath = path / include_filename
+            if include_filepath.is_file():
+                with include_filepath.open('rb') as file:
+                    data = file.read()
+                data = codecs.decode(data, 'utf8', 'strict')
+                if not data.endswith('\n'):
+                    data += '\n'
+                input_stream = antlr4.InputStream(data)
+
+                return ''.join([chr(i) for i in process_macros(input_stream, dict(), include_filepath, self.include_paths, True, self).data])
+
+        for path in self.include_paths:
+            include_filepath = path / include_filename
+            if include_filepath.is_file():
+                with include_filepath.open('rb') as file:
+                    data = file.read()
+                data = codecs.decode(data, 'utf8', 'strict')
+                if not data.endswith('\n'):
+                    data += '\n'
+                input_stream = antlr4.InputStream(data)
+                return ''.join([chr(i) for i in process_macros(input_stream, dict(), include_filepath, self.include_paths, True, self).data])
+
+        # Raise exception if there is no such file in include paths
+        raise CdmTempException(f'Include: file {include_filename} not found in any search path')
+
     # Returns a None for things as asect or empty line.
     # Returns string of macro
     def expand_macro(self, macro_name: str, macro_params: list[str]):
@@ -180,7 +230,7 @@ def expand_macro(self, macro_name: str, macro_params: list[str]):
                 # each line that does not contain another macro
                 # MUST add exactly ONE line to ret_parts
                 if instruction in macro_instructions:
-                    variables.update(macro_instructions[instruction](parameters))
+                    variables.update(macro_instructions[instruction](parameters, self.macro_stack))
                     if label != '':
                         ret_parts.append(f'{label}\n')
                     else:
@@ -219,14 +269,23 @@ def visitProgram(self, ctx: MacroParser.ProgramContext):
                     self.add_macro(self.visitMacro(child))
                 elif isinstance(child, MacroParser.LineContext):
                     label, instruction, parameters = self.visitLine(child)
-                    expanded_text = self.expand_macro(instruction, parameters)
+                    if instruction == 'include':
+                        if len(parameters) != 1:
+                            raise CdmTempException('Include: wrong amount of parameters, expected 1 parameter')
+                        expanded_text = self.include_file(parameters[0])
+                    else:
+                        expanded_text = self.expand_macro(instruction, parameters)
                     if expanded_text is not None:
                         if label != '':
                             expanded_text = f'{label}\n{expanded_text}'
 
-                        mstart = self._generate_location_line(self.filepath, child.start.line, "mstart")
-                        mstop = self._generate_location_line(self.filepath, child.stop.line + 1, "mstop")
-                        expanded_text = f'{mstart}{expanded_text}{mstop}'
+                        if self.nested:
+                            stop_mark = self._generate_location_line(self.filepath, child.stop.line + 1)
+                            expanded_text = f'{expanded_text}{stop_mark}'
+                        else:
+                            mstart = self._generate_location_line(self.filepath, child.start.line, "mstart")
+                            mstop = self._generate_location_line(self.filepath, child.stop.line + 1, "mstop")
+                            expanded_text = f'{mstart}{expanded_text}{mstop}'
                         self.rewriter.insertBeforeToken(child.start, expanded_text)
                         self.rewriter.delete(self.rewriter.DEFAULT_PROGRAM_NAME, child.start, child.stop)
             except CdmTempException as e:
@@ -327,11 +386,14 @@ def read_mlb(filepath: Path) -> dict[str, dict[int, MacroDefinition]]:
     token_stream = CommonTokenStream(lexer)
     parser = MacroParser(token_stream)
     cst = parser.mlb()
-    emv = ExpandMacrosVisitor(None, dict(), str_path)
+    emv = ExpandMacrosVisitor(None, dict(), str_path, [], False)
     return emv.visit(cst)
 
-
-def process_macros(input_stream: InputStream, library_macros, filepath: Path):
+# Added arguments:
+# include_paths - list of include paths
+# nested - if file macroprocessed for insertion into another file
+# init_ctx - ExpandMacrosVisitor - initial context which used for sharing same macros, nonce value and macro stack with some file, for insertion into which we macroprocessing this file
+def process_macros(input_stream: InputStream, library_macros, filepath: Path, include_paths: list[Path], nested, init_ctx: ExpandMacrosVisitor = None):
     str_path = filepath.absolute().as_posix()
     lexer = MacroLexer(input_stream)
     lexer.removeErrorListeners()
@@ -343,7 +405,16 @@ def process_macros(input_stream: InputStream, library_macros, filepath: Path):
     parser.addErrorListener(AntlrErrorListener(AssemblerExceptionTag.MACRO, str_path))
     cst = parser.program()
     rewriter = TokenStreamRewriter(token_stream)
-    emv = ExpandMacrosVisitor(rewriter, library_macros, str_path)
+
+    emv = ExpandMacrosVisitor(rewriter, library_macros, str_path, include_paths, nested)
+    # If init_ctx passed, take its nonce value, macro_stack and macros to emv (all except nonce by reference)
+    if init_ctx is not None:
+        emv.nonce = init_ctx.nonce
+        emv.macro_stack = init_ctx.macro_stack
+        emv.macros = init_ctx.macros
     emv.visit(cst)
     new_text = rewriter.getDefaultText()
+    # Modify init_ctx nonce
+    if init_ctx is not None:
+        init_ctx.nonce = emv.nonce
     return InputStream(new_text)

cocas/main.py

@@ -26,6 +26,7 @@ def main():
     parser.add_argument('-c', '--compile', action='store_true', help='compile into object files without linking')
     parser.add_argument('-m', '--merge', action='store_true', help='merge object files into one')
     parser.add_argument('-o', '--output', type=Path, help='specify output file name')
+    parser.add_argument('-I', metavar='', action='append', dest='include_paths', type=Path, help='specify include search path')
     debug_group = parser.add_argument_group('debug')
     debug_group.add_argument('--debug', type=Path, nargs='?', const=True, help='export debug information')
     debug_path_group = debug_group.add_mutually_exclusive_group()
@@ -36,6 +37,9 @@ def main():
     debug_group.add_argument('--realpath', action='store_true',
                              help='canonicalize paths by following symlinks and resolving . and ..')
     args = parser.parse_args()
+    if args.include_paths is None:
+        args.include_paths = []
+
     if args.list_targets:
         print('Available targets: ' + ', '.join(available_targets))
         return
@@ -90,7 +94,7 @@ def main():
     try:
         macro_libraries = [read_mlb(mlb) for mlb in mlb_files]
         objects: list[tuple[Path, ObjectModule]] = list(itertools.chain(
-            assemble_files(target, asm_files, bool(args.debug), relative_path, absolute_path, realpath,
+            assemble_files(target, asm_files, bool(args.debug), relative_path, absolute_path, realpath, args.include_paths,
                            macro_libraries=macro_libraries),
             read_object_files(target, obj_files, bool(args.debug), relative_path, absolute_path, realpath)
         ))

docs/README.md

@@ -18,6 +18,7 @@ This folder contains some useful information about various `cdm-devkit` parts.
 7. [Using CLI](./guides/7-using-cli.md)
 8. [Using external build system](./guides/8-external-build-systems.md)
 9. [Using Python environment](./guides/9-using-python-environments.md)
+10. [Macros and macro libraries](./guides/10-macros.md)
 
 ### Processors documentation