From c22d85ceb805aa2c47146bb1678bcc2c2d4c2359 Mon Sep 17 00:00:00 2001 From: varun-r-mallya Date: Wed, 15 Oct 2025 23:56:04 +0530 Subject: [PATCH] add array field generation support --- .../vmlinux_parser/dependency_handler.py | 4 + pythonbpf/vmlinux_parser/dependency_node.py | 1 + .../vmlinux_parser/ir_gen/ir_generation.py | 104 +++++++++++++++--- tests/failing_tests/xdp_pass.py | 3 +- 4 files changed, 92 insertions(+), 20 deletions(-) diff --git a/pythonbpf/vmlinux_parser/dependency_handler.py b/pythonbpf/vmlinux_parser/dependency_handler.py index b960ab3..b34d27f 100644 --- a/pythonbpf/vmlinux_parser/dependency_handler.py +++ b/pythonbpf/vmlinux_parser/dependency_handler.py @@ -167,3 +167,7 @@ class DependencyHandler: if name not in self._nodes: raise KeyError(f"No node with name '{name}' found") return self._nodes[name] + + @property + def nodes(self): + return self._nodes diff --git a/pythonbpf/vmlinux_parser/dependency_node.py b/pythonbpf/vmlinux_parser/dependency_node.py index feebec3..ddfd055 100644 --- a/pythonbpf/vmlinux_parser/dependency_node.py +++ b/pythonbpf/vmlinux_parser/dependency_node.py @@ -240,6 +240,7 @@ class DependencyNode: size_of_field = ctypes.sizeof(processing_field.type) return size_of_field elif processing_field.type.__module__ == "vmlinux": + #TODO: does not take into account offset calculation when not array but has type size if processing_field.ctype_complex_type is not None: if issubclass(processing_field.ctype_complex_type, ctypes.Array): if processing_field.containing_type.__module__ == ctypes.__name__: diff --git a/pythonbpf/vmlinux_parser/ir_gen/ir_generation.py b/pythonbpf/vmlinux_parser/ir_gen/ir_generation.py index c5fe740..6c7d3e3 100644 --- a/pythonbpf/vmlinux_parser/ir_gen/ir_generation.py +++ b/pythonbpf/vmlinux_parser/ir_gen/ir_generation.py @@ -1,8 +1,10 @@ +import ctypes import logging from ..dependency_handler import DependencyHandler from .debug_info_gen import debug_info_generation from ..dependency_node import DependencyNode import llvmlite.ir as ir +from typing import Optional logger = logging.getLogger(__name__) @@ -20,19 +22,50 @@ class IRGenerator: for struct in handler: self.struct_processor(struct) - def struct_processor(self, struct): - if struct.name not in self.generated: - print(f"IR generating for {struct.name}") + def struct_processor(self, struct, processing_stack=None): + # Initialize processing stack on first call + if processing_stack is None: + processing_stack = set() + + # If already generated, skip + if struct.name in self.generated: + return + + # Detect circular dependency + if struct.name in processing_stack: + logger.info(f"Circular dependency detected for {struct.name}, skipping recursive processing") + # For circular dependencies, we can either: + # 1. Use forward declarations (opaque pointers) + # 2. Mark as incomplete and process later + # 3. Generate a placeholder type + # Here we'll just skip and let it be processed in its own call + return + + logger.info(f"IR generating for {struct.name}") + + # Add to processing stack before processing dependencies + processing_stack.add(struct.name) + + try: + # Process all dependencies first for dependency in struct.depends_on: if dependency not in self.generated: - dep_node_from_dependency = self.handler[dependency] - self.struct_processor(dep_node_from_dependency) - self.generated.append(dependency) - # actual processor logic here after assuming all dependencies are resolved - # this part cannot yet resolve circular dependencies. Gets stuck on an infinite loop during that. + # Check if dependency exists in handler + if dependency in self.handler.nodes: + dep_node_from_dependency = self.handler[dependency] + # Pass the processing_stack down to track circular refs + self.struct_processor(dep_node_from_dependency, processing_stack) + else: + raise RuntimeError(f"Warning: Dependency {dependency} not found in handler") + + # Actual processor logic here after dependencies are resolved self.gen_ir(struct) self.generated.append(struct.name) + finally: + # Remove from processing stack after we're done + processing_stack.discard(struct.name) + def gen_ir(self, struct): # TODO: we add the btf_ama attribute by monkey patching in the end of compilation, but once llvmlite # accepts our issue, we will resort to normal accessed attribute based attribute addition @@ -41,19 +74,54 @@ class IRGenerator: field_index = 0 for field_name, field in struct.fields.items(): # does not take arrays and similar types into consideration yet. - field_co_re_name = self._struct_name_generator(struct, field, field_index) - field_index += 1 - globvar = ir.GlobalVariable( - self.llvm_module, ir.IntType(64), name=field_co_re_name - ) - globvar.linkage = "external" - globvar.set_metadata("llvm.preserve.access.index", debug_info) - print() + if field.ctype_complex_type is not None and issubclass(field.ctype_complex_type, ctypes.Array): + array_size = field.type_size + containing_type = field.containing_type + if containing_type.__module__ == ctypes.__name__: + containing_type_size = ctypes.sizeof(containing_type) + for i in range(0,array_size): + field_co_re_name = self._struct_name_generator(struct, field, field_index, True, i, containing_type_size) + globvar = ir.GlobalVariable( + self.llvm_module, ir.IntType(64), name=field_co_re_name + ) + globvar.linkage = "external" + globvar.set_metadata("llvm.preserve.access.index", debug_info) + field_index += 1 + elif field.type_size is not None: + array_size = field.type_size + containing_type = field.containing_type + if containing_type.__module__ == "vmlinux": + containing_type_size = self.handler[containing_type.__name__].current_offset + for i in range(0,array_size): + field_co_re_name = self._struct_name_generator(struct, field, field_index, True, i, containing_type_size) + globvar = ir.GlobalVariable( + self.llvm_module, ir.IntType(64), name=field_co_re_name + ) + globvar.linkage = "external" + globvar.set_metadata("llvm.preserve.access.index", debug_info) + field_index += 1 + else: + field_co_re_name = self._struct_name_generator(struct, field, field_index) + field_index += 1 + globvar = ir.GlobalVariable( + self.llvm_module, ir.IntType(64), name=field_co_re_name + ) + globvar.linkage = "external" + globvar.set_metadata("llvm.preserve.access.index", debug_info) def _struct_name_generator( - self, struct: DependencyNode, field, field_index: int + self, struct: DependencyNode, field, field_index: int, is_indexed: bool=False, index: Optional[int]=None, containing_type_size: Optional[int]=None ) -> str: - if struct.name.startswith("struct_"): + if is_indexed: + name = ( + "llvm." + + struct.name.removeprefix("struct_") + + f":0:{field.offset + index*containing_type_size}" + + "$" + + f"0:{field_index}:{index}" + ) + return name + elif struct.name.startswith("struct_"): name = ( "llvm." + struct.name.removeprefix("struct_") diff --git a/tests/failing_tests/xdp_pass.py b/tests/failing_tests/xdp_pass.py index 9900695..3354e75 100644 --- a/tests/failing_tests/xdp_pass.py +++ b/tests/failing_tests/xdp_pass.py @@ -3,8 +3,7 @@ from pythonbpf.maps import HashMap from pythonbpf.helper import XDP_PASS from vmlinux import TASK_COMM_LEN # noqa: F401 from vmlinux import struct_trace_event_raw_sys_enter # noqa: F401 - -# from vmlinux import struct_request +from vmlinux import struct_posix_cputimers from vmlinux import struct_xdp_md # from vmlinux import struct_trace_event_raw_sys_enter # noqa: F401 # from vmlinux import struct_ring_buffer_per_cpu # noqa: F401