Merge branch 'master' into fix-maps

This commit is contained in:
Pragyansh Chaturvedi
2025-10-02 22:11:38 +05:30
committed by GitHub
10 changed files with 146 additions and 107 deletions

View File

@ -1,5 +1,9 @@
import ast
from llvmlite import ir
from logging import Logger
import logging
logger: Logger = logging.getLogger(__name__)
def recursive_dereferencer(var, builder):
@ -17,7 +21,7 @@ def recursive_dereferencer(var, builder):
def handle_binary_op(rval, module, builder, var_name, local_sym_tab, map_sym_tab, func):
print(module)
logger.info(f"module {module}")
left = rval.left
right = rval.right
op = rval.op
@ -43,7 +47,7 @@ def handle_binary_op(rval, module, builder, var_name, local_sym_tab, map_sym_tab
else:
raise SyntaxError("Unsupported right operand type")
print(f"left is {left}, right is {right}, op is {op}")
logger.info(f"left is {left}, right is {right}, op is {op}")
if isinstance(op, ast.Add):
builder.store(builder.add(left, right), local_sym_tab[var_name].var)

View File

@ -5,15 +5,19 @@ from .functions_pass import func_proc
from .maps import maps_proc
from .structs import structs_proc
from .globals_pass import globals_processing
from .debuginfo import DW_LANG_C11, DwarfBehaviorEnum
from .debuginfo import DW_LANG_C11, DwarfBehaviorEnum, DebugInfoGenerator
import os
import subprocess
import inspect
from pathlib import Path
from pylibbpf import BpfProgram
import tempfile
from logging import Logger
import logging
VERSION = "v0.1.3"
logger: Logger = logging.getLogger(__name__)
VERSION = "v0.1.4"
def find_bpf_chunks(tree):
@ -30,11 +34,11 @@ def find_bpf_chunks(tree):
def processor(source_code, filename, module):
tree = ast.parse(source_code, filename)
print(ast.dump(tree, indent=4))
logger.debug(ast.dump(tree, indent=4))
bpf_chunks = find_bpf_chunks(tree)
for func_node in bpf_chunks:
print(f"Found BPF function/struct: {func_node.name}")
logger.info(f"Found BPF function/struct: {func_node.name}")
structs_sym_tab = structs_proc(tree, module, bpf_chunks)
map_sym_tab = maps_proc(tree, module, bpf_chunks)
@ -44,7 +48,10 @@ def processor(source_code, filename, module):
globals_processing(tree, module)
def compile_to_ir(filename: str, output: str):
def compile_to_ir(filename: str, output: str, loglevel=logging.WARNING):
logging.basicConfig(
level=loglevel, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s"
)
with open(filename) as f:
source = f.read()
@ -53,33 +60,17 @@ def compile_to_ir(filename: str, output: str):
module.triple = "bpf"
if not hasattr(module, "_debug_compile_unit"):
module._file_metadata = module.add_debug_info(
"DIFile",
{ # type: ignore
"filename": filename,
"directory": os.path.dirname(filename),
},
debug_generator = DebugInfoGenerator(module)
debug_generator.generate_file_metadata(filename, os.path.dirname(filename))
debug_generator.generate_debug_cu(
DW_LANG_C11,
f"PythonBPF {VERSION}",
True, # TODO: This is probably not true
# TODO: add a global field here that keeps track of all the globals. Works without it, but I think it might
# be required for kprobes.
True,
)
module._debug_compile_unit = module.add_debug_info(
"DICompileUnit",
{ # type: ignore
"language": DW_LANG_C11,
"file": module._file_metadata, # type: ignore
"producer": f"PythonBPF {VERSION}",
"isOptimized": True, # TODO: This is probably not true
# TODO: add a global field here that keeps track of all the globals. Works without it, but I think it might
# be required for kprobes.
"runtimeVersion": 0,
"emissionKind": 1,
"splitDebugInlining": False,
"nameTableKind": 0,
},
is_distinct=True,
)
module.add_named_metadata("llvm.dbg.cu", module._debug_compile_unit) # type: ignore
processor(source, filename, module)
wchar_size = module.add_metadata(
@ -121,7 +112,7 @@ def compile_to_ir(filename: str, output: str):
module.add_named_metadata("llvm.ident", [f"PythonBPF {VERSION}"])
print(f"IR written to {output}")
logger.info(f"IR written to {output}")
with open(output, "w") as f:
f.write(f'source_filename = "{filename}"\n')
f.write(str(module))
@ -130,7 +121,7 @@ def compile_to_ir(filename: str, output: str):
return output
def compile() -> bool:
def compile(loglevel=logging.WARNING) -> bool:
# Look one level up the stack to the caller of this function
caller_frame = inspect.stack()[1]
caller_file = Path(caller_frame.filename).resolve()
@ -139,7 +130,9 @@ def compile() -> bool:
o_file = caller_file.with_suffix(".o")
success = True
success = compile_to_ir(str(caller_file), str(ll_file)) and success
success = (
compile_to_ir(str(caller_file), str(ll_file), loglevel=loglevel) and success
)
success = bool(
subprocess.run(
@ -157,11 +150,11 @@ def compile() -> bool:
and success
)
print(f"Object written to {o_file}")
logger.info(f"Object written to {o_file}")
return success
def BPF() -> BpfProgram:
def BPF(loglevel=logging.WARNING) -> BpfProgram:
caller_frame = inspect.stack()[1]
src = inspect.getsource(caller_frame.frame)
with tempfile.NamedTemporaryFile(
@ -174,7 +167,7 @@ def BPF() -> BpfProgram:
f.write(src)
f.flush()
source = f.name
compile_to_ir(source, str(inter.name))
compile_to_ir(source, str(inter.name), loglevel=loglevel)
subprocess.run(
[
"llc",

View File

@ -12,6 +12,34 @@ class DebugInfoGenerator:
self.module = module
self._type_cache = {} # Cache for common debug types
def generate_file_metadata(self, filename, dirname):
self.module._file_metadata = self.module.add_debug_info(
"DIFile",
{ # type: ignore
"filename": filename,
"directory": dirname,
},
)
def generate_debug_cu(
self, language, producer: str, is_optimized: bool, is_distinct: bool
):
self.module._debug_compile_unit = self.module.add_debug_info(
"DICompileUnit",
{ # type: ignore
"language": language,
"file": self.module._file_metadata, # type: ignore
"producer": producer,
"isOptimized": is_optimized,
"runtimeVersion": 0,
"emissionKind": 1,
"splitDebugInlining": False,
"nameTableKind": 0,
},
is_distinct=is_distinct,
)
self.module.add_named_metadata("llvm.dbg.cu", self.module._debug_compile_unit) # type: ignore
def get_basic_type(self, name: str, size: int, encoding: int) -> Any:
"""Get or create a basic type with caching"""
key = (name, size, encoding)

View File

@ -1,5 +1,9 @@
import ast
from llvmlite import ir
from logging import Logger
import logging
logger: Logger = logging.getLogger(__name__)
def eval_expr(
@ -11,14 +15,14 @@ def eval_expr(
map_sym_tab,
structs_sym_tab=None,
):
print(f"Evaluating expression: {ast.dump(expr)}")
logger.info(f"Evaluating expression: {ast.dump(expr)}")
if isinstance(expr, ast.Name):
if expr.id in local_sym_tab:
var = local_sym_tab[expr.id].var
val = builder.load(var)
return val, local_sym_tab[expr.id].ir_type # return value and type
else:
print(f"Undefined variable {expr.id}")
logger.info(f"Undefined variable {expr.id}")
return None
elif isinstance(expr, ast.Constant):
if isinstance(expr.value, int):
@ -26,7 +30,7 @@ def eval_expr(
elif isinstance(expr.value, bool):
return ir.Constant(ir.IntType(1), int(expr.value)), ir.IntType(1)
else:
print("Unsupported constant type")
logger.info("Unsupported constant type")
return None
elif isinstance(expr, ast.Call):
# delayed import to avoid circular dependency
@ -35,9 +39,9 @@ def eval_expr(
if isinstance(expr.func, ast.Name):
# check deref
if expr.func.id == "deref":
print(f"Handling deref {ast.dump(expr)}")
logger.info(f"Handling deref {ast.dump(expr)}")
if len(expr.args) != 1:
print("deref takes exactly one argument")
logger.info("deref takes exactly one argument")
return None
arg = expr.args[0]
if (
@ -45,16 +49,16 @@ def eval_expr(
and isinstance(arg.func, ast.Name)
and arg.func.id == "deref"
):
print("Multiple deref not supported")
logger.info("Multiple deref not supported")
return None
if isinstance(arg, ast.Name):
if arg.id in local_sym_tab:
arg = local_sym_tab[arg.id].var
else:
print(f"Undefined variable {arg.id}")
logger.info(f"Undefined variable {arg.id}")
return None
if arg is None:
print("Failed to evaluate deref argument")
logger.info("Failed to evaluate deref argument")
return None
# Since we are handling only name case, directly take type from sym tab
val = builder.load(arg)
@ -72,7 +76,7 @@ def eval_expr(
structs_sym_tab,
)
elif isinstance(expr.func, ast.Attribute):
print(f"Handling method call: {ast.dump(expr.func)}")
logger.info(f"Handling method call: {ast.dump(expr.func)}")
if isinstance(expr.func.value, ast.Call) and isinstance(
expr.func.value.func, ast.Name
):
@ -107,15 +111,15 @@ def eval_expr(
attr_name = expr.attr
if var_name in local_sym_tab:
var_ptr, var_type, var_metadata = local_sym_tab[var_name]
print(f"Loading attribute {attr_name} from variable {var_name}")
print(f"Variable type: {var_type}, Variable ptr: {var_ptr}")
logger.info(f"Loading attribute {attr_name} from variable {var_name}")
logger.info(f"Variable type: {var_type}, Variable ptr: {var_ptr}")
metadata = structs_sym_tab[var_metadata]
if attr_name in metadata.fields:
gep = metadata.gep(builder, var_ptr, attr_name)
val = builder.load(gep)
field_type = metadata.field_type(attr_name)
return val, field_type
print("Unsupported expression evaluation")
logger.info("Unsupported expression evaluation")
return None
@ -129,7 +133,7 @@ def handle_expr(
structs_sym_tab,
):
"""Handle expression statements in the function body."""
print(f"Handling expression: {ast.dump(expr)}")
logger.info(f"Handling expression: {ast.dump(expr)}")
call = expr.value
if isinstance(call, ast.Call):
eval_expr(
@ -142,4 +146,4 @@ def handle_expr(
structs_sym_tab,
)
else:
print("Unsupported expression type")
logger.info("Unsupported expression type")

View File

@ -46,15 +46,15 @@ def handle_assign(
):
"""Handle assignment statements in the function body."""
if len(stmt.targets) != 1:
print("Unsupported multiassignment")
logger.info("Unsupported multiassignment")
return
num_types = ("c_int32", "c_int64", "c_uint32", "c_uint64")
target = stmt.targets[0]
print(f"Handling assignment to {ast.dump(target)}")
logger.info(f"Handling assignment to {ast.dump(target)}")
if not isinstance(target, ast.Name) and not isinstance(target, ast.Attribute):
print("Unsupported assignment target")
logger.info("Unsupported assignment target")
return
var_name = target.id if isinstance(target, ast.Name) else target.value.id
rval = stmt.value
@ -87,11 +87,11 @@ def handle_assign(
# print(f"Assigned to struct field {var_name}.{field_name}")
pass
if val is None:
print("Failed to evaluate struct field assignment")
logger.info("Failed to evaluate struct field assignment")
return
print(field_ptr)
logger.info(field_ptr)
builder.store(val[0], field_ptr)
print(f"Assigned to struct field {var_name}.{field_name}")
logger.info(f"Assigned to struct field {var_name}.{field_name}")
return
elif isinstance(rval, ast.Constant):
if isinstance(rval.value, bool):
@ -103,7 +103,7 @@ def handle_assign(
builder.store(
ir.Constant(ir.IntType(1), 0), local_sym_tab[var_name].var
)
print(f"Assigned constant {rval.value} to {var_name}")
logger.info(f"Assigned constant {rval.value} to {var_name}")
elif isinstance(rval.value, int):
# Assume c_int64 for now
# var = builder.alloca(ir.IntType(64), name=var_name)
@ -111,7 +111,7 @@ def handle_assign(
builder.store(
ir.Constant(ir.IntType(64), rval.value), local_sym_tab[var_name].var
)
print(f"Assigned constant {rval.value} to {var_name}")
logger.info(f"Assigned constant {rval.value} to {var_name}")
elif isinstance(rval.value, str):
str_val = rval.value.encode("utf-8") + b"\x00"
str_const = ir.Constant(
@ -125,13 +125,13 @@ def handle_assign(
global_str.initializer = str_const
str_ptr = builder.bitcast(global_str, ir.PointerType(ir.IntType(8)))
builder.store(str_ptr, local_sym_tab[var_name].var)
print(f"Assigned string constant '{rval.value}' to {var_name}")
logger.info(f"Assigned string constant '{rval.value}' to {var_name}")
else:
print("Unsupported constant type")
logger.info("Unsupported constant type")
elif isinstance(rval, ast.Call):
if isinstance(rval.func, ast.Name):
call_type = rval.func.id
print(f"Assignment call type: {call_type}")
logger.info(f"Assignment call type: {call_type}")
if (
call_type in num_types
and len(rval.args) == 1
@ -145,7 +145,7 @@ def handle_assign(
ir.Constant(ir_type, rval.args[0].value),
local_sym_tab[var_name].var,
)
print(
logger.info(
f"Assigned {call_type} constant "
f"{rval.args[0].value} to {var_name}"
)
@ -162,9 +162,9 @@ def handle_assign(
structs_sym_tab,
)
builder.store(val[0], local_sym_tab[var_name].var)
print(f"Assigned constant {rval.func.id} to {var_name}")
logger.info(f"Assigned constant {rval.func.id} to {var_name}")
elif call_type == "deref" and len(rval.args) == 1:
print(f"Handling deref assignment {ast.dump(rval)}")
logger.info(f"Handling deref assignment {ast.dump(rval)}")
val = eval_expr(
func,
module,
@ -175,22 +175,22 @@ def handle_assign(
structs_sym_tab,
)
if val is None:
print("Failed to evaluate deref argument")
logger.info("Failed to evaluate deref argument")
return
print(f"Dereferenced value: {val}, storing in {var_name}")
logger.info(f"Dereferenced value: {val}, storing in {var_name}")
builder.store(val[0], local_sym_tab[var_name].var)
print(f"Dereferenced and assigned to {var_name}")
logger.info(f"Dereferenced and assigned to {var_name}")
elif call_type in structs_sym_tab and len(rval.args) == 0:
struct_info = structs_sym_tab[call_type]
ir_type = struct_info.ir_type
# var = builder.alloca(ir_type, name=var_name)
# Null init
builder.store(ir.Constant(ir_type, None), local_sym_tab[var_name].var)
print(f"Assigned struct {call_type} to {var_name}")
logger.info(f"Assigned struct {call_type} to {var_name}")
else:
print(f"Unsupported assignment call type: {call_type}")
logger.info(f"Unsupported assignment call type: {call_type}")
elif isinstance(rval.func, ast.Attribute):
print(f"Assignment call attribute: {ast.dump(rval.func)}")
logger.info(f"Assignment call attribute: {ast.dump(rval.func)}")
if isinstance(rval.func.value, ast.Name):
if rval.func.value.id in map_sym_tab:
map_name = rval.func.value.id
@ -208,7 +208,7 @@ def handle_assign(
builder.store(val[0], local_sym_tab[var_name].var)
else:
# TODO: probably a struct access
print(f"TODO STRUCT ACCESS {ast.dump(rval)}")
logger.info(f"TODO STRUCT ACCESS {ast.dump(rval)}")
elif isinstance(rval.func.value, ast.Call) and isinstance(
rval.func.value.func, ast.Name
):
@ -229,15 +229,15 @@ def handle_assign(
# var.align = 8
builder.store(val[0], local_sym_tab[var_name].var)
else:
print("Unsupported assignment call structure")
logger.info("Unsupported assignment call structure")
else:
print("Unsupported assignment call function type")
logger.info("Unsupported assignment call function type")
elif isinstance(rval, ast.BinOp):
handle_binary_op(
rval, module, builder, var_name, local_sym_tab, map_sym_tab, func
)
else:
print("Unsupported assignment value type")
logger.info("Unsupported assignment value type")
def handle_cond(func, module, builder, cond, local_sym_tab, map_sym_tab):
@ -247,7 +247,7 @@ def handle_cond(func, module, builder, cond, local_sym_tab, map_sym_tab):
elif isinstance(cond.value, int):
return ir.Constant(ir.IntType(1), int(bool(cond.value)))
else:
print("Unsupported constant type in condition")
logger.info("Unsupported constant type in condition")
return None
elif isinstance(cond, ast.Name):
if cond.id in local_sym_tab:
@ -264,12 +264,12 @@ def handle_cond(func, module, builder, cond, local_sym_tab, map_sym_tab):
val = builder.icmp_signed("!=", val, zero)
return val
else:
print(f"Undefined variable {cond.id} in condition")
logger.info(f"Undefined variable {cond.id} in condition")
return None
elif isinstance(cond, ast.Compare):
lhs = eval_expr(func, module, builder, cond.left, local_sym_tab, map_sym_tab)[0]
if len(cond.ops) != 1 or len(cond.comparators) != 1:
print("Unsupported complex comparison")
logger.info("Unsupported complex comparison")
return None
rhs = eval_expr(
func, module, builder, cond.comparators[0], local_sym_tab, map_sym_tab
@ -284,7 +284,7 @@ def handle_cond(func, module, builder, cond, local_sym_tab, map_sym_tab):
elif lhs.type.width > rhs.type.width:
rhs = builder.sext(rhs, lhs.type)
else:
print("Type mismatch in comparison")
logger.info("Type mismatch in comparison")
return None
if isinstance(op, ast.Eq):
@ -300,10 +300,10 @@ def handle_cond(func, module, builder, cond, local_sym_tab, map_sym_tab):
elif isinstance(op, ast.GtE):
return builder.icmp_signed(">=", lhs, rhs)
else:
print("Unsupported comparison operator")
logger.info("Unsupported comparison operator")
return None
else:
print("Unsupported condition expression")
logger.info("Unsupported condition expression")
return None
@ -311,7 +311,7 @@ def handle_if(
func, module, builder, stmt, map_sym_tab, local_sym_tab, structs_sym_tab=None
):
"""Handle if statements in the function body."""
print("Handling if statement")
logger.info("Handling if statement")
# start = builder.block.parent
then_block = func.append_basic_block(name="if.then")
merge_block = func.append_basic_block(name="if.end")
@ -364,7 +364,7 @@ def process_stmt(
did_return,
ret_type=ir.IntType(64),
):
print(f"Processing statement: {ast.dump(stmt)}")
logger.info(f"Processing statement: {ast.dump(stmt)}")
if isinstance(stmt, ast.Expr):
handle_expr(
func,
@ -449,11 +449,11 @@ def allocate_mem(
)
elif isinstance(stmt, ast.Assign):
if len(stmt.targets) != 1:
print("Unsupported multiassignment")
logger.info("Unsupported multiassignment")
continue
target = stmt.targets[0]
if not isinstance(target, ast.Name):
print("Unsupported assignment target")
logger.info("Unsupported assignment target")
continue
var_name = target.id
rval = stmt.value
@ -464,25 +464,27 @@ def allocate_mem(
ir_type = ctypes_to_ir(call_type)
var = builder.alloca(ir_type, name=var_name)
var.align = ir_type.width // 8
print(f"Pre-allocated variable {var_name} of type {call_type}")
logger.info(
f"Pre-allocated variable {var_name} of type {call_type}"
)
elif HelperHandlerRegistry.has_handler(call_type):
# Assume return type is int64 for now
ir_type = ir.IntType(64)
var = builder.alloca(ir_type, name=var_name)
var.align = ir_type.width // 8
print(f"Pre-allocated variable {var_name} for helper")
logger.info(f"Pre-allocated variable {var_name} for helper")
elif call_type == "deref" and len(rval.args) == 1:
# Assume return type is int64 for now
ir_type = ir.IntType(64)
var = builder.alloca(ir_type, name=var_name)
var.align = ir_type.width // 8
print(f"Pre-allocated variable {var_name} for deref")
logger.info(f"Pre-allocated variable {var_name} for deref")
elif call_type in structs_sym_tab:
struct_info = structs_sym_tab[call_type]
ir_type = struct_info.ir_type
var = builder.alloca(ir_type, name=var_name)
has_metadata = True
print(
logger.info(
f"Pre-allocated variable {var_name} "
f"for struct {call_type}"
)
@ -490,38 +492,38 @@ def allocate_mem(
ir_type = ir.PointerType(ir.IntType(64))
var = builder.alloca(ir_type, name=var_name)
# var.align = ir_type.width // 8
print(f"Pre-allocated variable {var_name} for map")
logger.info(f"Pre-allocated variable {var_name} for map")
else:
print("Unsupported assignment call function type")
logger.info("Unsupported assignment call function type")
continue
elif isinstance(rval, ast.Constant):
if isinstance(rval.value, bool):
ir_type = ir.IntType(1)
var = builder.alloca(ir_type, name=var_name)
var.align = 1
print(f"Pre-allocated variable {var_name} of type c_bool")
logger.info(f"Pre-allocated variable {var_name} of type c_bool")
elif isinstance(rval.value, int):
# Assume c_int64 for now
ir_type = ir.IntType(64)
var = builder.alloca(ir_type, name=var_name)
var.align = ir_type.width // 8
print(f"Pre-allocated variable {var_name} of type c_int64")
logger.info(f"Pre-allocated variable {var_name} of type c_int64")
elif isinstance(rval.value, str):
ir_type = ir.PointerType(ir.IntType(8))
var = builder.alloca(ir_type, name=var_name)
var.align = 8
print(f"Pre-allocated variable {var_name} of type string")
logger.info(f"Pre-allocated variable {var_name} of type string")
else:
print("Unsupported constant type")
logger.info("Unsupported constant type")
continue
elif isinstance(rval, ast.BinOp):
# Assume c_int64 for now
ir_type = ir.IntType(64)
var = builder.alloca(ir_type, name=var_name)
var.align = ir_type.width // 8
print(f"Pre-allocated variable {var_name} of type c_int64")
logger.info(f"Pre-allocated variable {var_name} of type c_int64")
else:
print("Unsupported assignment value type")
logger.info("Unsupported assignment value type")
continue
if has_metadata:
@ -552,7 +554,7 @@ def process_func_body(
structs_sym_tab,
)
print(f"Local symbol table: {local_sym_tab.keys()}")
logger.info(f"Local symbol table: {local_sym_tab.keys()}")
for stmt in func_node.body:
did_return = process_stmt(
@ -624,7 +626,7 @@ def func_proc(tree, module, chunks, map_sym_tab, structs_sym_tab):
if is_global:
continue
func_type = get_probe_string(func_node)
print(f"Found probe_string of {func_node.name}: {func_type}")
logger.info(f"Found probe_string of {func_node.name}: {func_type}")
process_bpf_chunk(
func_node,

View File

@ -9,6 +9,10 @@ from .helper_utils import (
simple_string_print,
get_data_ptr_and_size,
)
from logging import Logger
import logging
logger: Logger = logging.getLogger(__name__)
class BPFHelperID(Enum):
@ -322,7 +326,7 @@ def handle_helper_call(
elif isinstance(call.func, ast.Attribute):
method_name = call.func.attr
value = call.func.value
print(f"Handling method call: {ast.dump(call.func)}")
logger.info(f"Handling method call: {ast.dump(call.func)}")
# Get map pointer from different styles of map access
if isinstance(value, ast.Call) and isinstance(value.func, ast.Name):
# Func style: my_map().lookup(key)

View File

@ -1,5 +1,9 @@
from llvmlite import ir
import ast
from logging import Logger
import logging
logger: Logger = logging.getLogger(__name__)
def emit_license(module: ir.Module, license_str: str):
@ -41,9 +45,9 @@ def license_processing(tree, module):
emit_license(module, node.body[0].value.value)
return "LICENSE"
else:
print("ERROR: LICENSE() must return a string literal")
logger.info("ERROR: LICENSE() must return a string literal")
return None
else:
print("ERROR: LICENSE already defined")
logger.info("ERROR: LICENSE already defined")
return None
return None

View File

@ -19,7 +19,7 @@ def structs_proc(tree, module, chunks):
structs_sym_tab = {}
for cls_node in chunks:
if is_bpf_struct(cls_node):
print(f"Found BPF struct: {cls_node.name}")
logger.info(f"Found BPF struct: {cls_node.name}")
struct_info = process_bpf_struct(cls_node, module)
structs_sym_tab[cls_node.name] = struct_info
return structs_sym_tab