diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..350fbce --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,25 @@ +name: Build ePass + +on: + push: + branches: [ "main", "dev" ] + +env: + BUILD_TYPE: Release + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Install Dependency + run: sudo apt install -y libbpf-dev + + - name: Configure CMake + run: cd core && make configure + + - name: Build + run: cd core && make + diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..84ed816 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,15 @@ +[submodule "third-party/ePass-xdp-tools"] + path = third-party/ePass-xdp-tools + url = git@github.com:OrderLab/ePass-xdp-tools.git +[submodule "third-party/ePass-falcolib"] + path = third-party/ePass-falcolib + url = git@github.com:OrderLab/ePass-falcolib.git +[submodule "ePass-kernel"] + path = ePass-kernel + url = git@github.com:OrderLab/ePass-kernel.git +[submodule "third-party/ePass-libbpf"] + path = third-party/ePass-libbpf + url = git@github.com:OrderLab/ePass-libbpf.git +[submodule "third-party/ePass-bpftool"] + path = third-party/ePass-bpftool + url = git@github.com:OrderLab/ePass-bpftool.git diff --git a/README.md b/README.md index c7a04db..07bd454 100644 --- a/README.md +++ b/README.md @@ -1 +1,68 @@ # ePass + +[![Build ePass](https://github.com/OrderLab/ePass/actions/workflows/build.yml/badge.svg)](https://github.com/OrderLab/ePass/actions/workflows/build.yml) + +ePass is an in-kernel LLVM-like compiler framework that introduces an SSA-based intermediate representation (IR) for eBPF programs. It provides a lifter that lifts eBPF bytecode to ePass IR, a pass runner that runs user-defined passes, and a code generator that compiles IR to eBPF bytecode. Users could write flexible passes using our LLVM-like APIs to analyze and manipulate the IR. +ePass also provides an in-kernel supervisor that cooperates ePass core with the verifier to improve its flexibility (i.e. reduce false rejections) and safety (i.e. reduce false acceptance at runtime). It could also be used in userspace for testing. + +## Key Features + +- **IR-based compilation**: Converts BPF programs to an SSA-based intermediate representation for code rewriting +- **Flexible passes**: ePass core provides various APIs to analyze and manipulate the IR, allowing users to write flexible passes including static analyzing, runtime checks, and optimization. +- **Verifier aware**: ePass works with the existing verifier. The verifier is better for static verification while ePass focuses more on code rewriting and runtime verification. +- **User-friendly debugging**: ePass supports compiling to both kernel and userspace for easier debugging. + +> ⚠️ Warning: ePass is under active development and we are improving its usability and safety for production use. We welcome any suggestions and feedback. Feel free to open issues or [contact us](#contact-and-citation). + +## Design Goals + +- Flexible passes for diverse use cases +- Working with existing verifier instead of replacing its +- Keeping kernel safety +- Support both userspace and kernel + +## Prerequisites + +- **clang >= 17** +- **Ninja** (optional, for faster compilation) +- **libbpf** + +## Project Components + +- `ePass core`: the core compiler framework, including a userspace CLI +- `ePass kernel`: Linux kernel 6.5 with ePass core built-in, along with the kernel component and kernel passes +- `ePass libbpf`: libbpf with ePass support for userspace ePass testing + +There are some testing projects including `bpftool`, `xdp-tools`, `falcolib` in `third-party`. They depend on `ePass libbpf`. + +### ePass Overview + +![Overview](./docs/overview.png) + +### ePass Core + +![Core Architecture](./docs/core_design.png) + +## Quick Start + +There are two ways to use ePass. The first way is to build a linux kernel with ePass builtin, which is used for production. Users could specify ePass options when calling the `BPF` system call. See [Kernel Testing](docs/KERNEL_TESTING.md). + +The second way is to build ePass in userspace and testing programs without changing the kernel, which is used mainly for testing. Users could specify ePass options via environment variable and use `ePass libbpf`. Programs will be modified in userspace before sending to the kernel. See [Userspace Testing](docs/USERSPACE_TESTING.md). + +We recommend users trying ePass in userspace before switching to the ePass kernel version! + +## Testing + +See [Testing](./docs/TESTING.md). + +## Development and Contribution + +See [Development](./docs/CONTRIBUTION_GUIDE.md). + +## Contact and Citation + +Feel free to open an issue for question, bug report or feature request! You could also email . + +## Acknowledgement + +ePass is sponsored by [OrderLab](https://orderlab.io/) from University of Michigan. diff --git a/core/Makefile b/core/Makefile index 8480253..1e1f2ab 100644 --- a/core/Makefile +++ b/core/Makefile @@ -28,8 +28,11 @@ configure: cmake -S . -B $(BUILD_DIR) -G Ninja; \ fi +install: build + sudo cmake --install $(BUILD_DIR) + buildobj: ./scripts/buildobj.sh -.PHONY: build format kernel genctor test configure buildobj +.PHONY: build format kernel genctor test configure buildobj install .DEFAULT_GOAL := build diff --git a/core/Readme.md b/core/Readme.md deleted file mode 100644 index 0c11c97..0000000 --- a/core/Readme.md +++ /dev/null @@ -1,53 +0,0 @@ -# Dependencies - -- clang >= 17 (<17 could build the epass tool but could not compile the BPF programs properly) -- Ninja (optional, for faster compilation speed) -- libbpf (you could use ePass libbpf if running ePass kernel) - -# Build - -If you are using normal kernel, inside this directory, run: - -``` -cmake -S . -B build -GNinja -DEPASS_LIBBPF=OFF -make -``` - -If you are running our custom kernel with custom libbpf, compile with: - -``` -cmake -S . -B build -GNinja -DEPASS_LIBBPF=ON -make -``` - -# Install - -After building, run: - -``` -sudo cmake --install build -``` - -## Bugs - -### Coalesce has some bugs - -Found root cause: you may not directly remove instructions like r1 = r1. - -Note. this should be refactored to the last step in the new pipeline. - -## Errors - -Reproduce: `ringbuf.c` enable coalesce will cause some error in CG - -Raw libbpf library loader doesn't change the "imm" value when calling the `callback_fn`. It doesn't support calling it after changing the resources. - -## ALLOCARRAY? - -Why pre-colored? - -# TODO - -- [ ] bpf-to-bpf calls - -Move CG builtin passes to `ir_code_gen`. diff --git a/core/TODO.md b/core/TODO.md deleted file mode 100644 index 63beb81..0000000 --- a/core/TODO.md +++ /dev/null @@ -1,3 +0,0 @@ -# TODOs - -- [ ] Huge tests in falco diff --git a/core/docs/CG.md b/docs/CG.md similarity index 100% rename from core/docs/CG.md rename to docs/CG.md diff --git a/docs/CONTRIBUTION_GUIDE.md b/docs/CONTRIBUTION_GUIDE.md new file mode 100644 index 0000000..7ecea16 --- /dev/null +++ b/docs/CONTRIBUTION_GUIDE.md @@ -0,0 +1,34 @@ +# Contribution Guide + + +## Project Structure + +``` +ePass/ +├── core/ # Main compiler implementation +│ ├── include/ # Header files +│ ├── docs/ # Technical documentation +│ ├── passes/ # Optimization passes +│ ├── aux/ # Auxiliary utilities +│ ├── epasstool/ # CLI tool +│ └── tests/ # Simple BPF tests +├── test/ # Integration tests and evaluation +├── rejected/ # Collected rejected programs +└── tools/ # Helper scripts and utilities +``` + +## Common Development Patterns + +### Iterating Through Instructions + +```c +struct ir_basic_block **pos; +array_for(pos, fun->reachable_bbs) +{ + struct ir_basic_block *bb = *pos; + struct ir_insn *insn; + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + // Process instruction + } +} +``` diff --git a/docs/CREATE_INSTRUCTION.md b/docs/CREATE_INSTRUCTION.md new file mode 100644 index 0000000..48138f5 --- /dev/null +++ b/docs/CREATE_INSTRUCTION.md @@ -0,0 +1 @@ +# Create New Instruction \ No newline at end of file diff --git a/core/docs/Common.md b/docs/Common.md similarity index 100% rename from core/docs/Common.md rename to docs/Common.md diff --git a/core/docs/Config.md b/docs/Config.md similarity index 100% rename from core/docs/Config.md rename to docs/Config.md diff --git a/docs/EPASS_OPTIONS.md b/docs/EPASS_OPTIONS.md new file mode 100644 index 0000000..1dadda9 --- /dev/null +++ b/docs/EPASS_OPTIONS.md @@ -0,0 +1,7 @@ +# Epass Options + +## Global Options + + + +## Pass Options \ No newline at end of file diff --git a/core/docs/IR.md b/docs/IR.md similarity index 100% rename from core/docs/IR.md rename to docs/IR.md diff --git a/docs/KERNEL_TESTING.md b/docs/KERNEL_TESTING.md new file mode 100644 index 0000000..b669276 --- /dev/null +++ b/docs/KERNEL_TESTING.md @@ -0,0 +1 @@ +# Kernel Testing \ No newline at end of file diff --git a/core/docs/Lift.md b/docs/Lift.md similarity index 100% rename from core/docs/Lift.md rename to docs/Lift.md diff --git a/docs/TESTING.md b/docs/TESTING.md new file mode 100644 index 0000000..8c18090 --- /dev/null +++ b/docs/TESTING.md @@ -0,0 +1 @@ +# ePass Testing \ No newline at end of file diff --git a/docs/TODO.md b/docs/TODO.md new file mode 100644 index 0000000..aebb85a --- /dev/null +++ b/docs/TODO.md @@ -0,0 +1,5 @@ +# TODOs + +- [ ] Huge tests in falco +- [ ] bpf-to-bpf calls +- [ ] Full test suite diff --git a/core/docs/Type.md b/docs/Type.md similarity index 100% rename from core/docs/Type.md rename to docs/Type.md diff --git a/docs/USERSPACE_TESTING.md b/docs/USERSPACE_TESTING.md new file mode 100644 index 0000000..16b6181 --- /dev/null +++ b/docs/USERSPACE_TESTING.md @@ -0,0 +1,65 @@ +# Userspace Testing + +The main development happens in `core` directory. To start, `cd` into `core`. + +### Build + +```bash +make configure # Do it once + +make build +``` + +### Install + +```bash +make install +``` + +### Basic Usage + +```bash +# Run ePass on the program +epass read prog.o + +# Run ePass on the program with gopt and popt +epass read --popt popts --gopt gopts prog.o + +# Print the BPF program +epass print prog.o +``` + +For `gopt` and `popt`, see [ePass Options](./EPASS_OPTIONS.md). + +### Use ePass with `libbpf` + +We may want to load a ePass-modified program to the kernel to see its effect. ePass provides a modified libbpf that allows users to run ePass before loading programs to the kernel. The advantage is that you do not need to change the kernel. However, running ePass in userspace cannot leverage the verifier, so it cannot use verifier information, cannot run verifier dependent passes, and cannot run kernel passes. + +First, initializing all submodules. + +```bash +git submodule update --init --recursive +``` + +Now open the `libbpf` source code directory and build: + +```bash +cd third-party/ePass-libbpf/src +make -j +``` + +To install `ePass libbpf`, install: + +```bash +sudo make install +``` + +After installing ePass libbpf, you could run any programs that depends on the `libbpf` shared library with `ePass` commands. + +For `bpftool`, you need to build `bpftool` because by default it statically link `libbpf`. + +An example of using ePass to load `test.o` eBPF program using `bpftool`: + +```bash +sudo LIBBPF_ENABLE_EPASS=1 LIBBPF_EPASS_GOPT="verbose=3" LIBBPF_EPASS_POPT="msan" bpftool prog load test.o /sys/fs/bpf/test +``` \ No newline at end of file diff --git a/core/docs/Userspace.md b/docs/Userspace.md similarity index 100% rename from core/docs/Userspace.md rename to docs/Userspace.md diff --git a/core/docs/Verifier.md b/docs/Verifier.md similarity index 100% rename from core/docs/Verifier.md rename to docs/Verifier.md diff --git a/docs/core_design.png b/docs/core_design.png new file mode 100644 index 0000000..74822a2 --- /dev/null +++ b/docs/core_design.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2353446b85a4e8021f7688cfc21187a4403dac5a127b886b08af2f071caf78c5 +size 7638 diff --git a/docs/overview.png b/docs/overview.png new file mode 100644 index 0000000..814708f --- /dev/null +++ b/docs/overview.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd7d261c18cfceed6cd0f9af065a54480a92e3b5d8d066728702a10e128b4020 +size 3619 diff --git a/ePass-kernel b/ePass-kernel new file mode 160000 index 0000000..be091e0 --- /dev/null +++ b/ePass-kernel @@ -0,0 +1 @@ +Subproject commit be091e0a4da8a0367eafb9c262150b0631277647 diff --git a/third-party/ePass-bpftool b/third-party/ePass-bpftool new file mode 160000 index 0000000..f970171 --- /dev/null +++ b/third-party/ePass-bpftool @@ -0,0 +1 @@ +Subproject commit f9701712ffa99ce245b0da8119f77441c0f5e6a7 diff --git a/third-party/ePass-falcolib b/third-party/ePass-falcolib new file mode 160000 index 0000000..ae37f40 --- /dev/null +++ b/third-party/ePass-falcolib @@ -0,0 +1 @@ +Subproject commit ae37f4027ece58f4c26f973f0712b5961bf0daac diff --git a/third-party/ePass-libbpf b/third-party/ePass-libbpf new file mode 160000 index 0000000..6297d29 --- /dev/null +++ b/third-party/ePass-libbpf @@ -0,0 +1 @@ +Subproject commit 6297d2969531f1310461d4ef1178c7e8ba41ec9b diff --git a/third-party/ePass-xdp-tools b/third-party/ePass-xdp-tools new file mode 160000 index 0000000..a0bbee0 --- /dev/null +++ b/third-party/ePass-xdp-tools @@ -0,0 +1 @@ +Subproject commit a0bbee0248699a10938e23c22275a70ed852dff1