diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml index b370b614..defe3df7 100644 --- a/.github/workflows/format-check.yml +++ b/.github/workflows/format-check.yml @@ -7,9 +7,7 @@ jobs: name: Check format of C runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - with: - submodules: recursive + - uses: actions/checkout@v6 - name: Run clang-format style check for C/C++ sources uses: Northeastern-Electric-Racing/clang-format-action@main with: diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..f9d61944 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "dev/OpenOCD"] + path = dev/OpenOCD + url = https://github.com/STMicroelectronics/OpenOCD diff --git a/dev/.clang-format-ignore b/dev/.clang-format-ignore new file mode 100644 index 00000000..3139ffaa --- /dev/null +++ b/dev/.clang-format-ignore @@ -0,0 +1 @@ +OpenOCD/* diff --git a/dev/.dockerignore b/dev/.dockerignore new file mode 100644 index 00000000..94143827 --- /dev/null +++ b/dev/.dockerignore @@ -0,0 +1 @@ +Dockerfile diff --git a/dev/Dockerfile b/dev/Dockerfile index 1a988da1..1df09d12 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -10,7 +10,6 @@ RUN apt-get update && apt-get install -y \ build-essential \ wget \ curl \ - openocd \ git \ gdb-multiarch \ minicom \ @@ -26,16 +25,17 @@ RUN apt-get update && apt-get install -y \ python3-pip \ ruby + RUN wget https://github.com/renode/renode/releases/download/v1.15.3/renode-1.15.3.linux-portable.tar.gz -RUN mkdir renode_portable && tar -xvf renode-*.linux-portable.tar.gz -C renode_portable --strip-components=1 +RUN mkdir renode_portable && tar -xvf renode-*.linux-portable.tar.gz -C renode_portable --strip-components=1 && rm renode-*.linux-portable.tar.gz ENV PATH $PATH:/renode_portable RUN wget https://github.com/ThrowTheSwitch/CMock/archive/refs/tags/v2.6.0.tar.gz -O cmock.tar.gz -RUN mkdir cmock_portable && tar -xvf cmock.tar.gz -C cmock_portable --strip-components=1 +RUN mkdir cmock_portable && tar -xvf cmock.tar.gz -C cmock_portable --strip-components=1 && rm cmock.tar.gz ENV PATH $PATH:/cmock_portable RUN wget https://github.com/ThrowTheSwitch/Unity/archive/refs/tags/v2.6.1.tar.gz -O unity.tar.gz -RUN mkdir -p /cmock_portable/vendor/unity && tar -xvf unity.tar.gz -C /cmock_portable/vendor/unity --strip-components=1 +RUN mkdir -p /cmock_portable/vendor/unity && tar -xvf unity.tar.gz -C /cmock_portable/vendor/unity --strip-components=1 && rm unity.tar.gz # Set up a development tools directory WORKDIR /home/dev @@ -53,6 +53,13 @@ RUN wget -qO- https://developer.arm.com/-/media/Files/downloads/gnu/11.3.rel1/bi ENV PATH $PATH:/home/dev/arm-gnu-toolchain-11.3.rel1-x86_64-arm-none-eabi/bin +# build and install customized openocd +RUN apt-get install -y libtool libusb-1.0.0-dev +RUN git clone https://github.com/STMicroelectronics/OpenOCD +RUN cd ./OpenOCD && ./bootstrap && ./configure --enable-stlink +RUN cd ./OpenOCD && make && make install +RUN rm -r ./OpenOCD + WORKDIR /home/app # Set up safe directory diff --git a/dev/OpenOCD b/dev/OpenOCD new file mode 160000 index 00000000..0de861e2 --- /dev/null +++ b/dev/OpenOCD @@ -0,0 +1 @@ +Subproject commit 0de861e21c12fd39fdce7b6296994d89eca6146f diff --git a/ner_environment/build_system/build_system.py b/ner_environment/build_system/build_system.py index b1cfd5bd..80df52c0 100644 --- a/ner_environment/build_system/build_system.py +++ b/ner_environment/build_system/build_system.py @@ -8,12 +8,14 @@ # Provided here is a comprehensive set of commands that allow for the complete interaction with NERs toolset. # Each command is defined as a function and is called by the main function, and has help messages and configuration options. # Commands can be run with the virtual environment active by prefixing the command with `ner `. -# +# # See more documentation at https://nerdocs.atlassian.net/wiki/spaces/NER/pages/611516420/NER+Build+System # # To see a list of available commands and additional configuration options, run `ner --help` # ============================================================================== +from enum import Enum import shutil +import requests import typer from rich import print import platform @@ -21,7 +23,9 @@ import sys import os import glob +import re import time +import json from pathlib import Path # custom modules for functinality that is too large to be included in this script directly @@ -31,19 +35,21 @@ # ============================================================================== # Typer application setup # ============================================================================== - + app = typer.Typer(help="Northeastern Electric Racing Firmware Build System", epilog="For more information, visit https://nerdocs.atlassian.net/wiki/spaces/NER/pages/524451844/2024+Firmware+Onboarding+Master", add_completion=False) -lp_app = typer.Typer(help="Install configure, and run launchpad environment items") -app.add_typer(lp_app, name="lp") - def unsupported_option_cb(value:bool): if value: print("[bold red] WARNING: the selected option is not currently implemented. This is either because is is an planned or deprecated feature") raise typer.Exit() - + +class OpenOCDLocation(str, Enum): + cube = "cube" + sys = "sys" + dl = "dl" + # ============================================================================== # Build command # ============================================================================== @@ -59,6 +65,7 @@ def build(profile: str = typer.Option(None, "--profile", "-p", callback=unsuppor print("[#cccccc](ner build):[/#cccccc] [green]Ran build-cleaning command.[/green]") else: run_command_docker("mkdir -p build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=cmake/gcc-arm-none-eabi.cmake .. && cmake --build .", stream_output=True) + run_command_docker('chmod 777 -R ./build/*') else: # Repo uses Make, so execute Make commands. print("[#cccccc](ner build):[/#cccccc] [blue]Makefile-based project detected.[/blue]") if clean: @@ -66,6 +73,9 @@ def build(profile: str = typer.Option(None, "--profile", "-p", callback=unsuppor else: run_command_docker(f"make -j{os.cpu_count()}", stream_output=True) + if not clean: + fix_compile_commands() + # ============================================================================== # Clang command # ============================================================================== @@ -74,7 +84,7 @@ def build(profile: str = typer.Option(None, "--profile", "-p", callback=unsuppor def clang(disable: bool = typer.Option(False, "--disable","-d", help="Disable clang-format"), enable: bool = typer.Option(False, "--enable", "-e", help="Enable clang-format"), run: bool = typer.Option(False, "--run", "-r", help="Run clang-format")): - + if disable: command = ["pre-commit", "uninstall"] elif enable: @@ -84,7 +94,7 @@ def clang(disable: bool = typer.Option(False, "--disable","-d", help="Disable cl else: print("[bold red] Error: No valid option specified") sys.exit(1) - + run_command(command) # ============================================================================== @@ -95,9 +105,11 @@ def clang(disable: bool = typer.Option(False, "--disable","-d", help="Disable cl def debug(ftdi: bool = typer.Option(False, "--ftdi", help="DEPRECATED (On by default) Set this flag if the device uses an FTDI chip"), no_ftdi: bool = typer.Option(False, "--no-ftdi", help="Set this flag if the device uses an CMSIS DAP chip"), custom: bool = typer.Option(False, "--custom", help="Set this flag if your flash.cfg has all definitions (LAUNCHPAD)."), - docker: bool = typer.Option(False, "--docker", callback=unsupported_option_cb, help="(deprecated) Use OpenOCD in the container instead of locally, requires linux")): + docker: bool = typer.Option(False, "--docker", callback=unsupported_option_cb, help="(deprecated) Use OpenOCD in the container instead of locally, requires linux"), + openocd_loc: OpenOCDLocation = typer.Option(OpenOCDLocation.dl, "--openocd-loc", help="Use the OpenOCD binary from where"), + ): - command = ["openocd"] + command = fetch_openocd_command(openocd_loc) current_directory = os.getcwd() if not no_ftdi: @@ -117,10 +129,10 @@ def debug(ftdi: bool = typer.Option(False, "--ftdi", help="DEPRECATED (On by def halt_command = command + ["-f", "flash.cfg", "-f", os.path.join(current_directory, "Drivers", "Embedded-Base", "openocd.cfg"), "-c", "init", "-c", "reset halt"] - + ocd = subprocess.Popen(halt_command) time.sleep(1) - + # for some reason the host docker internal thing is broken on linux despite compose being set correctly, hence this hack # TODO: fix this properly @@ -144,9 +156,11 @@ def debug(ftdi: bool = typer.Option(False, "--ftdi", help="DEPRECATED (On by def @app.command(help="Flash the firmware") def flash(ftdi: bool = typer.Option(False, "--ftdi", help="DEPRECATED (On by default): Set this flag if the device uses an FTDI chip"), - no_ftdi: bool = typer.Option(False, "--no-ftdi", help="Set this flag if the device uses an CMSIS DAP chip"), + no_ftdi: bool = typer.Option(False, "--no-ftdi", help="Set this flag if the device uses an CMSIS DAP chip"), custom: bool = typer.Option(False, "--custom", help="Set this flag if your flash.cfg has all definitions (LAUNCHPAD)."), - docker: bool = typer.Option(False, "--docker", help="Use OpenOCD in the container instead of locally, requires linux")): + docker: bool = typer.Option(False, "--docker", help="Use OpenOCD in the container instead of locally, requires linux"), + openocd_loc: OpenOCDLocation = typer.Option(OpenOCDLocation.dl, "--openocd-loc", help="Use the OpenOCD binary from where"), + ): command = [] @@ -156,8 +170,9 @@ def flash(ftdi: bool = typer.Option(False, "--ftdi", help="DEPRECATED (On by def if docker: command = ["docker", "compose", "run", "--rm", "ner-gcc-arm"] - - command = command + ["openocd"] + command = command + ["openocd"] + else: + command = command + fetch_openocd_command(openocd_loc) if not no_ftdi: current_directory = os.getcwd() @@ -165,7 +180,7 @@ def flash(ftdi: bool = typer.Option(False, "--ftdi", help="DEPRECATED (On by def command = command + ["-f", ftdi_path] elif not custom: command = command + ["-f", "interface/cmsis-dap.cfg"] - + build_directory = os.path.join("build", "*.elf") elf_files = glob.glob(build_directory) @@ -177,7 +192,7 @@ def flash(ftdi: bool = typer.Option(False, "--ftdi", help="DEPRECATED (On by def print(f"[bold blue] Found ELF file: [/bold blue] [blue] {elf_file}") command = command + ["-f", "flash.cfg", "-c", f"program {elf_file} verify reset exit"] - + run_command(command, stream_output=True) # ============================================================================== @@ -206,7 +221,7 @@ def serial2( graph: str = typer.Option(None, "--graph", help="Opens a live graph window of the specified title. (Note: A graph window can be created/configured using the serial_graph() function from serial.c)"), filter: str = typer.Option(None, "--filter", help="Only shows specific messages. Ex. 'ner serial2 --filter EXAMPLE' will only show printfs that contain the substring 'EXAMPLE'. ")): """Custom serial terminal.""" - + serial2_start(ls=ls, device=device, monitor=monitor, graph=graph, filter=filter) # ============================================================================== @@ -238,8 +253,8 @@ def update(): if "bedded" in dir: command = ["pip", "install", "-e", "ner_environment"] - else: - # app folder - go up one level to root + else: + # app folder - go up one level to root if os.path.exists(os.path.join(dir, "Drivers", "Embedded-Base")): os.chdir("..") @@ -252,7 +267,7 @@ def update(): sys.exit(1) run_command(command) - + # ============================================================================== # USBIP command # ============================================================================== @@ -265,9 +280,9 @@ def usbip(connect: bool = typer.Option(False, "--connect", help="Connect to a US if connect: if device is None: print("[bold red] Error --device must be specified when using --connect") - + disconnect_usbip() # Disconnect the current USB device - + if device == "shep" or device == "shepherd": commands = [ ["sudo", "modprobe", "vhci_hcd"], @@ -279,14 +294,14 @@ def usbip(connect: bool = typer.Option(False, "--connect", help="Connect to a US ["sudo", "modprobe", "vhci_hcd"], ["sudo", "usbip", "attach", "-r", "192.168.100.12", "-b", "1-1.3"] ] - + else: print("[bold red] Error: Invalid device name") sys.exit(1) - + run_command(commands[0]) run_command(commands[1]) - + elif disconnect: disconnect_usbip() @@ -295,62 +310,6 @@ def usbip(connect: bool = typer.Option(False, "--connect", help="Connect to a US # ============================================================================== -# ============================================================================== -# init -# ============================================================================== -@lp_app.command(help="Initialize launchpad, run before any commands") -def install(): - """Install PlatformIO package.""" - try: - # Install the platformio package - subprocess.check_call(['pip', 'install', 'platformio']) - - except subprocess.CalledProcessError as e: - print(f"Failed to install PlatformIO: {e}", file=sys.stderr) - sys.exit(1) - -# ============================================================================== -# uninstall -# ============================================================================== -@lp_app.command(help="Remove launchpad") -def uninstall(): - """Uninstall PlatformIO package.""" - try: - # Install the platformio package - subprocess.check_call(['pip', 'uninstall', '-y', 'platformio']) - - platformio_dir = os.path.expanduser('~/.platformio') - if os.path.isdir(platformio_dir): - shutil.rmtree(platformio_dir) - print("PlatformIO directory removed.") - else: - print("PlatformIO directory does not exist.") - - except subprocess.CalledProcessError as e: - print(f"Failed to uninstall PlatformIO: {e}", file=sys.stderr) - sys.exit(1) - -# ============================================================================== -# build -# ============================================================================== -@lp_app.command(help="Build your launchpad project") -def build(): - subprocess.run(['platformio', 'run']) - -# ============================================================================== -# flash -# ============================================================================== -@lp_app.command(help="Flash your launchpad project to a board") -def flash(): - subprocess.run(['platformio', 'run', '--target', 'upload']) - -# ============================================================================== -# serial -# ============================================================================== -@lp_app.command(help="View serial output from the device") -def serial(): - subprocess.run(['platformio', 'device', 'monitor']) - # ============================================================================== # ---- End Launchpad Section @@ -361,19 +320,19 @@ def serial(): # Helper functions - not direct commands # ============================================================================== -def run_command(command, stream_output=False, exit_on_fail=True): +def run_command(command, stream_output=False, exit_on_fail=False): """Run a shell command. Optionally stream the output in real-time.""" - + if stream_output: - + process = subprocess.Popen(command, text=True) - + returncode = process.wait() if returncode != 0: print(f"Error: Command exited with code {returncode}", file=sys.stderr) if exit_on_fail: sys.exit(returncode) - + else: try: result = subprocess.run(command, check=True, capture_output=True, text=True) @@ -385,11 +344,62 @@ def run_command(command, stream_output=False, exit_on_fail=True): if exit_on_fail: sys.exit(e.returncode) +def fix_compile_commands(): + ''' + Attempt to fix compile_commands.json so LSPs can read the project + If this command fails it should be ignored as compile_commands.json is only for user experience + ''' + # download the toolchain if needed, we always need the toolchain name + os_type = platform.system() + toolchain_name = "" + if os_type == "Linux": + toolchain_name = "arm-gnu-toolchain-14.3.rel1-x86_64-arm-none-eabi" + else: + toolchain_name = "arm-gnu-toolchain-14.3.rel1-darwin-arm64-arm-none-eabi" + + toolchain_name_ext = toolchain_name + ".tar.xz" + + if os.path.exists(f'{os.environ.get('VIRTUAL_ENV')}/{toolchain_name}'): + pass + else: + # download the toolchain if it doesnt exist + print('Downloading ', toolchain_name) + subprocess.run([ + "wget", + f"https://developer.arm.com/-/media/Files/downloads/gnu/14.3.rel1/binrel/{toolchain_name_ext}", + ]) + subprocess.run(["tar", "-xvf", toolchain_name_ext, "-C", os.environ.get('VIRTUAL_ENV')]) + os.remove(toolchain_name_ext) + + jsf = None + with open(f'{os.getcwd()}/build/compile_commands.json', "r") as f: + jsf = json.load(f) + + + if jsf is not None: + docker_toolchain_name = re.search(r"arm-gnu-toolchain.*?\/", jsf[0]['command']).group(0)[:-1] + print('Found docker toolchain ', docker_toolchain_name) + for item in jsf: + item['directory'] = item['directory'].replace('/home/app', os.getcwd()) + item['command'] = item['command'].replace('/home/app', os.getcwd()) + item['file'] = item['file'].replace('/home/app', os.getcwd()) + item['output'] = item['output'].replace('/home/app', os.getcwd()) + + item['directory'] = item['directory'].replace(f'/home/dev/{docker_toolchain_name}', f'{os.environ.get('VIRTUAL_ENV')}/{toolchain_name}') + item['command'] = item['command'].replace(f'/home/dev/{docker_toolchain_name}', f'{os.environ.get('VIRTUAL_ENV')}/{toolchain_name}') + item['file'] = item['file'].replace(f'/home/dev/{docker_toolchain_name}', f'{os.environ.get('VIRTUAL_ENV')}/{toolchain_name}') + item['output'] = item['output'].replace(f'/home/dev/{docker_toolchain_name}', f'{os.environ.get('VIRTUAL_ENV')}/{toolchain_name}') + + with open(f'{os.getcwd()}/build/compile_commands.json', "w") as f: + json.dump(jsf, f) + + print('Successfully patched compile_commands.json') + def run_command_docker(command, stream_output=False): """Run a command in the Docker container.""" docker_command = ["docker", "compose", "run", "--rm", "ner-gcc-arm", "sh", "-c", command] print(f"[bold blue](ner-gcc-arm): Running command '{command}' in Docker container.") - run_command(docker_command, stream_output=stream_output, exit_on_fail=True) + run_command(docker_command, stream_output=stream_output) def disconnect_usbip(): """Disconnect the current USB device.""" @@ -409,12 +419,47 @@ def contains_subdir(base_path, search_str): return True return False +def fetch_openocd_command(openocd_loc: OpenOCDLocation) -> list[str]: + ''' + Creates an OpenOCD command + ''' + command: list[str] = [] + os_type = platform.system() + + if openocd_loc.value == 'sys': + command.append('openocd') + elif openocd_loc.value == 'cube': + if os_type == "Darwin": + # for cubeIDE + command.append(os.path.normpath(glob.glob("/Applications/STM32CubeIDE.app/Contents/Eclipse/plugins/com.st.stm32cube.ide.mcu.externaltools.openocd.**/tools/bin/openocd")[0])) + elif os_type == "Linux": + # for cubeIDE + command.append(os.path.normpath(glob.glob(os.path.expanduser("~/st/stm32cubeide_1.19.0/plugins/com.st.stm32cube.ide.mcu.externaltools.openocd.**/tools/bin/openocd"))[0])) + else: + if os.path.exists(f"{os.environ.get('VIRTUAL_ENV')}/openocd_ner"): + pass + else: + # download the binary if it doesnt exist + # TODO make builds on ner repo, for now this is good enough + r: requests.Response + if os_type == "Linux": + r = requests.get('https://github.com/bjackson312006/ner-openocd/releases/download/tools-v1.0.4/openocd-linux-x64', allow_redirects=True) + else: + r = requests.get('https://github.com/bjackson312006/ner-openocd/releases/download/tools-v1.0.4/openocd-macos-arm64', allow_redirects=True) + print(r) + open(f"{os.environ.get('VIRTUAL_ENV')}/openocd_ner", 'wb').write(r.content) + os.chmod(f"{os.environ.get('VIRTUAL_ENV')}/openocd_ner", 0o777) + + command.append(f"{os.environ.get('VIRTUAL_ENV')}/openocd_ner") + + command.append("-s") + command.append("./Drivers/Embedded-Base/dev/OpenOCD/tcl") + + return command + # ============================================================================== # Entry # ============================================================================== if __name__ == "__main__": app(prog_name="ner") - - - diff --git a/ner_environment/ner_setup.py b/ner_environment/ner_setup.py index 39d4d17b..750e8e46 100644 --- a/ner_environment/ner_setup.py +++ b/ner_environment/ner_setup.py @@ -86,21 +86,23 @@ def install_launchpad(venv_python): sys.exit(1) def install_openocd(): - os_type = platform.system() + #os_type = platform.system() try: - if os_type == "Darwin": - run_command(["brew", "install", "openocd"]) - - elif os_type == "Linux": - distro_name = distro.id() - if distro_name in ["ubuntu", "debian"]: - run_command(["sudo", "apt-get", "install", "-y", "openocd"]) - elif distro_name == "fedora": - run_command(["sudo", "dnf", "install", "-y", "openocd"]) - elif distro_name == "arch": - run_command(["sudo", "pacman", "-S", "--noconfirm", "openocd"]) - else: - print("We haven't added OpenOCD install support for your distro, but if you're actually on linux, you can install it manually!") + print("OpenOCD installation via package manager is no longer supported!") + print("Instead, install STM32CubeIDE and OpenOCD will be automatically found: \nhttps://www.st.com/en/development-tools/stm32cubeide.html") + # if os_type == "Darwin": + # run_command(["brew", "install", "openocd"]) + + # elif os_type == "Linux": + # distro_name = distro.id() + # if distro_name in ["ubuntu", "debian"]: + # run_command(["sudo", "apt-get", "install", "-y", "openocd"]) + # elif distro_name == "fedora": + # run_command(["sudo", "dnf", "install", "-y", "openocd"]) + # elif distro_name == "arch": + # run_command(["sudo", "pacman", "-S", "--noconfirm", "openocd"]) + # else: + # print("We haven't added OpenOCD install support for your distro, but if you're actually on linux, you can install it manually!") except Exception as e: print(f"Failed to install OpenOCD: {e}", file=sys.stderr) diff --git a/ner_environment/requirements.txt b/ner_environment/requirements.txt index 58e9a7d4..5d7367ae 100644 --- a/ner_environment/requirements.txt +++ b/ner_environment/requirements.txt @@ -5,3 +5,5 @@ pyserial typer matplotlib rich +requests +jinja2