From 74b51fc0444bc82d108f05b7ef7ff60f91d75eb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Mon, 17 Sep 2018 14:54:15 +0100 Subject: [PATCH] gem5: update to 7bfb7f3a43f382eb49853f47b140bfd6caad0fb8 The update is required to include 3c3ca64b5f0dd9eef7b1ce1c65cc6e8e9147dd38 otherwise baremetal does not on VExpress. baremetal: create a baremetal setup with crosstool-ng buildroot: improve directory location: move out/dl inside out/buildroot/download, and add a new out/buildroot/build level tagline: generalize, deliver more value than howto, since now howtos are starting to multiply rename all top scripts to separate words with hyphen more consistently, e.g. run-gdb instead of rungdb getvar: list all variables gem5: make m5out section to focus all releated information at Prevent m5term Text file busy when rebuilding gem5 while it is running. --- .gitmodules | 3 + README.adoc | 833 +++++++++++++++--- baremetal/arch/arm/gem5_assert.S | 19 + .../arch/arm/no_bootloader/semihost_exit.S | 5 + baremetal/arch/arm/semihost_exit.S | 5 + baremetal/exit.c | 7 + baremetal/lib/aarch64.S | 11 + baremetal/lib/arm.S | 5 + baremetal/lib/common.c | 65 ++ baremetal/link.ld | 19 + baremetal/prompt.c | 22 + bench-all | 2 +- bench-boot | 30 +- build-all | 8 +- build-baremetal | 151 ++++ build-buildroot | 24 +- build-crosstool-ng | 83 ++ build-gem5 | 8 +- common.py | 157 +++- configure | 167 ++-- crosstool_ng_config/aarch64 | 9 + crosstool_ng_config/arm | 12 + gem5-bench-dhrystone | 4 +- getvar | 21 +- packages/kernel_modules/user/common.h | 6 +- patches/manual/gem5-aarch64-baremetal.patch | 12 + patches/manual/gem5-wait-gdb.patch | 11 + qemumonitor => qemu-monitor | 0 qemurr => qemu-rr | 0 ...t_build_script => rootfs-post-build-script | 0 run | 93 +- rungdb => run-gdb | 37 +- rungdb-user => run-gdb-user | 2 +- rungdbserver => run-gdbserver | 0 run-toolchain | 2 - submodules/crosstool-ng | 1 + submodules/gem5 | 2 +- 37 files changed, 1545 insertions(+), 291 deletions(-) create mode 100644 baremetal/arch/arm/gem5_assert.S create mode 100644 baremetal/arch/arm/no_bootloader/semihost_exit.S create mode 100644 baremetal/arch/arm/semihost_exit.S create mode 100644 baremetal/exit.c create mode 100644 baremetal/lib/aarch64.S create mode 100644 baremetal/lib/arm.S create mode 100644 baremetal/lib/common.c create mode 100644 baremetal/link.ld create mode 100644 baremetal/prompt.c create mode 100755 build-baremetal create mode 100755 build-crosstool-ng create mode 100644 crosstool_ng_config/aarch64 create mode 100644 crosstool_ng_config/arm create mode 100644 patches/manual/gem5-aarch64-baremetal.patch create mode 100644 patches/manual/gem5-wait-gdb.patch rename qemumonitor => qemu-monitor (100%) rename qemurr => qemu-rr (100%) rename rootfs_post_build_script => rootfs-post-build-script (100%) rename rungdb => run-gdb (78%) rename rungdb-user => run-gdb-user (98%) rename rungdbserver => run-gdbserver (100%) create mode 160000 submodules/crosstool-ng diff --git a/.gitmodules b/.gitmodules index 492e2d7f..b020cd46 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,6 +2,9 @@ path = submodules/buildroot url = https://github.com/cirosantilli/buildroot ignore = dirty +[submodule "submodules/crosstool-ng"] + path = submodules/crosstool-ng + url = https://github.com/crosstool-ng/crosstool-ng [submodule "submodules/gem5"] path = submodules/gem5 url = https://gem5.googlesource.com/public/gem5 diff --git a/README.adoc b/README.adoc index e4dd94b0..4f1814d7 100644 --- a/README.adoc +++ b/README.adoc @@ -9,7 +9,7 @@ :toclevels: 6 :toc-title: -Run one command, get a QEMU or gem5 Buildroot BusyBox virtual machine built from source with several minimal Linux kernel 4.18 module development example tutorials with <> and <> step debugging and minimal educational hardware device models. "Tested" in Ubuntu 18.04 host, x86 and ARM guests. +The perfect setup to study and modify the <>, kernel modules, <> and <>. Highly automated. Thoroughly documented. <> and <> just work. Powered by <> and <>. "Tested" in Ubuntu 18.04 host, x86 and ARM guests with kernel v4.18. TL;DR: <> @@ -65,14 +65,14 @@ The trade-offs are basically a balance between: This is the best setup if you are on Ubuntu. We tend to test this repo the most on the latest Ubuntu, and on the latest Ubuntu LTS. -Everything will likely also work on other Linux distros if you install the analogous required packages for your distro from link:configure[], but this is not currently well tested. Compatibility patches are welcome. You can also try <> if you are on other Linux distros. Native Windows is unlikely feasible because Buildroot is a huge set of GNU Make scripts + host tools, just use an Ubuntu VM in that case. +Everything will likely also work on other Linux distros if you do what install the analogous required packages for your distro from link:configure[], but this is not currently well tested. Compatibility patches are welcome. You can also try <> if you are on other Linux distros. Native Windows is unlikely feasible because Buildroot is a huge set of GNU Make scripts + host tools, just use an Ubuntu VM in that case. Reserve 12Gb of disk and run: .... git clone https://github.com/cirosantilli/linux-kernel-module-cheat cd linux-kernel-module-cheat -./configure && \ +./configure --qemu && \ ./build-qemu && ./build-buildroot && \ ./run && \ @@ -392,7 +392,13 @@ Internal error: Oops - undefined instruction Workaround by checking out kernel 4.17 as explained at <>. Bisected down to kernel 1d4238c56f9816ce0f9c8dbe42d7f2ad81cb6613, gem5 is not implementing the `CSDB` instruction. -More information at: <> +More gem5 information is present at: <> + +Good next steps are: + +* <> +* <> +* <> [[docker]] === Docker host setup @@ -461,7 +467,7 @@ or even better, by starting a <> session inside the container. We install You can start a second shell and run a command in it at the same time with: .... -./rundocker sh ./rungdb start_kernel +./rundocker sh ./run-gdb start_kernel .... Docker stops if and only if you quit the initial shell, you can quit this one without consequences. @@ -663,6 +669,159 @@ rmmod hello.ko dmesg .... +=== Baremetal setup + +==== About the baremetal setup + +This setup does not use the Linux kernel nor Buildroot at all: it just runs your very own minimal OS. + +`x86_64` is not currently supported, only `arm` and `aarch64`: I had made some x86 bare metal examples at: https://github.com/cirosantilli/x86-bare-metal-examples but I'm lazy to port them here now. Pull requests are welcome. + +The main reason this setup is included in this project, despite the word "Linux" being on the project name, is that a lot of the emulator boilerplate can be reused for both use cases. + +This setup allows you to make a tiny OS and that runs just a few instructions, use it to fully control the CPU to better understand the simulators for example, or develop your own OS if you are into that. + +You can also use C and a subset of the C standard library because we enable link:https://en.wikipedia.org/wiki/Newlib[Newlib] by default. + +Our C bare-metal compiler is built with link:https://github.com/crosstool-ng/crosstool-ng[crosstool-NG]. If you have already built <> previously, you will end up with two GCCs installed. Unfortunately I don't see a solution for this, since we need separate toolchains for Newlib on baremetal and glibc on Linux: https://stackoverflow.com/questions/38956680/difference-between-arm-none-eabi-and-arm-linux-gnueabi/38989869#38989869 + +==== Baremetal setup getting started + +QEMU: + +.... +./configure --baremetal --qemu && \ +./build-qemu --arch arm && \ +./build-crosstool-ng --arch arm && \ +./build-baremetal --arch arm && \ +./run --arch arm --baremetal prompt && \ +:; +.... + +You are now left inside QEMU running the tiny baremetal system link:baremetal/prompt.c[], which uses the UART to: + +* print characters to the terminal +* read characters from your keyboard + +A session looks like this after typing `abc`: + +.... +enter a character +got: a +new alloc of 1 bytes at address 0x0x4000a2c8 +enter a character +got: b +new alloc of 2 bytes at address 0x0x4000a2c8 +enter a character +got: c +new alloc of 4 bytes at address 0x0x4000a2c8 +.... + +`./build-baremetal` is the command that actually builds the baremetal system for us. It uses crosstool-NG, so that command must be preceded by `./build-crosstool-ng`. + +Every `.c` file inside link:baremetal/[] and `.S` file inside `baremetal/arch//` generates a separate baremetal image. You can run a different image with commands such as: + +.... +./run --arch arm --baremetal exit +./run --arch arm --baremetal arch/arm/semihost_exit +.... + +which will run respectively: + +* link:baremetal/exit.c[] +* link:baremetal/arch/arm/m5exit.S[] + +which just make the emulator quit via <>. + +Alternatively, for the sake of tab completion, we also accept full paths inside `baremetal`: + +.... +./run --arch arm --baremetal baremetal/exit.c +./run --arch arm --baremetal "$(pwd)/baremetal/exit.c" +./run --arch arm --baremetal baremetal/arch/arm/semihost_exit.c +./run --arch arm --baremetal "$(pwd)/baremetal/arch/arm/semihost_exit.c" +.... + +To use gem5 instead of QEMU do: + +.... +./configure --baremetal --gem5 && \ +./build-gem5 --arch arm && \ +./build-crosstool-ng --arch arm && \ +./build-baremetal --arch arm --gem5 && \ +./run --arch arm --baremetal prompt --gem5 && \ +:; +.... + +and then <> open a shell with: + +.... +./gem5-shell +.... + +TODO: + +* the carriage returns are a bit different than in QEMU, see: <>. +* semihosting is disabled, and all examples that rely on it fail, including how to enable it? See <> + +Note that `./build-baremetal` requires the `--gem5` option, and generates separate executable images for both, as can be seen from: + +.... +echo "$(./getvar --arch arm --baremetal prompt image)" +echo "$(./getvar --arch arm --baremetal prompt --gem5 image)" +.... + +This is unlike the Linux kernel that has a single image for both QEMU and gem5: + +.... +echo "$(./getvar --arch arm image)" +echo "$(./getvar --arch arm --gem5 image)" +.... + +The reason for that is that on baremetal we don't parse the <> from memory like the Linux kernel does, which tells the kernel for example the UART address, and many other system parameters. + +`gem5` also supports the `RealViewPBX` machine, which represents an older hardware compared to the default `VExpress_GEM5_V1`: + +.... +./build-baremetal --arch arm --gem5 --machine RealViewPBX +./run --arch arm --baremetal prompt --gem5 --machine RealViewPBX +.... + +This generates yet new separate images with new magic constants: + +.... +echo "$(./getvar --arch arm --baremetal prompt --gem5 --machine VExpress_GEM5_V1 image)" +echo "$(./getvar --arch arm --baremetal prompt --gem5 --machine RealViewPBX image)" +.... + +But just stick to newer and better `VExpress_GEM5_V1` unless you have a good reason to use `RealViewPBX`. + +gem5 aarch64 baremetal currently requires you to apply a small patch before running: + +.... +./build-baremetal --arch aarch64 --gem5 +patch -d "$(./getvar gem5_src_dir)" -p 1 < patches/manual/gem5-aarch64-baremetal.patch +./run --arch arm --baremetal prompt --gem5 +.... + +Source: link:patches/manual/gem5-aarch64-baremetal.patch[] + +This patch is required only for baremetal because the Linux bootloader enters 32 bit mode and switches to 64 for us, but the in the baremetal system we use our own bootloader. + +Therefore we must tell gem5 to start the system in aarch64 mode for us with the options: + +.... +highest_el_is_64 = True +auto_reset_addr_64 = True +.... + +That patch breaks `--arch arm`, so don't forget to remove it if you want to go back to it. + +For more information on baremetal, see the section: <>. The following subjects are particularly important: + +* <> +* <> + [[gdb]] == GDB step debug @@ -677,13 +836,13 @@ dmesg Say you want to break at `start_kernel`. So on another shell: .... -./rungdb start_kernel +./run-gdb start_kernel .... or at a given line: .... -./rungdb init/main.c:1088 +./run-gdb init/main.c:1088 .... Now QEMU will stop there, and you can use the normal GDB commands: @@ -729,7 +888,7 @@ which counts to infinity to stdout. Source: link:rootfs_overlay/count.sh[]. Then in another shell, run: .... -./rungdb +./run-gdb .... and then hit: @@ -813,7 +972,7 @@ From inside tmux, you can do that with `Ctrl-B C` or `Ctrl-B %`. To see the debugger by default instead of the terminal, run: .... -./tmu ./rungdb && ./run --debug-guest --gem5 +./tmu ./run-gdb && ./run --debug-guest --gem5 .... === GDB step debug kernel module @@ -845,7 +1004,7 @@ This prints a message to dmesg every second. Shell 2: .... -./rungdb +./run-gdb .... In GDB, hit `Ctrl-C`, and note how it says: @@ -889,7 +1048,7 @@ Error occurred in Python command: Cannot access memory at address 0xbf0000cc Can't reproduce on `x86_64` and `aarch64` are fine. -It is kind of random: if you just `insmod` manually and then immediately `./rungdb --arch arm`, then it usually works. +It is kind of random: if you just `insmod` manually and then immediately `./run-gdb --arch arm`, then it usually works. But this fails most of the time: shell 1: @@ -900,7 +1059,7 @@ But this fails most of the time: shell 1: shell 2: .... -./rungdb --arch arm +./run-gdb --arch arm .... then hit `Ctrl-C` on shell 2, and voila. @@ -951,7 +1110,7 @@ As of 4.16, the call happens in `do_one_initcall`, so we can do in shell 1: shell 2 after boot finishes (because there are other calls to `do_init_module` at boot, presumably for the built-in modules): .... -./rungdb do_one_initcall +./run-gdb do_one_initcall .... then step until the line: @@ -965,7 +1124,7 @@ which does the actual call, and then step into it. For the next time, you can also put a breakpoint there directly: .... -./rungdb init/main.c:833 +./run-gdb init/main.c:833 .... How we found this out: first we got <> working, and then we did a `bt`. AKA cheating :-) @@ -1021,7 +1180,7 @@ Now we can just do a fresh boot on shell 1: and on shell 2: .... -./rungdb '*0xffffffffc0000240' +./run-gdb '*0xffffffffc0000240' .... GDB then breaks, and `lx-symbols` works. @@ -1089,7 +1248,7 @@ directly gives an <> as I'd expect. Useless, but a good way to show how hardcore you are. Disable `lx-symbols` with: .... -./rungdb --no-lxsymbols +./run-gdb --no-lxsymbols .... From inside guest: @@ -1136,14 +1295,14 @@ And then search for a line of type: Break at the very first instruction executed by QEMU: .... -./rungdb --no-continue +./run-gdb --no-continue .... TODO why can't we break at early startup stuff such as: .... -./rungdb extract_kernel -./rungdb main +./run-gdb extract_kernel +./run-gdb main .... Maybe it is because they are being copied around at specific locations instead of being run directly from inside the main image, which is where the debug information points to? @@ -1170,12 +1329,12 @@ and break there: .... ./run --arch arm --debug-guest -./rungdb --arch arm '*0x1000' +./run-gdb --arch arm '*0x1000' .... but TODO: it does not show the source assembly under `arch/arm`: https://stackoverflow.com/questions/11423784/qemu-arm-linux-kernel-boot-debug-no-source-code -I also tried to hack `rungdb` with: +I also tried to hack `run-gdb` with: .... @@ -81,7 +81,7 @@ else @@ -1226,7 +1385,7 @@ since GDB does not know that libc is loaded. * Shell 2: + .... -./rungdb-user kernel_modules-1.0/user/sleep_forever.out main +./run-gdb-user kernel_modules-1.0/user/sleep_forever.out main .... TODO not working as of f8c0502bb2680f2dbe7c1f3d7958f60265347005, does not break. Bisect on recent QEMU and kernel. Debug by creating an executable that prints the address of `main`. @@ -1243,7 +1402,7 @@ BusyBox custom init process: * Shell 2: + .... -./rungdb-user busybox-1.26.2/busybox ls_main +./run-gdb-user busybox-1.26.2/busybox ls_main .... This follows BusyBox' convention of calling the main for each executable as `_main` since the `busybox` executable has many "mains". @@ -1258,7 +1417,7 @@ BusyBox default init process: * Shell 2: + .... -./rungdb-user busybox-1.26.2/busybox init_main +./run-gdb-user busybox-1.26.2/busybox init_main .... This cannot be debugged in another way without modifying the source, or `/sbin/init` exits early with: @@ -1279,7 +1438,7 @@ Non-init process: * Shell 2: + .... -./rungdb-user kernel_modules-1.0/user/myinsmod.out main +./run-gdb-user kernel_modules-1.0/user/myinsmod.out main .... * Shell 1 after the boot finishes: + @@ -1301,7 +1460,7 @@ TODO: on QEMU bfba11afddae2f7b2c1335b4e23133e9cd3c9126, it works on `x86_64` and * Shell 2: wait for boot to finish, and run: + .... -./rungdb-user --arch arm kernel_modules-1.0/user/hello.out main +./run-gdb-user --arch arm kernel_modules-1.0/user/hello.out main .... * Shell 1: + @@ -1309,7 +1468,7 @@ TODO: on QEMU bfba11afddae2f7b2c1335b4e23133e9cd3c9126, it works on `x86_64` and /hello.out .... -The problem is that the `b main` that we do inside `./rungdb-user` says: +The problem is that the `b main` that we do inside `./run-gdb-user` says: .... Cannot access memory at address 0x10604 @@ -1465,7 +1624,7 @@ We will run our `/sched_getaffinity.out` infinitely many time, on core 0 and cor on another shell: .... -./rungdb-user kernel_modules-1.0/user/sched_getaffinity.out main +./run-gdb-user kernel_modules-1.0/user/sched_getaffinity.out main .... Then, inside GDB: @@ -1496,7 +1655,7 @@ TODO we then tried: and: .... -./rungdb-user kernel_modules-1.0/user/sched_getaffinity_threads.out +./run-gdb-user kernel_modules-1.0/user/sched_getaffinity_threads.out .... to switch between two simultaneous live threads with different affinities, it just didn't break on our threads: @@ -1618,7 +1777,7 @@ Usage: .... ./run --kgdb -./rungdb --kgdb +./run-gdb --kgdb .... In GDB: @@ -1646,7 +1805,7 @@ c And now you can count from GDB! -If you do: `b __x64_sys_write` immediately after `./rungdb --kgdb`, it fails with `KGDB: BP remove failed:
`. I think this is because it would break too early on the boot sequence, and KGDB is not yet ready. +If you do: `b __x64_sys_write` immediately after `./run-gdb --kgdb`, it fails with `KGDB: BP remove failed:
`. I think this is because it would break too early on the boot sequence, and KGDB is not yet ready. See also: @@ -1758,7 +1917,7 @@ Source: link:rootfs_overlay/gdbserver.sh[]. Host: .... -./rungdbserver kernel_modules-1.0/user/myinsmod.out +./run-gdbserver kernel_modules-1.0/user/myinsmod.out .... You can find the executable with: @@ -1793,7 +1952,7 @@ An implementation overview can be found at: https://reverseengineering.stackexch As usual, different archs work with: .... -./rungdbserver --arch arm kernel_modules-1.0/user/myinsmod.out +./run-gdbserver --arch arm kernel_modules-1.0/user/myinsmod.out .... === gdbserver BusyBox @@ -1807,7 +1966,7 @@ BusyBox executables are all symlinks, so if you do on guest: on host you need: .... -./rungdbserver busybox-1.26.2/busybox +./run-gdbserver busybox-1.26.2/busybox .... === gdbserver shared libraries @@ -1857,7 +2016,7 @@ Debug: .... ./run --arch arm --debug-guest # On another terminal. -./rungdb --arch arm +./run-gdb --arch arm .... We also have one letter shorthand names for the architectures and `--arch` option: @@ -1933,7 +2092,7 @@ TODO Can you run arm executables in the aarch64 guest? https://stackoverflow.com I've tried: .... -./run-toolchain --arch aarch64 gcc -- -static ~/test/hello_world.c -o data/9p/a.out +./run-toolchain --arch aarch64 gcc -- -static ~/test/hello_world.c -o "$(./getvar p9_dir)/a.out" ./run --arch aarch64 --eval-busybox '/mnt/9p/a.out' .... @@ -2756,7 +2915,7 @@ Text mode has the following limitations over graphics mode: `x86_64` has a VGA device enabled by default, as can be seen as: .... -./qemumonitor info qtree +./qemu-monitor info qtree .... and the Linux kernel picks it up through the link:https://en.wikipedia.org/wiki/Linux_framebuffer[fbdev] graphics system as can be seen from: @@ -3962,7 +4121,7 @@ says that the function `myinit` is in the module `panic`. To find the line that panicked, do: .... -./rungdb +./run-gdb .... and then: @@ -4192,7 +4351,7 @@ RIP: 0010:myinit+0x18/0x30 [oops] and then on GDB: .... -./rungdb +./run-gdb .... run @@ -4999,7 +5158,7 @@ so only `1` has `myirqhandler0` attached but not `0`. The <> also has some interrupt statistics for x86_64: .... -./qemumonitor info irq +./qemu-monitor info irq .... TODO: properly understand how each IRQ maps to what number. @@ -5118,7 +5277,7 @@ virt_to_phys(&static_var) = 0x40002308 We can confirm that the `kmalloc_ptr` translation worked with: .... -./qemumonitor 'xp 0xe169ae8' +./qemu-monitor 'xp 0xe169ae8' .... which reads four bytes from a given physical address, and gives the expected: @@ -5130,7 +5289,7 @@ which reads four bytes from a given physical address, and gives the expected: TODO it only works for kmalloc however, for the static variable: .... -./qemumonitor 'xp 0x40002308' +./qemu-monitor 'xp 0x40002308' .... it gave a wrong value of `00000000`. @@ -5207,7 +5366,7 @@ First launch `virt_to_phys_user.out` as described at <> under cow sections, but hacking them to true did not work: .... diff --git a/configs/common/FSConfig.py b/configs/common/FSConfig.py @@ -6532,7 +6694,7 @@ To test it out, login into the VM with and run: On another shell, take a snapshot: .... -./qemumonitor savevm my_snap_id +./qemu-monitor savevm my_snap_id .... The counting continues. @@ -6540,7 +6702,7 @@ The counting continues. Restore the snapshot: .... -./qemumonitor loadvm my_snap_id +./qemu-monitor loadvm my_snap_id .... and the counting goes back to where we saved. This shows that CPU and memory states were reverted. @@ -6560,7 +6722,7 @@ echo 0 >f Monitor: .... -./qemumonitor savevm my_snap_id +./qemu-monitor savevm my_snap_id .... Guest: @@ -6572,7 +6734,7 @@ echo 1 >f Monitor: .... -./qemumonitor loadvm my_snap_id +./qemu-monitor loadvm my_snap_id .... Guest: @@ -6739,7 +6901,7 @@ which we identify as being `edu` and `pci_min` respectively by the magic numbers Alternatively, we can also do use the QEMU monitor: .... -./qemumonitor info qtree +./qemu-monitor info qtree .... which gives: @@ -7061,7 +7223,7 @@ TODO performance compared to NFS. As usual, we have already set everything up for you. On host: .... -cd data/9p +cd "$(./getvar p9_dir)" uname -a > host .... @@ -7236,7 +7398,7 @@ A simpler and possibly less overhead alternative to <<9P>> would be to generate Then you can `umount` and re-mount on guest without reboot. -We don't support this yet, but it should not be too hard to hack it up, maybe by hooking into link:rootfs_post_build_script[]. +We don't support this yet, but it should not be too hard to hack it up, maybe by hooking into link:rootfs-post-build-script[]. This was not possible from gem5 `fs.py` as of 60600f09c25255b3c8f72da7fb49100e2682093a: https://stackoverflow.com/questions/50862906/how-to-attach-multiple-disk-images-in-a-simulation-with-gem5-fs-py/51037661#51037661 @@ -7258,7 +7420,7 @@ The reason this is cool, is that `ls` is not statically compiled, but since we h In other words, much cooler than: .... -./run-toolchain --arch arm gcc -- -static ./packages/kernel_modules/user/hello.c +./run-toolchain --arch arm gcc -- -static "$(./getvar kernel_modules_src_dir)/user/hello.c" qemu-arm a.out .... @@ -7309,9 +7471,9 @@ First we try some `-static` sanity checks. Works and prints `hello`: .... -./run-toolchain --arch x86_64 gcc -- -static -o x86_64.out ./packages/kernel_modules/user/hello.c -./run-toolchain --arch arm gcc -- -static -o arm.out ./packages/kernel_modules/user/hello.c -./run-toolchain --arch aarch64 gcc -- -static -o aarch64.out ./packages/kernel_modules/user/hello.c +./run-toolchain --arch x86_64 gcc -- -static -o x86_64.out "$(./getvar kernel_modules_src_dir)/user/hello.c" +./run-toolchain --arch arm gcc -- -static -o arm.out "$(./getvar kernel_modules_src_dir)/user/hello.c" +./run-toolchain --arch aarch64 gcc -- -static -o aarch64.out "$(./getvar kernel_modules_src_dir)/user/hello.c" "$(./getvar --arch x86_64 --gem5 exec)" "$(./getvar gem5_se_file)" -c ./x86_64.out "$(./getvar --arch arm --gem5 exec)" "$(./getvar gem5_se_file)" -c ./arm.out "$(./getvar --arch aarch64 --gem5 exec)" "$(./getvar gem5_se_file)" -c ./aarch64.out @@ -7404,20 +7566,20 @@ The QEMU monitor is a terminal that allows you to send text commands to the QEMU On another terminal, run: .... -./qemumonitor +./qemu-monitor .... or send one command such as `info qtree` and quit the monitor: .... -./qemumonitor info qtree +./qemu-monitor info qtree .... -Source: link:qemumonitor[] +Source: link:qemu-monitor[] -`qemumonitor` uses the `-monitor` QEMU command line option, which makes the monitor listen from a socket. +`qemu-monitor` uses the `-monitor` QEMU command line option, which makes the monitor listen from a socket. -`qemumonitor` does not support input from an stdin pipe currently, see comments on the source for rationale. +`qemu-monitor` does not support input from an stdin pipe currently, see comments on the source for rationale. Alternatively, from text mode: @@ -7442,7 +7604,7 @@ Ctrl-Alt ? where `?` is a digit `1`, or `2`, or, `3`, etc. depending on what else is available on the GUI: serial, parallel and frame buffer. -In general, `./qemumonitor` is the best option, as it: +In general, `./qemu-monitor` is the best option, as it: * works on both modes * allows to use the host Bash history to re-run one off commands @@ -7664,7 +7826,7 @@ This awesome feature allows you to examine a single run as many times as you wou A convenient shortcut to do both at once to test the feature is: .... -./qemurr --eval-busybox '/rand_check.out;/poweroff.out;' +./qemu-rr --eval-busybox '/rand_check.out;/poweroff.out;' .... By comparing the terminal output of both runs, we can see that they are the exact same, including things which normally differ across runs: @@ -7691,7 +7853,7 @@ EXT4-fs (sda): re-mounted. Opts: block_validity,barrier,user_xattr TODO replay with network gets stuck: .... -./qemurr --eval-busybox '/sbin/ifup -a;wget -S google.com;/poweroff.out;' +./qemu-rr --eval-busybox '/sbin/ifup -a;wget -S google.com;/poweroff.out;' .... after the message: @@ -7710,7 +7872,7 @@ Then, when I tried with <> and no disk: .... ./build-buildroot --arch aarch64 -i -./qemurr --arch aarch64 --eval-busybox '/rand_check.out;/poweroff.out;' -i +./qemu-rr --arch aarch64 --eval-busybox '/rand_check.out;/poweroff.out;' -i .... QEMU crashes with: @@ -7737,7 +7899,7 @@ QEMU replays support checkpointing, and this allows for a simplistic "reverse de On another shell: .... -./rungdb start_kernel +./run-gdb start_kernel .... In GDB: @@ -7958,19 +8120,7 @@ Now you can play a fun little game with your friends: * make a program that solves the computation problem, and outputs output to stdout * write the code that runs the correct computation in the smallest number of cycles possible -To find out why your program is slow, a good first step is to have a look at the statistics for the run: - -.... -cat "$(./getvar --arch aarch64 m5out_dir)/stats.txt" -.... - -Whenever we run `m5 dumpstats` or `m5 exit`, a section with the following format is added to that file: - -.... ----------- Begin Simulation Statistics ---------- -[the stats] ----------- End Simulation Statistics ---------- -.... +To find out why your program is slow, a good first step is to have a look at <> file. ==== Skip extra benchmark instructions @@ -8425,7 +8575,7 @@ If you want to remove PARSEC later, Buildroot doesn't provide an automated packa .... rm -rf \ - "$(./getvar dl_dir)"/parsec-* \ + "$(./getvar buildroot_download_dir)"/parsec-* \ "$(./getvar buildroot_build_dir)"/build/parsec-* \ "$(./getvar buildroot_build_dir)"/build/packages-file-list.txt \ "$(./getvar buildroot_build_dir)"/images/rootfs.* \ @@ -8505,7 +8655,7 @@ Analogous <>, on the first shell: On the second shell: .... -./rungdb --arch arm --gem5 +./run-gdb --arch arm --gem5 .... On a third shell: @@ -8518,7 +8668,42 @@ When you want to break, just do a `Ctrl-C` on GDB shell, and then `continue`. And we now see the boot messages, and then get a shell. Now try the `/count.sh` procedure described for QEMU: <>. -TODO: how to stop at `start_kernel`? gem5 listens for GDB by default, and therefore does not wait for a GDB connection to start like QEMU does. So when GDB connects we might have already passed `start_kernel`. Maybe `--debug-break=0` can be used? https://stackoverflow.com/questions/49296092/how-to-make-gem5-wait-for-gdb-to-connect-to-reliably-break-at-start-kernel-of-th +==== gem5 wait for GDB to connect before running + +By default, gem5 does not wait for GDB to connect, which means that you might go past your point of interest. + +This can however be enabled with the `wait_for_remote_gdb` option: + +.... +patch -d "$(./getvar gem5_src_dir)" -p 1 < patches/manual/gem5-wait-gdb.patch +.... + +Just just proceed as usual: + +.... +./run --gem5 +.... + +TODO why is this required, on another shell: + +.... +./gem5-shell +.... + +and finally: + +.... +./run-gdb --gem5 start_kernel +.... + +If I don't connect to the other shell on baremetal, the simulation breaks with: + +.... +.... + +You have to remove the patch if you want to go back to running without waiting for GDB to connect. + +Bibliography: https://stackoverflow.com/questions/49296092/how-to-make-gem5-wait-for-gdb-to-connect-to-reliably-break-at-start-kernel-of-th ===== gem5 GDB step debug kernel aarch64 @@ -8538,7 +8723,7 @@ warn: Couldn't read data from debugger. 4107767500: system.remote_gdb: remote gdb detached .... -I've also tried the fix at: https://stackoverflow.com/questions/27411621/remote-g-packet-reply-is-too-long-aarch64-arm64 by adding to the link:rungdb[] script: +I've also tried the fix at: https://stackoverflow.com/questions/27411621/remote-g-packet-reply-is-too-long-aarch64-arm64 by adding to the link:run-gdb[] script: .... -ex 'set tdesc filename out/aarch64/buildroot/build/gdb-7.11.1/./gdb/features/aarch64.xml' @@ -8570,7 +8755,7 @@ I press `n`, it just runs the program until the end, instead of stopping on the TODO: .... -./rungdb-user --arch arm --gem5 gem5-1.0/gem5/util/m5/m5 main +./run-gdb-user --arch arm --gem5 gem5-1.0/gem5/util/m5/m5 main .... breaks when `m5` is run on guest, but does not show the source code. @@ -8640,7 +8825,7 @@ since boot has already happened, and the parameters are already in the RAM of th ==== gem5 checkpoint internals -Checkpoints are stored inside the `m5out` directory at: +Checkpoints are stored inside the <> at: .... "$(./getvar --gem5 run_dir)/m5out/cpt." @@ -9042,25 +9227,51 @@ So let's explain them one by one here as we understand them: * `drm: Add component-aware simple encoder` allows you to see images through VNC: <> * `gem5: Add support for gem5's extended GIC mode` adds support for more than 8 cores: https://stackoverflow.com/questions/50248067/how-to-run-a-gem5-arm-aarch64-full-system-simulation-with-fs-py-with-more-than-8/50248068#5024806 -=== m5term +=== m5out directory -We use the `m5term` in-tree executable to connect to the terminal instead of a direct `telnet`. +When you run gem5, it generates an `m5out` directory at: -If you use `telnet` directly, it mostly works, but certain interactive features don't, e.g.: +.... +echo $(./getvar --arch arm --gem5 m5out_dir)" +.... -* up and down arrows for history havigation -* tab to complete paths -* `Ctrl-C` to kill processes +The location of that directory can be set with `./gem5.opt -d`, and defaults to `./m5out`. -TODO understand in detail what `m5term` does differently than `telnet`. +The files in that directory contains some very important information about the run, and you should become familiar with every one of them. + +==== system.terminal + +Contains UART output, both from the Linux kernel or from the baremetal system. + +Can also be seen live on <>. + +==== stats.txt + +This file contains important statistics about the run: + +.... +cat "$(./getvar --arch aarch64 m5out_dir)/stats.txt" +.... + +Whenever we run `m5 dumpstats` or `m5 exit`, a section with the following format is added to that file: + +.... +---------- Begin Simulation Statistics ---------- +[the stats] +---------- End Simulation Statistics ---------- +.... -=== gem5 stats +That file contains several important execution metrics, e.g. number of cycles and several types of cache misses: -Lets try to understand some stats better. +.... +system.cpu.numCycles +system.cpu.dtb.inst_misses +system.cpu.dtb.inst_hits +.... ==== rdtsc -link:https://en.wikipedia.org/wiki/Time_Stamp_Counter[x86 instruction] that returns the cycle count since reset: +Let's have some fun and try to correlate the gem5 cycle count `system.cpu.numCycles` with the link:https://en.wikipedia.org/wiki/Time_Stamp_Counter[x86 `rdtsc` instruction] that is supposed to do the same thing: .... ./build-buildroot --gem5 --kernel-modules && \ @@ -9093,6 +9304,68 @@ TODO We didn't manage to find a working ARM analogue to <>: link:packages * https://stackoverflow.com/questions/31620375/arm-cortex-a7-returning-pmccntr-0-in-kernel-mode-and-illegal-instruction-in-u/31649809#31649809 * https://blog.regehr.org/archives/794 +==== config.ini + +The `config.ini` file, contains a very good high level description of the system: + +.... +less $(./getvar --arch arm --gem5 m5out_dir)" +.... + +That file contains a tree representation of the system, sample excerpt: + +.... +[root] +type=Root +children=system +full_system=true + +[system] +type=ArmSystem +children=cpu cpu_clk_domain +auto_reset_addr_64=false +semihosting=Null + +[system.cpu] +type=AtomicSimpleCPU +children=dstage2_mmu dtb interrupts isa istage2_mmu itb tracer +branchPred=Null + +[system.cpu_clk_domain] +type=SrcClockDomain +clock=500 +.... + +Each node has: + +* a list of child nodes, e.g. `system` is a child of `root`, and both `cpu` and `cpu_clk_domain` are children of +* a list of parameters, e.g. `system.semihosting` is `Null`, which means that <> was turned off +** the `type` parameter shows is present on every node, and it maps to a `Python` object that inherits from `SimObject`. ++ +For example, `AtomicSimpleCPU` maps is defined at link:https://github.com/gem5/gem5/blob/05c4c2b566ce351ab217b2bd7035562aa7a76570/src/cpu/simple/AtomicSimpleCPU.py#L45[src/cpu/simple/AtomicSimpleCPU.py]. + +You can also get a simplified graphical view of the tree with: + +.... +xdg-open "$(./getvar --arch arm --gem5 m5out_dir)/config.dot.pdf" +.... + +Modifying the `config.ini` file does nothing since it gets overwritten every time. + +You can modify those configurations in `fs.py` easily by hacking up the `fs.py` file just before it runs, see for example: link:patches/manual/gem5-aarch64-baremetal.patch[] + +=== m5term + +We use the `m5term` in-tree executable to connect to the terminal instead of a direct `telnet`. + +If you use `telnet` directly, it mostly works, but certain interactive features don't, e.g.: + +* up and down arrows for history havigation +* tab to complete paths +* `Ctrl-C` to kill processes + +TODO understand in detail what `m5term` does differently than `telnet`. + === gem5 Python scripts without rebuild We have made a crazy setup that allows you to just `cd` into `submodules/gem5`, and edit Python scripts directly there. @@ -9140,7 +9413,7 @@ Disadvantages over `fs.py`: * only works for ARM, not other archs * not as many configuration options as `fs.py`, many things are hardcoded -We setup 2 big and 2 small CPUs, but `cat /proc/cpuinfo` shows 4 identical CPUs instead of 2 of two different types, likely because gem5 does not expose some informational register much like the caches: https://www.mail-archive.com/gem5-users@gem5.org/msg15426.html `config.ini` does show that the two big ones are `DerivO3CPU` and the small ones are `MinorCPU`. +We setup 2 big and 2 small CPUs, but `cat /proc/cpuinfo` shows 4 identical CPUs instead of 2 of two different types, likely because gem5 does not expose some informational register much like the caches: https://www.mail-archive.com/gem5-users@gem5.org/msg15426.html <> does show that the two big ones are `DerivO3CPU` and the small ones are `MinorCPU`. TODO: why is the `--dtb` required despite `fs_bigLITTLE.py` having a DTB generation capability? Without it, nothing shows on terminal, and the simulation terminates with `simulate() limit reached @ 18446744073709551615`. The magic `vmlinux.vexpress_gem5_v1.20170616` works however without a DTB. @@ -9404,7 +9677,7 @@ vim "$(./getvar --arch arm run_cmd_file)" ./"$(./getvar --arch arm run_cmd_file)" .... -Next, you will also want to give the relevant images to save them time, see: <>. +Next, you will also want to give the relevant images to save them time, see: <>. Finally, do a clone of the relevant repository out of tree and reproduce the bug there, to be 100% sure that it is an actual upstream bug, and to provide developers with the cleanest possible commands. @@ -9413,6 +9686,342 @@ For QEMU and Buildroot, we have the following convenient setups respectively: * https://github.com/cirosantilli/qemu-test * https://github.com/cirosantilli/buildroot/tree/in-tree-package-master +== Baremetal + +=== Baremetal GDB step debug + +GDB step debug works on baremetal exactly as it does on the Linux kernel, except that is is even cooler here since we can easily control and understand every single instruction that is being run! + +For example, on the first shell: + +.... +./run --arch arm --baremetal prompt --debug-guest +.... + +then on the second shell: + +.... +./run-gdb --arch arm --baremetal prompt --no-continue +.... + +and now we are left at the very first executed instruction of our tiny bootloader: link:baremetal/lib/arm.S[] + +Then just use `stepi` to when jumping into main to go to the C code in link:baremetal/prompt.c[]. + +The bootloader is used to put the hardware in its main operating mode before we run our main payload on it. + +You can also find executables that don't use the bootloader at all under `baremetal/arch//no_bootloader/*.S`, e.g.: + +.... +./run --arch arm --baremetal arch/arm/no_bootloader/semihost_exit --debug-guest +.... + +Alternatively, skip directly to the C program main function with: + +.... +./run-gdb --arch arm --baremetal prompt main +.... + +gem5 step debug requires a patch as mentioned at <>: + +.... +patch -d "$(./getvar gem5_src_dir)" -p 1 < patches/manual/gem5-wait-gdb.patch +.... + +and then proceed as usual: + +.... +./run --arch arm --baremetal prompt --debug-guest --gem5 +.... + +and on another shell: + +.... +./run-gdb --arch arm --baremetal prompt --gem5 --no-continue +.... + +`aarch64` GDB step debug is broken as mentioned at: <>. + +=== Semihosting + +Semihosting is a publicly documented interface specified by ARM Holdings that allows us to do some magic operations very useful in development. + +Semihosting is implemented both on some real devices and on simulators such as QEMU and gem5. + +It is documented at: https://developer.arm.com/docs/100863/latest/introduction + +Example: + +.... +./run --arch arm --baremetal arch/arm/semihost_exit +.... + +makes both the QEMU and gem5 host executables exit. + +Source: link:baremetal/arch/arm/no_bootloader/semihost_exit.S[] + +That program program contains the code: + +.... +mov r0, #0x18 +ldr r1, =#0x20026 +svc 0x00123456 +.... + +and we can see from the docs that `0x18` stands for the `SYS_EXIT` command. + +This is also how we implement the `exit(0)` system call in C for link:baremetal/exit.c[] through the Newlib via the function `_exit` at link:baremetal/lib/common.c[]. + +Other magic operations we can do with semihosting besides exiting the on the host include: + +* exit +* read and write to host stdin and stdout +* read and write to host files + +Alternatives exist for some semihosting operations, e.g.: + +* UART IO for host stdin and stdout in both emulators and real hardware +* <> for <>, e.g. `m5 exit` makes the emulator quit + +The big advantage of semihosting is that it is standardized across all ARM boards, and therefore allows you to make a single image that does those magic operations instead of having to compile multiple images with different magic addresses. + +The downside of semihosting is that it is ARM specific. TODO is it an open standard that other vendors can implement? + +In QEMU, we enable semihosting with: + +.... +-semihosting +.... + +Newlib 9c84bfd47922aad4881f80243320422b621c95dc already has a semi-hosting implementation at: + +.... +newlib/libc/sys/arm/syscalls.c +.... + +TODO: how to use it? Possible through crosstool-NG? In the worst case we could just copy it. + +Bibliography: + +* https://stackoverflow.com/questions/31990487/how-to-cleanly-exit-qemu-after-executing-bare-metal-program-without-user-interve/40957928#40957928 +* https://balau82.wordpress.com/2010/11/04/qemu-arm-semihosting/ + +==== gem5 semihosting + +TODO how to enable it? <> contains: + +.... +root.system.semihosting=Null +.... + +and there are no hits under `configs/`. + +=== gem5 baremetal carriage return + +TODO: our example is printing newlines without automatic carriage return `\r` as in: + +.... +enter a character + got: a +.... + +We use `m5term` by default, and if we try `telnet` instead: + +.... +telnet localhost 3456 +.... + +it does add the carriage returns automatically. + +=== Baremetal host packaged toolchain + +For `arm`, some baremetal examples compile fine with: + +.... +sudo apt-get install gcc-arm-none-eabi qemu-system-arm +./build-baremetal --arch arm --prebuilt +./run --arch arm --baremetal prompt --prebuilt +.... + +However, there are as usual limitations to using prebuilts: + +* certain examples fail to build with the Ubuntu packaged toolchain. E.g.: link:baremetal/exit.c[] fails with: ++ +.... +/usr/lib/gcc/arm-none-eabi/6.3.1/../../../arm-none-eabi/lib/libg.a(lib_a-fini.o): In function `__libc_fini_array': +/build/newlib-8gJlYR/newlib-2.4.0.20160527/build/arm-none-eabi/newlib/libc/misc/../../../../../newlib/libc/misc/fini.c:33: undefined reference to `_fini' +collect2: error: ld returned 1 exit status +.... ++ +with the prebuilt toolchain, and I'm lazy to debug. +* there seems to to be no analogous `aarch64` Ubuntu package to `gcc-arm-none-eabi` + +=== C++ baremetal + +TODO I tried by there was an error. Not yet properly reported. Should not be hard in theory since `libstdc++` is just part of GCC, as shown at: https://stackoverflow.com/questions/21872229/how-to-edit-and-re-build-the-gcc-libstdc-c-standard-library-source/51946224#51946224 + +=== GDB builtin CPU simulator + +It is incredible, but GDB also has a CPU simulator inside of it as documented at: https://sourceware.org/gdb/onlinedocs/gdb/Target-Commands.html + +TODO: any advantage over QEMU? I doubt it, mostly using it as as toy for now: + +Without running `./run`, do directly: + +.... +./run-gdb --arch arm --baremetal prompt --sim +.... + +Then inside GDB: + +.... +load +starti +.... + +and now you can debug normally. + +Enabled with the crosstool-NG configuration: + +.... +CT_GDB_CROSS_SIM=y +.... + +which by grepping crosstool-NG we can see does on GDB: + +.... +./configure --enable-sim +.... + +Those are not set by default on `gdb-multiarch` in Ubuntu 16.04. + +Bibliography: + +* https://stackoverflow.com/questions/49470659/arm-none-eabi-gdb-undefined-target-command-sim +* http://cs107e.github.io/guides/gdb/ + +==== GDB builtin CPU simulator userland + +Since I had this compiled, I also decided to try it out on userland. + +I was also able to run a freestanding Linux userland example on it: https://github.com/cirosantilli/arm-assembly-cheat/blob/cd232dcaf32c0ba6399b407e0b143d19b6ec15f4/v7/linux/hello.S + +It just ignores the `swi` however, and does not forward syscalls to the host like QEMU does. + +Then I tried a glibc example: https://github.com/cirosantilli/arm-assembly-cheat/blob/cd232dcaf32c0ba6399b407e0b143d19b6ec15f4/v7/mov.S + +First it wouldn't break, so I added `-static` to the `Makefile`, and then it started failing with: + +.... +Unhandled v6 thumb insn +.... + +Doing: + +.... +help architecture +.... + +shows ARM version up to `armv6`, so maybe `armv6` is not implemented? + +=== How we got some baremetal stuff to work + +It is nice when thing just work. + +But you can also learn a thing or two from how I actually made them work in the first place. + +==== Find the UART address + +Enter the QEMU console: + +.... +Ctrl-X C +.... + +Then do: + +.... +info mtree +.... + +And look for `pl011`: + +.... + 0000000009000000-0000000009000fff (prio 0, i/o): pl011 +.... + +On gem5, it is easy to find it on the source. We are using the machine `RealView_PBX`, and a quick grep leads us to: https://github.com/gem5/gem5/blob/a27ce59a39ec8fa20a3c4e9fa53e9b3db1199e91/src/dev/arm/RealView.py#L615 + +.... +class RealViewPBX(RealView): + uart = Pl011(pio_addr=0x10009000, int_num=44) +.... + +==== aarch64 NEON setup + +Inside link:baremetal/lib/aarch64.S[] there is a chunk of code called "NEON setup". + +Without that, the `printf`: + +.... +printf("got: %c\n", c); +.... + +compiled to a: + +.... +str q0, [sp, #80] +.... + +which uses NEON registers, and goes into an exception loop. + +It was a bit confusing because there was a previous `printf`: + +.... +printf("enter a character\n"); +.... + +which did not blow up because GCC compiles it into `puts` directly since it has no arguments, and that does not generate NEON instructions. + +The last instructions ran was found with: + +.... +while(1) +stepi +end +.... + +or by hacking the QEMU CLI to contain: + +..... +-D log.log -d in_asm +..... + +I could not find any previous NEON instruction executed so this led me to suspect that some NEON initialization was required: + +* http://infocenter.arm.com/help/topic/com.arm.doc.dai0527a/DAI0527A_baremetal_boot_code_for_ARMv8_A_processors.pdf "Bare-metal Boot Code for ARMv8-A Processors" +* https://community.arm.com/processors/f/discussions/5409/how-to-enable-neon-in-cortex-a8 +* https://stackoverflow.com/questions/19231197/enable-neon-on-arm-cortex-a-series + +We then tried to copy the code from the "Bare-metal Boot Code for ARMv8-A Processors" document: + +.... +// Disable trapping of accessing in EL3 and EL2. +MSR CPTR_EL3, XZR +MSR CPTR_EL3, XZR +// Disable access trapping in EL1 and EL0. +MOV X1, #(0x3 << 20) // FPEN disables trapping to EL1. +MSR CPACR_EL1, X1 +ISB +.... + +but it entered an exception loop at `MSR CPTR_EL3, XZR`. + +We then found out that QEMU starts in EL1, and so we kept just the EL1 part, and it worked. Related: + +* https://stackoverflow.com/questions/42824706/qemu-system-aarch64-entering-el1-when-emulating-a53-power-up +* https://stackoverflow.com/questions/37299524/neon-support-in-armv8-system-mode-qemu + == Benchmark this repo In this section document how benchmark builds and runs of this repo, and how to investigate what the bottleneck is. @@ -9785,6 +10394,16 @@ We have link:https://buildroot.org/downloads/manual/manual.html#ccache[enabled c * absolute paths are used and GDB can find source files * but builds are not reused across separated LKMC directories +=== Rebuild while running + +Not possible because + +.... +Text file busy +.... + +openat(AT_FDCWD, "sleep.out", O_WRONLY) = -1 ETXTBSY () + === Simultaneous runs When doing long simulations sweeping across multiple system parameters, it becomes fundamental to do multiple simulations in parallel. @@ -9821,7 +10440,7 @@ Each run gets a separate output directory. For example: ./run --arch aarch64 --gem5 --run-id 1 &>/dev/null & .... -produces two separate `m5out` directories: +produces two separate <>: .... echo "$(./getvar --arch aarch64 --gem5 --run-id 0 m5out_dir)" @@ -9849,7 +10468,7 @@ Like <>, you will need to pass the `-n` option to anything tha .... ./run --run-id 1 -./rungdb --run-id 1 +./run-gdb --run-id 1 .... To run multiple gem5 checkouts, see: <>. @@ -9865,7 +10484,7 @@ If a port is not free, it just crashes. We assign a contiguous port range for each run ID. ** gem5 automatically increments ports until it finds a free one. + -gem5 60600f09c25255b3c8f72da7fb49100e2682093a does not seem to expose a way to set the terminal and VNC ports from `fs.py`, so we just let gem5 assign the ports itself, and use `-n` only to match what it assigned. Those ports both appear on `config.ini`. +gem5 60600f09c25255b3c8f72da7fb49100e2682093a does not seem to expose a way to set the terminal and VNC ports from `fs.py`, so we just let gem5 assign the ports itself, and use `-n` only to match what it assigned. Those ports both appear on <>. + The GDB port can be assigned on `gem5.opt --remote-gdb-port`, but it does not appear on `config.ini`. @@ -9903,7 +10522,7 @@ The `git fetch --unshallow` is needed the first time because link:configure[] on The `--linux-build-id` option should be passed to all scripts that support it, much like `--arch` for the <>, e.g. to step debug: ..... -./rungdb --linux-build-id v4.16 +./run-gdb --linux-build-id v4.16 ..... To run both kernels simultaneously, one on each QEMU instance, see: <>. @@ -10212,7 +10831,7 @@ Shell 1: Shell 2: .... -./rungdb start_kernel +./run-gdb start_kernel .... Should break GDB at `start_kernel`. diff --git a/baremetal/arch/arm/gem5_assert.S b/baremetal/arch/arm/gem5_assert.S new file mode 100644 index 00000000..8c351178 --- /dev/null +++ b/baremetal/arch/arm/gem5_assert.S @@ -0,0 +1,19 @@ +/* assert 0x12345678 + 1 == 0x12345679 */ +.global main +main: + movw r0, #:lower16:myvar + movt r0, #:upper16:myvar + ldr r1, [r0] + add r1, r1, #1 + str r1, [r0] + movw r2, #0x5679 + movt r2, #0x1234 + cmp r1, r2 + beq ok + # m5 fail 1 + mov r0, #0; mov r1, #0; mov r2, #1; mov r3, #0; .inst 0xEE000110 | (0x22 << 16); +ok: + # m5 exit + mov r0, #0; mov r1, #0; .inst 0xEE000110 | (0x21 << 16); +myvar: + .word 0x12345678 diff --git a/baremetal/arch/arm/no_bootloader/semihost_exit.S b/baremetal/arch/arm/no_bootloader/semihost_exit.S new file mode 100644 index 00000000..c9c64678 --- /dev/null +++ b/baremetal/arch/arm/no_bootloader/semihost_exit.S @@ -0,0 +1,5 @@ +.global mystart +mystart: + mov r0, #0x18 + ldr r1, =#0x20026 + svc 0x00123456 diff --git a/baremetal/arch/arm/semihost_exit.S b/baremetal/arch/arm/semihost_exit.S new file mode 100644 index 00000000..8e485d6f --- /dev/null +++ b/baremetal/arch/arm/semihost_exit.S @@ -0,0 +1,5 @@ +.global main +main: + mov r0, #0x18 + ldr r1, =#0x20026 + svc 0x00123456 diff --git a/baremetal/exit.c b/baremetal/exit.c new file mode 100644 index 00000000..07bdfadd --- /dev/null +++ b/baremetal/exit.c @@ -0,0 +1,7 @@ +#include +#include + +void main(void) { + exit(0); +} + diff --git a/baremetal/lib/aarch64.S b/baremetal/lib/aarch64.S new file mode 100644 index 00000000..e61e812f --- /dev/null +++ b/baremetal/lib/aarch64.S @@ -0,0 +1,11 @@ +.global mystart +mystart: + /* = NEON setup */ + mov x1, #(0x3 << 20) + msr cpacr_el1, x1 + isb + + ldr x0, =stack_top + mov sp, x0 + bl main + b . diff --git a/baremetal/lib/arm.S b/baremetal/lib/arm.S new file mode 100644 index 00000000..c1058396 --- /dev/null +++ b/baremetal/lib/arm.S @@ -0,0 +1,5 @@ +.global mystart +mystart: + ldr sp, =stack_top + bl main + b . diff --git a/baremetal/lib/common.c b/baremetal/lib/common.c new file mode 100644 index 00000000..a74780b9 --- /dev/null +++ b/baremetal/lib/common.c @@ -0,0 +1,65 @@ +#include + +enum { + UART_FR_RXFE = 0x10, +}; + +#define UART_DR(baseaddr) (*(unsigned int *)(baseaddr)) +#define UART_FR(baseaddr) (*(((unsigned int *)(baseaddr))+6)) + +int _close(int file) { return -1; } + +int _fstat(int file, struct stat *st) { + st->st_mode = S_IFCHR; + return 0; +} + +int _isatty(int file) { return 1; } +int _lseek(int file, int ptr, int dir) { return 0; } +int _open(const char *name, int flags, int mode) { return -1; } + +int _read(int file, char *ptr, int len) { + int todo; + if (len == 0) + return 0; + while (UART_FR(UART0_ADDR) & UART_FR_RXFE); + *ptr++ = UART_DR(UART0_ADDR); + for (todo = 1; todo < len; todo++) { + if (UART_FR(UART0_ADDR) & UART_FR_RXFE) { + break; + } + *ptr++ = UART_DR(UART0_ADDR); + } + return todo; +} + +char *heap_end = 0; +caddr_t _sbrk(int incr) { + extern char heap_low; + extern char heap_top; + char *prev_heap_end; + if (heap_end == 0) { + heap_end = &heap_low; + } + prev_heap_end = heap_end; + if (heap_end + incr > &heap_top) { + /* Heap and stack collision */ + return (caddr_t)0; + } + heap_end += incr; + return (caddr_t)prev_heap_end; +} + +int _write(int file, char *ptr, int len) { + int todo; + for (todo = 0; todo < len; todo++) { + UART_DR(UART0_ADDR) = *ptr++; + } + return len; +} + +void _exit(int status) { +#if defined(__arm__) + __asm__ __volatile__ ("mov r0, #0x18; ldr r1, =#0x20026; svc 0x00123456"); +#endif +} diff --git a/baremetal/link.ld b/baremetal/link.ld new file mode 100644 index 00000000..2a3ef8c5 --- /dev/null +++ b/baremetal/link.ld @@ -0,0 +1,19 @@ +ENTRY(mystart) +SECTIONS +{ + .text : { + */bootloader.o(.text) + *(.text) + *(.rodata) + *(.data) + *(COMMON) + } + /* gem5 uses the bss as a measure of the kernel size. */ + .bss : { *(.bss) } + heap_low = .; + . = . + 0x1000000; + heap_top = .; + . = . + 0x1000000; + stack_top = .; +} + diff --git a/baremetal/prompt.c b/baremetal/prompt.c new file mode 100644 index 00000000..ecae2d39 --- /dev/null +++ b/baremetal/prompt.c @@ -0,0 +1,22 @@ +#include +#include + +void main(void) { + char c; + char *ptr = NULL; + size_t alloc_size = 1; + while (1) { + printf("enter a character\n"); + c = getchar(); + printf("got: %c\n", c); + ptr = realloc(ptr, alloc_size); + if (ptr == NULL) { + puts("out of memory"); + break; + } else { + printf("new alloc of %d bytes at address 0x%p\n", alloc_size, ptr); + alloc_size <<= 1; + } + } +} + diff --git a/bench-all b/bench-all index 928a0924..7f26f9ce 100755 --- a/bench-all +++ b/bench-all @@ -79,7 +79,7 @@ do_bench_buildroot_build() ( baseline_suffix= fi common_build_dir="$("$getvar" --arch "$arch" --buildroot-build-id "$build_id" build_dir)" - common_images_dir="$("$getvar" --arch "$arch" --buildroot-build-id "$build_id" images_dir)" + common_images_dir="$("$getvar" --arch "$arch" --buildroot-build-id "$build_id" buildroot_images_dir)" "${root_dir}/build-buildroot" --arch "$arch" $baseline --buildroot-build-id "$build_id" --clean "${root_dir}/build-buildroot" --arch "$arch" $baseline --buildroot-build-id "$build_id" --no-all -- source "${root_dir}/build-buildroot" --arch "$arch" $baseline --buildroot-build-id "$build_id" diff --git a/bench-boot b/bench-boot index 3c04ac46..8d0e72fb 100755 --- a/bench-boot +++ b/bench-boot @@ -30,8 +30,13 @@ bench() ( "${root_dir}/bench-cmd" "./run --arch ${1}${extra_args}" "$common_bench_boot" ) +newline() ( + echo >> "$common_bench_boot" +) + gem5_insts() ( printf "instructions $(./gem5-stat --arch "$1" sim_insts)\n" >> "$common_bench_boot" + newline ) qemu_insts() ( @@ -39,10 +44,7 @@ qemu_insts() ( ./qemu-trace2txt --arch "$common_arch" common_qemu_trace_txt_file="$("$getvar" --arch "$common_arch" qemu_trace_txt_file)" printf "instructions $(wc -l "${common_qemu_trace_txt_file}" | cut -d' ' -f1)\n" >> "$common_bench_boot" -) - -newline() ( - echo >> "$common_bench_boot" + newline ) rm -f "${common_bench_boot}" @@ -55,31 +57,26 @@ newline if [ "$test_size" -ge 2 ]; then bench "${arch} --eval '/poweroff.out' --trace exec_tb" qemu_insts "$arch" - newline bench "$arch --eval 'm5 exit' --gem5" gem5_insts "$arch" - newline fi #bench "$arch --eval 'm5 exit' --gem5 -- --cpu-type=DerivO3CPU ${caches}" #gem5_insts "$arch" -#newline arch=arm bench "$arch --eval '/poweroff.out'" -newline if [ "$test_size" -ge 2 ]; then bench "$arch --eval '/poweroff.out' --trace exec_tb" qemu_insts "$arch" - newline #bench "$arch --eval 'm5 exit' --gem5" #gem5_insts "$arch" - #newline fi -#if [ "$test_size" -ge 3 ]; then -# bench "$arch --eval 'm5 exit' --gem5 -- --cpu-type=HPI ${caches}" -# gem5_insts "$arch" -# newline -#fi +if [ "$test_size" -ge 3 ]; then + #bench "$arch --eval 'm5 exit' --gem5 -- --cpu-type=HPI ${caches}" + #gem5_insts "$arch" + bench "$arch --eval 'm5 exit' --gem5 --gem5-biglittle" + gem5_insts "$arch" +fi arch=aarch64 bench "$arch --eval '/poweroff.out'" @@ -87,13 +84,10 @@ newline if [ "$test_size" -ge 2 ]; then bench "$arch --eval '/poweroff.out' --trace exec_tb" qemu_insts "$arch" - newline bench "$arch --eval 'm5 exit' --gem5" gem5_insts "$arch" - newline fi if [ "$test_size" -ge 3 ]; then bench "$arch --eval 'm5 exit' --gem5 -- --cpu-type=HPI ${caches}" gem5_insts "$arch" - newline fi diff --git a/build-all b/build-all index 333ee1db..a7d3de3f 100755 --- a/build-all +++ b/build-all @@ -15,8 +15,14 @@ done shift "$(($OPTIND - 1))" for arch in $archs; do ./build-qemu --arch "$arch" - ./build-buildroot --arch "$arch" --gem5 --kernel-modules -l "$@" if "$gem5"; then ./build-gem5 --arch "$arch" fi + ./build-buildroot --arch "$arch" --gem5 --kernel-modules -l "$@" + if [ ! "$arch" = x86_64 ]; then + ./build-crosstool-ng --arch "$arch" + ./build-baremetal --arch "$arch" + ./build-baremetal --arch "$arch" -g + ./build-baremetal --arch "$arch" -g --machine RealViewPBX + fi done diff --git a/build-baremetal b/build-baremetal new file mode 100755 index 00000000..fb086474 --- /dev/null +++ b/build-baremetal @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 + +import glob +import multiprocessing +import os +import shutil +import sys +import time + +import common + +def build_dir(subpath, gcc, cflags, entry_address, bootloader_obj, common_obj, bootloader=True): + """ + Build all .c and .S files in a given subpath of the baremetal source + directory non recursively. + + Place outputs on the same subpath or the output directory. + """ + in_dir = os.path.join(common.baremetal_src_dir, subpath) + out_dir = os.path.join(common.baremetal_out_dir, subpath) + os.makedirs(out_dir, exist_ok=True) + if bootloader: + bootloader_cmd = [bootloader_obj] + else: + bootloader_cmd = [] + for in_basename in os.listdir(in_dir): + in_path = os.path.join(in_dir, in_basename) + if os.path.isfile(in_path) and os.path.splitext(in_basename)[1] in (common.c_ext, common.asm_ext): + in_name = os.path.splitext(in_basename)[0] + main_obj = os.path.join(common.baremetal_out_dir, subpath, '{}{}'.format(in_name, common.obj_ext)) + assert common.run_cmd( + [gcc] + + cflags + + [ + '-c', + '-o', main_obj, + os.path.join(common.baremetal_src_dir, in_path), + ] + ) == 0 + assert common.run_cmd( + [gcc] + + cflags + + [ + '-Wl,--section-start=.text={:#x}'.format(entry_address), + '-o', os.path.join(common.baremetal_out_dir, subpath, in_name + common.baremetal_out_ext), + '-T', os.path.join(common.baremetal_src_dir, 'link.ld'), + ] + + bootloader_cmd + + [ + common_obj, + main_obj, + ] + ) == 0 + +def get_argparse(): + parser = common.get_argparse(argparse_args={ + 'description': 'Build the baremetal examples with crosstool-NG' + }) + common.add_build_arguments(parser) + return parser + +def main(args, extra_args=None): + if args.clean: + common.rmrf(common.baremetal_out_dir) + else: + common.raise_no_x86(args.arch) + bootloader_obj = os.path.join(common.baremetal_out_lib_dir, 'bootloader{}'.format(common.obj_ext)) + common_obj = os.path.join(common.baremetal_out_lib_dir, 'common{}'.format(common.obj_ext)) + cflags = [ + '-ggdb3', + '-mcpu={}'.format(common.mcpu), + '-nostartfiles', + '-O0', + ] + if args.prebuilt: + gcc = 'arm-none-eabi-gcc' + else: + os.environ['PATH'] = common.crosstool_ng_bin_dir + os.environ['PATH'] + gcc = common.get_toolchain_tool('gcc') + if args.gem5: + if common.machine == 'VExpress_GEM5_V1': + entry_address = 0x80000000 + uart_address = 0x1c090000 + elif common.machine == 'RealViewPBX': + entry_address = 0x10000 + uart_address = 0x10009000 + else: + raise Exception('unknown machine: ' + common.machine) + else: + entry_address = 0x40000000 + uart_address = 0x09000000 + os.makedirs(common.baremetal_out_dir, exist_ok=True) + os.makedirs(common.baremetal_out_lib_dir, exist_ok=True) + assert common.run_cmd( + [gcc] + + cflags + + [ + '-c', + '-o', bootloader_obj, + os.path.join(common.baremetal_src_lib_dir, '{}{}'.format(args.arch, common.asm_ext)), + ] + ) == 0 + assert common.run_cmd( + [gcc] + + cflags + + [ + '-c', + '-D', + 'UART0_ADDR={:#x}'.format(uart_address), + '-o', common_obj, + os.path.join(common.baremetal_src_lib_dir, 'common' + common.c_ext), + ] + ) == 0 + build_dir( + '', + gcc=gcc, + cflags=cflags, + entry_address=entry_address, + bootloader_obj=bootloader_obj, + common_obj=common_obj, + ) + build_dir( + os.path.join('arch', args.arch), + gcc=gcc, + cflags=cflags, + entry_address=entry_address, + bootloader_obj=bootloader_obj, + common_obj=common_obj, + ) + build_dir( + os.path.join('arch', args.arch, 'no_bootloader'), + gcc=gcc, + cflags=cflags, + entry_address=entry_address, + bootloader_obj=bootloader_obj, + common_obj=common_obj, + bootloader=False, + ) + return 0 + +if __name__ == '__main__': + parser = common.get_argparse( + default_args={'baremetal': 'all'}, + ) + common.add_build_arguments(parser) + args = common.setup(parser) + start_time = time.time() + exit_status = main(args) + end_time = time.time() + common.print_time(end_time - start_time) + sys.exit(exit_status) diff --git a/build-buildroot b/build-buildroot index 8f44d9d2..7cbce9e2 100755 --- a/build-buildroot +++ b/build-buildroot @@ -168,9 +168,9 @@ def main(args, extra_args=None): buildroot_configs = args.buildroot_config buildroot_configs.extend([ 'BR2_JLEVEL={}'.format(nproc), - 'BR2_DL_DIR="{}"'.format(common.dl_dir), + 'BR2_DL_DIR="{}"'.format(common.buildroot_download_dir), ]) - write_configs(buildroot_configs) + common.write_configs(common.buildroot_config_file, buildroot_configs) if not args.baseline: buildroot_configs.extend([ 'BR2_GLOBAL_PATCH_DIR="{}"'.format( @@ -182,7 +182,7 @@ def main(args, extra_args=None): 'BR2_ROOTFS_OVERLAY="{}"'.format( path_relative_to_buildroot(os.path.join(common.root_dir, 'rootfs_overlay'))), 'BR2_ROOTFS_POST_BUILD_SCRIPT="{}"'.format( - path_relative_to_buildroot(os.path.join(common.root_dir, 'rootfs_post_build_script'))), + path_relative_to_buildroot(os.path.join(common.root_dir, 'rootfs-post-build-script'))), 'BR2_ROOTFS_USERS_TABLES="{}"'.format( path_relative_to_buildroot(os.path.join(common.root_dir, 'user_table'))), ]) @@ -253,7 +253,7 @@ def main(args, extra_args=None): kernel_config_fragments[i] = path_relative_to_buildroot(frag) buildroot_kernel_config_fragment_str = 'BR2_LINUX_KERNEL_CONFIG_FRAGMENT_FILES="{}"'.format(' '.join(kernel_config_fragments)) buildroot_configs.append(buildroot_kernel_config_fragment_str) - write_configs(buildroot_configs, buildroot_config_fragments) + common.write_configs(common.buildroot_config_file, buildroot_configs, buildroot_config_fragments) subprocess.check_call( [ 'make', @@ -304,22 +304,6 @@ def main(args, extra_args=None): def path_relative_to_buildroot(abspath): return os.path.relpath(abspath, common.buildroot_src_dir) -def write_configs(buildroot_configs, buildroot_config_fragments=None): - """ - Write extra configs into the Buildroot config file. - TODO Can't get rid of these for now with nice fragments: - http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config - """ - if buildroot_config_fragments is None: - buildroot_config_fragments = [] - with open(common.buildroot_config_file, 'a') as br2_config_file: - for buildroot_config_fragment in buildroot_config_fragments: - with open(buildroot_config_fragment, 'r') as br2_config_fragment: - for line in br2_config_fragment: - br2_config_file.write(line) - for buildroot_config in buildroot_configs: - br2_config_file.write(buildroot_config + '\n') - if __name__ == '__main__': parser = get_argparse() args = common.setup(parser) diff --git a/build-crosstool-ng b/build-crosstool-ng new file mode 100755 index 00000000..f9ac2970 --- /dev/null +++ b/build-crosstool-ng @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 + +import multiprocessing +import os +import shutil +import sys +import time + +import common + +def get_argparse(): + parser = common.get_argparse(argparse_args={ + 'description': 'Build crosstool-NG with Newlib for bare metal compilation' + }) + common.add_build_arguments(parser) + return parser + +def main(args, extra_args=None): + common.raise_no_x86(args.arch) + nproc = multiprocessing.cpu_count() + defconfig_dest = os.path.join(common.crosstool_ng_util_dir, 'defconfig') + os.makedirs(common.crosstool_ng_util_dir, exist_ok=True) + os.makedirs(common.crosstool_ng_download_dir, exist_ok=True) + + # Bootstrap out-ot-tree WONTFIX. I've tried. + # https://github.com/crosstool-ng/crosstool-ng/issues/1021 + os.chdir(common.crosstool_ng_src_dir) + assert common.run_cmd( + [os.path.join(common.crosstool_ng_src_dir, 'bootstrap')], + ) == 0 + os.chdir(common.crosstool_ng_util_dir) + assert common.run_cmd( + [ + os.path.join(common.crosstool_ng_src_dir, 'configure'), + '--enable-local', + ], + ) == 0 + assert common.run_cmd( + [ + 'make', + '-j', str(nproc) + ], + ) == 0 + + # Build the toolchain. + shutil.copy2( + os.path.join(common.root_dir, 'crosstool_ng_config', args.arch), + defconfig_dest + ) + common.write_configs( + common.crosstool_ng_defconfig, + [ + 'CT_PREFIX_DIR="{}"'.format(common.crosstool_ng_install_dir), + 'CT_WORK_DIR="{}"'.format(common.crosstool_ng_build_dir), + 'CT_LOCAL_TARBALLS_DIR="{}"'.format(common.crosstool_ng_download_dir), + ] + ) + assert common.run_cmd( + [ + common.crosstool_ng_executable, + 'defconfig', + ], + ) == 0 + os.unlink(defconfig_dest) + assert common.run_cmd( + [ + common.crosstool_ng_executable, + 'build', + '-j', str(nproc) + ], + out_file=os.path.join(common.crosstool_ng_build_dir, 'lkmc.log'), + delete_env=['LD_LIBRARY_PATH'], + extra_env={'PATH': common.ccache_dir + ':' + os.environ['PATH']}, + ) == 0 + +if __name__ == '__main__': + parser = get_argparse() + args = common.setup(parser) + start_time = time.time() + exit_status = main(args) + end_time = time.time() + common.print_time(end_time - start_time) + sys.exit(exit_status) diff --git a/build-gem5 b/build-gem5 index 8d7c38cd..28246ec4 100755 --- a/build-gem5 +++ b/build-gem5 @@ -84,7 +84,13 @@ else: extra_env={'PATH': common.ccache_dir + ':' + os.environ['PATH']}, ) == 0 term_src_dir = os.path.join(common.gem5_src_dir, 'util/term') + m5term_build = os.path.join(term_src_dir, 'm5term') subprocess.check_call(['make', '-C', term_src_dir]) - shutil.copy2(os.path.join(term_src_dir, 'm5term'), common.gem5_m5term) + if os.path.exists(common.gem5_m5term): + # Otherwise shutil.copy2 would fail with "Text file busy" if you + # tried to rebuild while running m5term: + # https://stackoverflow.com/questions/16764946/what-generates-the-text-file-busy-message-in-unix/52427512#52427512 + os.unlink(common.gem5_m5term) + shutil.copy2(m5term_build, common.gem5_m5term) end_time = time.time() common.print_time(end_time - start_time) diff --git a/common.py b/common.py index 14bef609..13bc687c 100644 --- a/common.py +++ b/common.py @@ -26,9 +26,11 @@ gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5') out_dir = os.path.join(root_dir, 'out') bench_boot = os.path.join(out_dir, 'bench-boot.txt') -dl_dir = os.path.join(out_dir, 'dl') +packages_dir = os.path.join(root_dir, 'packages') +kernel_modules_src_dir = os.path.join(this.packages_dir, 'kernel_modules') submodules_dir = os.path.join(root_dir, 'submodules') buildroot_src_dir = os.path.join(submodules_dir, 'buildroot') +crosstool_ng_src_dir = os.path.join(submodules_dir, 'crosstool-ng') gem5_default_src_dir = os.path.join(submodules_dir, 'gem5') linux_src_dir = os.path.join(submodules_dir, 'linux') extract_vmlinux = os.path.join(linux_src_dir, 'scripts', 'extract-vmlinux') @@ -97,6 +99,14 @@ def get_argparse(default_args=None, argparse_args=None): '-a', '--arch', choices=arch_choices, default='x86_64', help='CPU architecture. Default: %(default)s' ) + parser.add_argument( + '--baremetal', + help='Use Baremetal examples instead of Linux kernel ones' + ) + parser.add_argument( + '--crosstool-ng-build-id', default=default_build_id, + help='Crosstool-NG build ID. Allows you to keep multiple separate crosstool-NG builds. Default: %(default)s' + ) parser.add_argument( '-g', '--gem5', default=False, action='store_true', help='Use gem5 instead of QEMU' @@ -105,6 +115,14 @@ def get_argparse(default_args=None, argparse_args=None): '-L', '--linux-build-id', default=default_build_id, help='Linux build ID. Allows you to keep multiple separate Linux builds. Default: %(default)s' ) + parser.add_argument( + '--machine', + help='''Machine type. +QEMU default: virt +gem5 default: VExpress_GEM5_V1 +See the documentation for other values known to work. +''' + ) parser.add_argument( '-M', '--gem5-build-id', default=default_build_id, help='gem5 build ID. Allows you to keep multiple separate gem5 builds. Default: %(default)s' @@ -123,6 +141,14 @@ def get_argparse(default_args=None, argparse_args=None): ID for run outputs such as gem5's m5out. Allows you to do multiple runs, and then inspect separate outputs later in different output directories. Default: %(default)s +''' + ) + parser.add_argument( + '-P', '--prebuilt', default=False, action='store_true', + help='''\ +Use prebuilt packaged host utilities as much as possible instead +of the ones we built ourselves. Saves build time, but decreases +the likelihood of compatibility. ''' ) parser.add_argument( @@ -190,7 +216,10 @@ def get_stats(stat_re=None, stats_file=None): def get_toolchain_tool(tool): global this - return glob.glob(os.path.join(this.host_bin_dir, '*-buildroot-*-{}'.format(tool)))[0] + if this.baremetal is None: + return glob.glob(os.path.join(this.host_bin_dir, '*-buildroot-*-{}'.format(tool)))[0] + else: + return os.path.join(this.crosstool_ng_bin_dir, '{}-{}'.format(this.crosstool_ng_prefix, tool)) def github_make_request( authenticate=False, @@ -228,6 +257,14 @@ def github_make_request( def log_error(msg): print('error: {}'.format(msg), file=sys.stderr) +def mkdir(): + global this + os.makedirs(this.build_dir, exist_ok=True) + os.makedirs(this.gem5_build_dir, exist_ok=True) + os.makedirs(this.gem5_run_dir, exist_ok=True) + os.makedirs(this.qemu_run_dir, exist_ok=True) + os.makedirs(this.p9_dir, exist_ok=True) + def print_cmd(cmd, cmd_file=None, extra_env=None): ''' Format a command given as a list of strings so that it can @@ -284,6 +321,10 @@ def raw_to_qcow2(prebuilt=False, reverse=False): outfile, ]) == 0 +def raise_no_x86(arch): + if (arch == 'x86_64'): + raise Exception('x86_64 not yet supported') + def resolve_args(defaults, args, extra_args): if extra_args is None: extra_args = {} @@ -362,6 +403,7 @@ def run_cmd( # https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802 with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc: if out_file is not None: + os.makedirs(os.path.split(os.path.abspath(out_file))[0], exist_ok=True) with open(out_file, 'bw') as logfile: while True: byte = proc.stdout.read(1) @@ -385,15 +427,41 @@ def setup(parser): args = parser.parse_args() if args.arch in this.arch_map: args.arch = this.arch_map[args.arch] + this.machine = args.machine if args.arch == 'arm': this.armv = 7 this.gem5_arch = 'ARM' + this.mcpu = 'cortex-a15' + this.crosstool_ng_prefix = 'arm-unknown-eabi' + if args.gem5: + if this.machine is None: + this.machine = 'VExpress_GEM5_V1' + else: + if this.machine is None: + this.machine = 'virt' elif args.arch == 'aarch64': this.armv = 8 this.gem5_arch = 'ARM' + this.mcpu = 'cortex-a57' + this.crosstool_ng_prefix = 'aarch64-unknown-elf' + if args.gem5: + if this.machine is None: + this.machine = 'VExpress_GEM5_V1' + else: + if this.machine is None: + this.machine = 'virt' elif args.arch == 'x86_64': + this.crosstool_ng_prefix = 'TODO' this.gem5_arch = 'X86' - this.buildroot_build_dir = os.path.join(this.out_dir, 'buildroot', args.arch, args.buildroot_build_id) + if args.gem5: + if this.machine is None: + this.machine = 'TODO' + else: + if this.machine is None: + this.machine = 'pc' + this.buildroot_out_dir = os.path.join(this.out_dir, 'buildroot') + this.buildroot_build_dir = os.path.join(this.buildroot_out_dir, 'build', args.buildroot_build_id, args.arch) + this.buildroot_download_dir = os.path.join(this.buildroot_out_dir, 'download') this.buildroot_config_file = os.path.join(this.buildroot_build_dir, '.config') this.build_dir = os.path.join(this.buildroot_build_dir, 'build') this.linux_build_dir = os.path.join(this.build_dir, 'linux-custom') @@ -407,8 +475,8 @@ def setup(parser): this.qemu_guest_build_dir = os.path.join(this.build_dir, 'qemu-custom') this.host_dir = os.path.join(this.buildroot_build_dir, 'host') this.host_bin_dir = os.path.join(this.host_dir, 'usr', 'bin') - this.images_dir = os.path.join(this.buildroot_build_dir, 'images') - this.rootfs_raw_file = os.path.join(this.images_dir, 'rootfs.ext2') + this.buildroot_images_dir = os.path.join(this.buildroot_build_dir, 'images') + this.rootfs_raw_file = os.path.join(this.buildroot_images_dir, 'rootfs.ext2') this.qcow2_file = this.rootfs_raw_file + '.qcow2' this.staging_dir = os.path.join(this.buildroot_build_dir, 'staging') this.target_dir = os.path.join(this.buildroot_build_dir, 'target') @@ -425,11 +493,23 @@ def setup(parser): this.qemu_trace_txt_file = os.path.join(this.qemu_run_dir, 'trace.txt') this.qemu_termout_file = os.path.join(this.qemu_run_dir, 'termout.txt') this.qemu_rrfile = os.path.join(this.qemu_run_dir, 'rrfile') - this.gem5_build_dir = os.path.join(this.out_dir, 'gem5', args.gem5_build_id) + this.gem5_out_dir = os.path.join(this.out_dir, 'gem5') + this.gem5_build_dir = os.path.join(this.gem5_out_dir, args.gem5_build_id) + this.gem5_fake_iso = os.path.join(this.gem5_out_dir, 'fake.iso') this.gem5_m5term = os.path.join(this.gem5_build_dir, 'm5term') this.gem5_build_build_dir = os.path.join(this.gem5_build_dir, 'build') this.gem5_executable = os.path.join(this.gem5_build_build_dir, gem5_arch, 'gem5.{}'.format(args.gem5_build_type)) this.gem5_system_dir = os.path.join(this.gem5_build_dir, 'system') + this.crosstool_ng_out_dir = os.path.join(this.out_dir, 'crosstool-ng') + this.crosstool_ng_buildid_dir = os.path.join(this.crosstool_ng_out_dir, 'build', args.crosstool_ng_build_id) + this.crosstool_ng_install_dir = os.path.join(this.crosstool_ng_buildid_dir, 'install', args.arch) + this.crosstool_ng_bin_dir = os.path.join(this.crosstool_ng_install_dir, 'bin') + this.crosstool_ng_util_dir = os.path.join(this.crosstool_ng_buildid_dir, 'util') + this.crosstool_ng_config = os.path.join(this.crosstool_ng_util_dir, '.config') + this.crosstool_ng_defconfig = os.path.join(this.crosstool_ng_util_dir, 'defconfig') + this.crosstool_ng_executable = os.path.join(this.crosstool_ng_util_dir, 'ct-ng') + this.crosstool_ng_build_dir = os.path.join(this.crosstool_ng_buildid_dir, 'build') + this.crosstool_ng_download_dir = os.path.join(this.crosstool_ng_out_dir, 'download') if args.gem5_worktree is not None: this.gem5_src_dir = os.path.join(this.gem5_non_default_src_root_dir, args.gem5_worktree) else: @@ -470,12 +550,63 @@ def setup(parser): this.qemu_hostfwd_ssh_port = this.qemu_base_port + 2 this.qemu_gdb_port = this.qemu_base_port + 3 this.gdb_port = this.qemu_gdb_port + + # Baremetal. + this.baremetal = args.baremetal + this.baremetal_lib_basename = 'lib' + this.baremetal_src_dir = os.path.join(this.root_dir, 'baremetal') + this.baremetal_src_lib_dir = os.path.join(this.baremetal_src_dir, this.baremetal_lib_basename) + this.c_ext = '.c' + this.asm_ext = '.S' + this.obj_ext = '.o' + if args.gem5: + this.simulator_name = 'gem5' + else: + this.simulator_name = 'qemu' + this.baremetal_out_dir = os.path.join(out_dir, 'baremetal', args.arch, this.simulator_name, this.machine) + this.baremetal_out_lib_dir = os.path.join(this.baremetal_out_dir, this.baremetal_lib_basename) + this.baremetal_out_ext = '.elf' + + # Image. + if args.baremetal is None: + if args.gem5: + this.image = this.vmlinux + this.disk_image = this.rootfs_raw_file + else: + this.image = this.linux_image + this.disk_image = this.qcow2_file + else: + this.disk_image = this.gem5_fake_iso + paths = [ + os.path.join(this.baremetal_out_dir, this.baremetal), + os.path.join( + this.baremetal_out_dir, + os.path.relpath(this.baremetal, this.baremetal_src_dir), + ) + ] + paths[:] = [os.path.splitext(path)[0] + this.baremetal_out_ext for path in paths] + found = False + for path in paths: + if os.path.exists(path): + found = True + break + if not found and this.baremetal != 'all': + raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths)) + this.image = path return args -def mkdir(): - global this - os.makedirs(this.build_dir, exist_ok=True) - os.makedirs(this.gem5_build_dir, exist_ok=True) - os.makedirs(this.gem5_run_dir, exist_ok=True) - os.makedirs(this.qemu_run_dir, exist_ok=True) - os.makedirs(this.p9_dir, exist_ok=True) +def write_configs(config_path, configs, config_fragments=None): + """ + Write extra configs into the Buildroot config file. + TODO Can't get rid of these for now with nice fragments: + http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config + """ + if config_fragments is None: + config_fragments = [] + with open(config_path, 'a') as config_file: + for config_fragment in config_fragments: + with open(config_fragment, 'r') as config_fragment: + for line in config_fragment: + config_file.write(line) + for config in configs: + config_file.write(config + '\n') diff --git a/configure b/configure index 14e55fa3..465f5e8a 100755 --- a/configure +++ b/configure @@ -1,29 +1,51 @@ #!/usr/bin/env bash set -eu +apt_get=true +baremetal=false +baremetal_given=false +buildroot=true +buildroot_given=false +linux=true +linux_given=false interactive_pkgs=libsdl2-dev gem5=false gem5_given=false qemu=true qemu_given=false submodules_dir=submodules -submodules=buildroot +submodules= y= while [ $# -gt 0 ]; do case "$1" in + --baremetal) + baremetal=true + baremetal_given=true + shift + ;; + --buildroot) + buildroot_given=true + shift + ;; --gem5) gem5_given=true shift ;; --parsec-benchmark) submodules="${submodules} parsec-benchmark" + shift ;; --qemu) qemu_given=true shift ;; + --no-apt-get) + apt_get=false + shift + ;; --travis) interactive_pkgs= y=-y + shift ;; *) echo 'unknown option' 1>&2 @@ -37,84 +59,93 @@ fi if "$gem5_given"; then gem5=true fi - -## apt-get - -pkgs="\ -automake \ -bc \ -build-essential \ -coreutils \ -cpio \ -expect \ -git \ -moreutils \ -rsync \ -tmux \ -unzip \ -vinagre \ -wget \ -" -if "$gem5"; then - pkgs="$pkgs \ -ccache \ -gcc-aarch64-linux-gnu \ -gcc-arm-linux-gnueabi \ -libgoogle-perftools-dev \ -protobuf-compiler \ -python-dev \ -python-pip \ -scons \ -" +if "$baremetal_given" && ! "$buildroot_given"; then + buildroot=false fi -command -v apt-get >/dev/null 2>&1 || { - cat </dev/null 2>&1 || { + cat </dev/null && pwd)" @@ -10,7 +10,7 @@ cmd="./run -a '$arch' --gem5 --eval-busybox '/gem5.sh'" # These cache sizes roughly match the ARM Cortex A75 # https://en.wikipedia.org/wiki/ARM_Cortex-A75 -restore='-l 1 -- --cpu-type=HPI --restore-with-cpu=HPI --caches --l2cache --l1d_size=128kB --l1i_size=1024kB --l2_size=256kB' +restore='-l 1 -- --cpu-type=HPI --restore-with-cpu=HPI --caches --l2cache --l1d_size=64kB --l1i_size=64kB --l2_size=256kB' # Generate a checkpoint after Linux boots, using the faster and less detailed CPU. # The boot takes a while, be patient young Padawan. diff --git a/getvar b/getvar index 12c535e4..431d4f99 100755 --- a/getvar +++ b/getvar @@ -1,5 +1,9 @@ #!/usr/bin/env python3 + +import types + import common + parser = common.get_argparse(argparse_args={ 'description': '''Print the value of a common.py variable. @@ -14,8 +18,21 @@ For example, to get the Buildroot output directory for an ARM build, use: ./%(prog)s -a arm buildroot_build_dir .... +List all available variables: + +.... +./%(prog)s +.... +.... ''' }) -parser.add_argument('variable') +parser.add_argument('variable', nargs='?') args = common.setup(parser) -print(getattr(common, args.variable)) +if args.variable: + print(getattr(common, args.variable)) +else: + for attr in dir(common): + if not attr.startswith('__'): + val = getattr(common, attr) + if not callable(val) and not type(val) is types.ModuleType: + print('{} {}'.format(attr, val)) diff --git a/packages/kernel_modules/user/common.h b/packages/kernel_modules/user/common.h index bf39fb5e..19b2965e 100644 --- a/packages/kernel_modules/user/common.h +++ b/packages/kernel_modules/user/common.h @@ -27,7 +27,7 @@ typedef struct { * @param[out] entry the parsed entry * @param[in] pagemap_fd file descriptor to an open /proc/pid/pagemap file * @param[in] vaddr virtual address to get entry for - * @return 0 for success, 1 for failure + * @return 0 for success, 1 for failure */ int pagemap_get_entry(PagemapEntry *entry, int pagemap_fd, uintptr_t vaddr) { @@ -60,8 +60,8 @@ int pagemap_get_entry(PagemapEntry *entry, int pagemap_fd, uintptr_t vaddr) * * @param[out] paddr physical address * @param[in] pid process to convert for - * @param[in] vaddr virtual address to get entry for - * @return 0 for success, 1 for failure + * @param[in] vaddr virtual address to get entry for + * @return 0 for success, 1 for failure */ int virt_to_phys_user(uintptr_t *paddr, pid_t pid, uintptr_t vaddr) { diff --git a/patches/manual/gem5-aarch64-baremetal.patch b/patches/manual/gem5-aarch64-baremetal.patch new file mode 100644 index 00000000..71c485da --- /dev/null +++ b/patches/manual/gem5-aarch64-baremetal.patch @@ -0,0 +1,12 @@ +diff --git a/configs/example/fs.py b/configs/example/fs.py +index 3997ed76c..c3259825d 100644 +--- a/configs/example/fs.py ++++ b/configs/example/fs.py +@@ -376,5 +376,7 @@ if buildEnv['TARGET_ISA'] == "arm" and options.generate_dtb: + sys = getattr(root, sysname) + sys.dtb_filename = create_dtb_for_system(sys, '%s.dtb' % sysname) + ++test_sys.highest_el_is_64 = True ++test_sys.auto_reset_addr_64 = True + Simulation.setWorkCountOptions(test_sys, options) + Simulation.run(options, root, test_sys, FutureClass) diff --git a/patches/manual/gem5-wait-gdb.patch b/patches/manual/gem5-wait-gdb.patch new file mode 100644 index 00000000..3c77b573 --- /dev/null +++ b/patches/manual/gem5-wait-gdb.patch @@ -0,0 +1,11 @@ +diff --git a/configs/example/fs.py b/configs/example/fs.py +index 3997ed76c..b4267ebc0 100644 +--- a/configs/example/fs.py ++++ b/configs/example/fs.py +@@ -376,5 +376,6 @@ if buildEnv['TARGET_ISA'] == "arm" and options.generate_dtb: + sys = getattr(root, sysname) + sys.dtb_filename = create_dtb_for_system(sys, '%s.dtb' % sysname) + ++test_sys.cpu[0].wait_for_remote_gdb = True + Simulation.setWorkCountOptions(test_sys, options) + Simulation.run(options, root, test_sys, FutureClass) diff --git a/qemumonitor b/qemu-monitor similarity index 100% rename from qemumonitor rename to qemu-monitor diff --git a/qemurr b/qemu-rr similarity index 100% rename from qemurr rename to qemu-rr diff --git a/rootfs_post_build_script b/rootfs-post-build-script similarity index 100% rename from rootfs_post_build_script rename to rootfs-post-build-script diff --git a/run b/run index c9cc0f68..c3d4628a 100755 --- a/run +++ b/run @@ -29,7 +29,6 @@ defaults = { 'kgdb': False, 'kvm': False, 'memory': '256M', - 'prebuilt': False, 'record': False, 'replay': False, 'terminal': False, @@ -98,17 +97,27 @@ def main(args, extra_args=None): def raise_rootfs_not_found(): raise Exception('Root filesystem not found. Did you build it?\n' \ - 'Tried to use: ' + common.rootfs_raw_file) + 'Tried to use: ' + common.disk_image) + def raise_image_not_found(): + raise Exception('Executable image not found. Did you build it?\n' \ + 'Tried to use: ' + common.image) + if common.image is None: + raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths)) if args.gem5: - if not os.path.exists(common.rootfs_raw_file): - if not os.path.exists(common.qcow2_file): - raise_rootfs_not_found() - common.raw_to_qcow2(prebuilt=args.prebuilt, reverse=True) - if not os.path.exists(common.vmlinux): + if args.baremetal is None: + if not os.path.exists(common.rootfs_raw_file): + if not os.path.exists(common.qcow2_file): + raise_rootfs_not_found() + common.raw_to_qcow2(prebuilt=args.prebuilt, reverse=True) + else: + if not os.path.exists(common.gem5_fake_iso): + os.makedirs(os.path.dirname(common.gem5_fake_iso), exist_ok=True) + with open(common.gem5_fake_iso, 'w') as f: + f.write('a' * 512) + if not os.path.exists(common.image): # This is to run gem5 from a prebuilt download. - if not os.path.exists(common.linux_image): - raise Exception('Linux kernel image not found. Did you compile it?\n' \ - 'Tried: ' + common.vmlinux) + if (not args.baremetal is None) or (not os.path.exists(common.linux_image)): + raise_image_not_found() assert common.run_cmd([os.path.join(common.extract_vmlinux, common.linux_image)]) == 0 os.makedirs(os.path.dirname(common.gem5_readfile), exist_ok=True) with open(common.gem5_readfile, 'w') as readfile: @@ -139,9 +148,9 @@ def main(args, extra_args=None): os.path.join(common.gem5_src_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), '--big-cpus', '2', '--cpu-type', 'atomic', - '--disk', common.rootfs_raw_file, + '--disk', common.disk_image, '--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'), - '--kernel', common.vmlinux, + '--kernel', common.image, '--little-cpus', '2' ] else: @@ -152,8 +161,8 @@ def main(args, extra_args=None): extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)]) cmd += [ common.gem5_fs_file, - '--disk-image', common.rootfs_raw_file, - '--kernel', common.vmlinux, + '--disk-image', common.disk_image, + '--kernel', common.image, '--mem-size', memory, '--num-cpus', str(args.cpus), '--script', common.gem5_readfile, @@ -168,9 +177,13 @@ def main(args, extra_args=None): cmd += [ '--command-line', 'earlyprintk=pl011,0x1c090000 console=ttyAMA0 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli), '--dtb-file', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(common.armv, args.cpus)), - '--machine-type', 'VExpress_GEM5_V1', + '--machine-type', common.machine, ] + if not args.baremetal is None: + cmd.append('--bare-metal') else: + if not os.path.exists(common.image): + raise_image_not_found() extra_emulator_args.extend(extra_qemu_args) os.makedirs(common.run_dir, exist_ok=True) if args.prebuilt: @@ -183,10 +196,6 @@ def main(args, extra_args=None): if not qemu_found: raise Exception('QEMU executable not found, did you forget to build or install it?\n' \ 'Tried to use: ' + qemu_executable) - if not os.path.exists(common.qcow2_file): - if not os.path.exists(common.rootfs_raw_file): - raise_rootfs_not_found() - common.raw_to_qcow2(prebuilt=args.prebuilt) if args.debug_vm: serial_monitor = [] else: @@ -201,7 +210,7 @@ def main(args, extra_args=None): qemu_executable, '-device', 'rtl8139,netdev=net0', '-gdb', 'tcp::{}'.format(common.gdb_port), - '-kernel', common.linux_image, + '-kernel', common.image, '-m', args.memory, '-monitor', 'telnet::{},server,nowait'.format(common.qemu_monitor_port), '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(common.qemu_hostfwd_generic_port, common.qemu_hostfwd_generic_port, common.qemu_hostfwd_ssh_port), @@ -215,7 +224,7 @@ def main(args, extra_args=None): vnc ) if args.initrd: - extra_emulator_args.extend(['-initrd', os.path.join(common.images_dir, 'rootfs.cpio')]) + extra_emulator_args.extend(['-initrd', os.path.join(common.buildroot_images_dir, 'rootfs.cpio')]) rr = args.record or args.replay if ramfs: # TODO why is this needed, and why any string works. @@ -231,15 +240,20 @@ def main(args, extra_args=None): root = 'root=/dev/vda' rrid = '' snapshot = ',snapshot' - extra_emulator_args.extend([ - '-drive', - 'file={},format=qcow2,if={}{}{}'.format(common.qcow2_file, driveif, snapshot, rrid) - ]) - if rr: + if args.baremetal is None: + if not os.path.exists(common.qcow2_file): + if not os.path.exists(common.rootfs_raw_file): + raise_rootfs_not_found() + common.raw_to_qcow2(prebuilt=args.prebuilt) extra_emulator_args.extend([ - '-drive', 'driver=blkreplay,if=none,image=img-direct,id=img-blkreplay', - '-device', 'ide-hd,drive=img-blkreplay' - ]) + '-drive', + 'file={},format=qcow2,if={}{}{}'.format(common.disk_image, driveif, snapshot, rrid) + ]) + if rr: + extra_emulator_args.extend([ + '-drive', 'driver=blkreplay,if=none,image=img-direct,id=img-blkreplay', + '-device', 'ide-hd,drive=img-blkreplay' + ]) if rr: extra_emulator_args.extend([ '-object', 'filter-replay,id=replay,netdev=net0', @@ -251,29 +265,32 @@ def main(args, extra_args=None): if args.arch == 'x86_64': if args.kgdb: kernel_cli += ' kgdboc=ttyS0,115200' + append = ['-append', '{} nopat {}'.format(root, kernel_cli)] cmd.extend([ - '-M', 'pc', - '-append', '{} nopat {}'.format(root, kernel_cli), + '-M', common.machine, '-device', 'edu', ]) elif args.arch == 'arm' or args.arch == 'aarch64': + extra_qemu_args.append('-semihosting') if args.kgdb: kernel_cli += ' kgdboc=ttyAMA0,115200' if args.arch == 'arm': cpu = 'cortex-a15' else: cpu = 'cortex-a57' - # highmem=off needed since v3.0.0 due to: - # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html + append = ['-append', '{} {}'.format(root, kernel_cli)] cmd = ( cmd + [ - '-M', 'virt,highmem=off', - '-append', '{} {}'.format(root, kernel_cli), + # highmem=off needed since v3.0.0 due to: + # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html + '-M', '{},highmem=off'.format(common.machine), '-cpu', cpu, ] + virtio_gpu_pci ) + if args.baremetal is None: + cmd.extend(append) if args.tmux: if args.gem5: subprocess.Popen([os.path.join(common.root_dir, 'tmu'), @@ -286,7 +303,7 @@ def main(args, extra_args=None): # but it cannot be used as a library properly it seems, and it is # slower than tmux. subprocess.Popen([os.path.join(common.root_dir, 'tmu'), - "sleep 2;./rungdb -a '{}' -L '{}' -n '{}' {}" \ + "sleep 2;./run-gdb -a '{}' -L '{}' -n '{}' {}" \ .format(args.arch, args.linux_build_id, args.run_id, args.tmux_args) ]) cmd.extend(extra_emulator_args) @@ -414,10 +431,6 @@ some arch to fail to boot. Default: %(default)s ''' ) - parser.add_argument( - '-P', '--prebuilt', default=defaults['prebuilt'], action='store_true', - help='Run the downloaded prebuilt images with pre-packaged host tools.' - ) group = parser.add_mutually_exclusive_group() group.add_argument( '-R', '--replay', default=defaults['replay'], action='store_true', diff --git a/rungdb b/run-gdb similarity index 78% rename from rungdb rename to run-gdb index 4d52b46e..e80a5c25 100755 --- a/rungdb +++ b/run-gdb @@ -11,6 +11,7 @@ import common defaults = { 'after': '', 'before': '', + 'sim': False, 'no_continue': False, 'kgdb': False, 'no_lxsymbols': False, @@ -33,7 +34,7 @@ def main(args, extra_args=None): args = common.resolve_args(defaults, args, extra_args) after = shlex.split(args.after) before = shlex.split(args.before) - if args.no_lxsymbols: + if args.no_lxsymbols or args.baremetal is not None: lx_symbols = [] else: lx_symbols = ['-ex', 'lx-symbols ../kernel_modules-1.0/'] @@ -41,19 +42,25 @@ def main(args, extra_args=None): break_at = ['-ex', 'break {}'.format(args.break_at)] else: break_at = [] + if args.baremetal is None: + image = common.vmlinux + else: + image = common.image cmd = ( - [ - os.path.join(common.host_bin_dir, - '{}-linux-gdb'.format(args.arch)) - ] + + [common.get_toolchain_tool('gdb')] + before + - [ - '-q', - '-ex', 'add-auto-load-safe-path {}'.format(common.linux_variant_dir), - '-ex', 'file {}'.format(common.vmlinux), - '-ex', 'target remote localhost:{}'.format(common.gdb_port), - ] + ['-q'] ) + if args.baremetal is None: + cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(common.linux_variant_dir)]) + if args.sim: + target = 'sim' + else: + target = 'remote localhost:{}'.format(common.gdb_port) + cmd.extend([ + '-ex', 'file {}'.format(image), + '-ex', 'target {}'.format(target), + ]) if not args.kgdb: cmd.extend(break_at) if not args.no_continue: @@ -78,7 +85,7 @@ def main(args, extra_args=None): # which gets put on the kernel build root when python debugging scripts are enabled. cmd.extend(['-ex', 'continue'] + lx_symbols) cmd.extend(after) - return common.run_cmd(cmd, cmd_file=os.path.join(common.run_dir, 'rungdb.sh'), cwd=common.linux_variant_dir) + return common.run_cmd(cmd, cmd_file=os.path.join(common.run_dir, 'run-gdb.sh'), cwd=common.linux_variant_dir) if __name__ == '__main__': parser = common.get_argparse(argparse_args={'description': 'Connect with GDB to an emulator to debug Linux itself'}) @@ -97,6 +104,12 @@ if __name__ == '__main__': parser.add_argument( '-k', '--kgdb', default=defaults['kgdb'], action='store_true' ) + parser.add_argument( + '--sim', default=defaults['sim'], action='store_true', + help='''Use the built-in GDB CPU simulator +See: https://github.com/cirosantilli/linux-kernel-module-cheat#gdb-builtin-cpu-simulator +''' + ) parser.add_argument( '-X', '--no-lxsymbols', default=defaults['no_lxsymbols'], action='store_true' ) diff --git a/rungdb-user b/run-gdb-user similarity index 98% rename from rungdb-user rename to run-gdb-user index a97c7515..63c10848 100755 --- a/rungdb-user +++ b/run-gdb-user @@ -5,7 +5,7 @@ import os import sys import common -rungdb = imp.load_source('rungdb', os.path.join(common.root_dir, 'rungdb')) +rungdb = imp.load_source('rungdb', os.path.join(common.root_dir, 'run-gdb')) parser = common.get_argparse(argparse_args={ 'description': '''GDB step debug guest userland processes without gdbserver. diff --git a/rungdbserver b/run-gdbserver similarity index 100% rename from rungdbserver rename to run-gdbserver diff --git a/run-toolchain b/run-toolchain index 62e2cdb3..6a5b29c3 100755 --- a/run-toolchain +++ b/run-toolchain @@ -42,6 +42,4 @@ else: sys.exit(common.run_cmd( [common.get_toolchain_tool(args.tool)] + args.extra_args, cmd_file=os.path.join(common.run_dir, 'run-toolchain.sh'), - cwd=common.linux_variant_dir, - show_cmd=False, )) diff --git a/submodules/crosstool-ng b/submodules/crosstool-ng new file mode 160000 index 00000000..d5900deb --- /dev/null +++ b/submodules/crosstool-ng @@ -0,0 +1 @@ +Subproject commit d5900debd397b8909d9cafeb9a1093fb7a5dc6e6 diff --git a/submodules/gem5 b/submodules/gem5 index 200281b0..7bfb7f3a 160000 --- a/submodules/gem5 +++ b/submodules/gem5 @@ -1 +1 @@ -Subproject commit 200281b08ca21f0d2678e23063f088960d3c0819 +Subproject commit 7bfb7f3a43f382eb49853f47b140bfd6caad0fb8