From bb3650fcd85e1238b2ee02dee9430c9b44b67689 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 16 Mar 2023 19:00:01 +0100 Subject: [PATCH 1/4] test: add integration test for multiboot2 and multiboot2-header --- .editorconfig | 2 +- .github/workflows/rust.yml | 12 + Cargo.toml | 3 + integration-test/.envrc | 1 + integration-test/.run.sh | 35 +++ integration-test/README.md | 17 ++ integration-test/bins/.cargo/config.toml | 11 + integration-test/bins/Cargo.lock | 285 ++++++++++++++++++ integration-test/bins/Cargo.toml | 14 + integration-test/bins/README.md | 4 + .../bins/multiboot2_chainloader/Cargo.toml | 16 + .../bins/multiboot2_chainloader/build.rs | 5 + .../bins/multiboot2_chainloader/link.ld | 40 +++ .../bins/multiboot2_chainloader/src/loader.rs | 97 ++++++ .../bins/multiboot2_chainloader/src/main.rs | 28 ++ .../multiboot2_chainloader/src/multiboot.rs | 41 +++ .../bins/multiboot2_chainloader/src/start.S | 63 ++++ .../bins/multiboot2_payload/Cargo.toml | 14 + .../bins/multiboot2_payload/build.rs | 5 + .../bins/multiboot2_payload/link.ld | 40 +++ .../bins/multiboot2_payload/src/main.rs | 37 +++ .../src/multiboot2_header.S | 66 ++++ .../bins/multiboot2_payload/src/start.S | 52 ++++ .../src/verify/chainloader.rs | 31 ++ .../multiboot2_payload/src/verify/grub.rs | 30 ++ .../bins/multiboot2_payload/src/verify/mod.rs | 106 +++++++ integration-test/bins/rust-toolchain.toml | 6 + integration-test/bins/util/Cargo.toml | 12 + integration-test/bins/util/src/allocator.rs | 17 ++ integration-test/bins/util/src/debugcon.rs | 68 +++++ integration-test/bins/util/src/lib.rs | 41 +++ integration-test/bins/util/src/macros.rs | 10 + integration-test/bins/x86-unknown-none.json | 15 + integration-test/nix/sources.json | 14 + integration-test/nix/sources.nix | 198 ++++++++++++ integration-test/run.sh | 7 + integration-test/shell.nix | 12 + integration-test/tests/README.md | 4 + .../tests/multiboot2-header/README.md | 7 + .../tests/multiboot2-header/run_qemu.sh | 41 +++ integration-test/tests/multiboot2/.gitignore | 2 + integration-test/tests/multiboot2/README.md | 5 + .../tests/multiboot2/build_img.sh | 28 ++ integration-test/tests/multiboot2/grub.cfg | 13 + integration-test/tests/multiboot2/run_qemu.sh | 40 +++ 45 files changed, 1594 insertions(+), 1 deletion(-) create mode 100644 integration-test/.envrc create mode 100755 integration-test/.run.sh create mode 100644 integration-test/README.md create mode 100644 integration-test/bins/.cargo/config.toml create mode 100644 integration-test/bins/Cargo.lock create mode 100644 integration-test/bins/Cargo.toml create mode 100644 integration-test/bins/README.md create mode 100644 integration-test/bins/multiboot2_chainloader/Cargo.toml create mode 100644 integration-test/bins/multiboot2_chainloader/build.rs create mode 100644 integration-test/bins/multiboot2_chainloader/link.ld create mode 100644 integration-test/bins/multiboot2_chainloader/src/loader.rs create mode 100644 integration-test/bins/multiboot2_chainloader/src/main.rs create mode 100644 integration-test/bins/multiboot2_chainloader/src/multiboot.rs create mode 100644 integration-test/bins/multiboot2_chainloader/src/start.S create mode 100644 integration-test/bins/multiboot2_payload/Cargo.toml create mode 100644 integration-test/bins/multiboot2_payload/build.rs create mode 100644 integration-test/bins/multiboot2_payload/link.ld create mode 100644 integration-test/bins/multiboot2_payload/src/main.rs create mode 100644 integration-test/bins/multiboot2_payload/src/multiboot2_header.S create mode 100644 integration-test/bins/multiboot2_payload/src/start.S create mode 100644 integration-test/bins/multiboot2_payload/src/verify/chainloader.rs create mode 100644 integration-test/bins/multiboot2_payload/src/verify/grub.rs create mode 100644 integration-test/bins/multiboot2_payload/src/verify/mod.rs create mode 100644 integration-test/bins/rust-toolchain.toml create mode 100644 integration-test/bins/util/Cargo.toml create mode 100644 integration-test/bins/util/src/allocator.rs create mode 100644 integration-test/bins/util/src/debugcon.rs create mode 100644 integration-test/bins/util/src/lib.rs create mode 100644 integration-test/bins/util/src/macros.rs create mode 100644 integration-test/bins/x86-unknown-none.json create mode 100644 integration-test/nix/sources.json create mode 100644 integration-test/nix/sources.nix create mode 100755 integration-test/run.sh create mode 100644 integration-test/shell.nix create mode 100644 integration-test/tests/README.md create mode 100644 integration-test/tests/multiboot2-header/README.md create mode 100755 integration-test/tests/multiboot2-header/run_qemu.sh create mode 100644 integration-test/tests/multiboot2/.gitignore create mode 100644 integration-test/tests/multiboot2/README.md create mode 100755 integration-test/tests/multiboot2/build_img.sh create mode 100644 integration-test/tests/multiboot2/grub.cfg create mode 100755 integration-test/tests/multiboot2/run_qemu.sh diff --git a/.editorconfig b/.editorconfig index 2537322d..f5f22af5 100644 --- a/.editorconfig +++ b/.editorconfig @@ -11,5 +11,5 @@ indent_size = 4 trim_trailing_whitespace = true max_line_length = 80 -[*.yml] +[{*.nix, *.yml}] indent_size = 2 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index bcf41503..d53a0fc0 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -127,3 +127,15 @@ jobs: do-style-check: true do-test: false features: builder,unstable + + integrationtest: + name: multiboot2 integrationtest + needs: + - build_nightly + - build_nostd_nightly + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@v3 + - uses: cachix/install-nix-action@v20 + - run: integration-test/run.sh diff --git a/Cargo.toml b/Cargo.toml index b88c4703..3b6f2682 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,9 @@ members = [ "multiboot2", "multiboot2-header", ] +exclude = [ + "integration-test" +] [workspace.dependencies] bitflags = "2" diff --git a/integration-test/.envrc b/integration-test/.envrc new file mode 100644 index 00000000..1d953f4b --- /dev/null +++ b/integration-test/.envrc @@ -0,0 +1 @@ +use nix diff --git a/integration-test/.run.sh b/integration-test/.run.sh new file mode 100755 index 00000000..318557b5 --- /dev/null +++ b/integration-test/.run.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +DIR=$(dirname "$(realpath "$0")") +cd "$DIR" || exit + +function fn_main() { + fn_build_rust_bins + fn_multiboot2_integrationtest + fn_multiboot2_header_integrationtest +} + +function fn_build_rust_bins() { + cd "bins" + cargo build --release + cd "$DIR" +} + +function fn_multiboot2_integrationtest() { + cd tests/multiboot2 + ./build_img.sh + ./run_qemu.sh + cd "$DIR" +} + +function fn_multiboot2_header_integrationtest() { + cd tests/multiboot2-header + ./run_qemu.sh + cd "$DIR" +} + +fn_main diff --git a/integration-test/README.md b/integration-test/README.md new file mode 100644 index 00000000..e7e82479 --- /dev/null +++ b/integration-test/README.md @@ -0,0 +1,17 @@ +# Integrationtests + +This directory contains integration tests for the `multiboot2` and the +`multiboot2-header` crate. The integration tests start a QEMU VM and do certain +checks at runtime. If something fails, they instruct QEMU to exit with an error +code. All output of the VM is printed to the screen. If + +The `bins` directory contains binaries that **are** the tests. The `tests` +directory contains test definitions, run scripts, and other relevant files. The +main entry to run all tests is `./run.sh` in this directory. + +## TL;DR: +- `$ ./run.sh` to execute the integration tests + +## Prerequisites +The tests rely on [`nix`](https://nixos.org/) being installed / `nix-shell` +being available to get the relevant tools. diff --git a/integration-test/bins/.cargo/config.toml b/integration-test/bins/.cargo/config.toml new file mode 100644 index 00000000..b02f2b2e --- /dev/null +++ b/integration-test/bins/.cargo/config.toml @@ -0,0 +1,11 @@ +[unstable] +build-std = ["core", "compiler_builtins", "alloc"] +build-std-features = ["compiler-builtins-mem"] + +[build] +target = "x86-unknown-none.json" +rustflags = [ + "-C", "code-model=kernel", + # "-C", "link-arg=-Tlink.ld", + "-C", "relocation-model=static", +] diff --git a/integration-test/bins/Cargo.lock b/integration-test/bins/Cargo.lock new file mode 100644 index 00000000..2e7b98a6 --- /dev/null +++ b/integration-test/bins/Cargo.lock @@ -0,0 +1,285 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bit_field" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbe3c979c178231552ecba20214a8272df4e09f232a87aef4320cf06539aded" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + +[[package]] +name = "elf_rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf723f70efb0373c0b2501d943cf20ac1adbbd8e7c8eef926b2be545e5a33e8" +dependencies = [ + "bitflags 1.3.2", + "num-traits", +] + +[[package]] +name = "good_memory_allocator" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1651659e016ea4259760966432aebcc96c81e26743fb018c59585ddd677127e" +dependencies = [ + "either", + "spin", +] + +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "multiboot" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87ad3b7b7bcf5da525c22221e3eb3a020cd68b2d55ae62f629c15e8bc3bd56e" +dependencies = [ + "paste", +] + +[[package]] +name = "multiboot2" +version = "0.16.0" +dependencies = [ + "bitflags 2.3.2", + "derive_more", + "log", + "ptr_meta", + "uefi-raw", +] + +[[package]] +name = "multiboot2-header" +version = "0.3.0" +dependencies = [ + "derive_more", + "multiboot2", +] + +[[package]] +name = "multiboot2_chainloader" +version = "0.1.0" +dependencies = [ + "anyhow", + "elf_rs", + "good_memory_allocator", + "log", + "multiboot", + "multiboot2", + "multiboot2-header", + "util", +] + +[[package]] +name = "multiboot2_payload" +version = "0.1.0" +dependencies = [ + "anyhow", + "good_memory_allocator", + "log", + "multiboot2", + "util", + "x86", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "paste" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" + +[[package]] +name = "proc-macro2" +version = "1.0.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e472a104799c74b514a57226160104aa483546de37e839ec50e3c2e41dd87534" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "ptr_meta" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcada80daa06c42ed5f48c9a043865edea5dc44cbf9ac009fda3b89526e28607" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bca9224df2e20e7c5548aeb5f110a0f3b77ef05f8585139b7148b59056168ed2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "qemu-exit" +version = "3.0.1" +source = "git+https://github.com/rust-embedded/qemu-exit.git?rev=3cee0efb5c1842b5261850c57b3b4d608542ff03#3cee0efb5c1842b5261850c57b3b4d608542ff03" + +[[package]] +name = "quote" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "spin" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5d6e0250b93c8427a177b849d144a96d5acc57006149479403d7861ab721e34" +dependencies = [ + "lock_api", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "uefi-raw" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62642516099c6441a5f41b0da8486d5fc3515a0603b0fdaea67b31600e22082e" +dependencies = [ + "bitflags 2.3.2", + "ptr_meta", + "uguid", +] + +[[package]] +name = "uguid" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "594cc87e268a7b43d625d46c63cf1605d0e61bf66e4b1cd58c058ec0191e1f81" + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "util" +version = "0.1.0" +dependencies = [ + "good_memory_allocator", + "log", + "qemu-exit", +] + +[[package]] +name = "x86" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2781db97787217ad2a2845c396a5efe286f87467a5810836db6d74926e94a385" +dependencies = [ + "bit_field", + "bitflags 1.3.2", + "raw-cpuid", +] diff --git a/integration-test/bins/Cargo.toml b/integration-test/bins/Cargo.toml new file mode 100644 index 00000000..9c58b106 --- /dev/null +++ b/integration-test/bins/Cargo.toml @@ -0,0 +1,14 @@ +[workspace] +resolver = "2" +members = [ + "multiboot2_chainloader", + "multiboot2_payload", + "util" +] + +[profile.release] +codegen-units = 1 +lto = true + +[patch.crates-io] +multiboot2 = { path = "../../multiboot2" } diff --git a/integration-test/bins/README.md b/integration-test/bins/README.md new file mode 100644 index 00000000..ca74ca0e --- /dev/null +++ b/integration-test/bins/README.md @@ -0,0 +1,4 @@ +# Integrationtest Rust Binaries + +This Cargo workspace contains binaries that are the actual integration tests. +They use the toolchain pinned in `rust-toolchain.toml`. diff --git a/integration-test/bins/multiboot2_chainloader/Cargo.toml b/integration-test/bins/multiboot2_chainloader/Cargo.toml new file mode 100644 index 00000000..42e085cd --- /dev/null +++ b/integration-test/bins/multiboot2_chainloader/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "multiboot2_chainloader" +description = "Multiboot chainloader that loads a Multiboot2 payload" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +util = { path = "../util" } +multiboot2 = { path = "../../../multiboot2" } +multiboot2-header = { path = "../../../multiboot2-header" } +anyhow = { version = "1.0.69", default-features = false } +elf_rs = "0.3.0" +log = { version = "0.4.17", default-features = false } +good_memory_allocator = "0.1.7" +multiboot = "0.8.0" diff --git a/integration-test/bins/multiboot2_chainloader/build.rs b/integration-test/bins/multiboot2_chainloader/build.rs new file mode 100644 index 00000000..cdf8dfbf --- /dev/null +++ b/integration-test/bins/multiboot2_chainloader/build.rs @@ -0,0 +1,5 @@ +fn main() { + let linker_script = "multiboot2_chainloader/link.ld"; + println!("cargo:rerun-if-changed={linker_script}"); + println!("cargo:rustc-link-arg=-T{linker_script}"); +} diff --git a/integration-test/bins/multiboot2_chainloader/link.ld b/integration-test/bins/multiboot2_chainloader/link.ld new file mode 100644 index 00000000..754a44f9 --- /dev/null +++ b/integration-test/bins/multiboot2_chainloader/link.ld @@ -0,0 +1,40 @@ +ENTRY(start) + +PHDRS +{ + /* PT_LOAD FLAGS (page table permissions) not necessary, as we perform + legacy boot. Linker probably sets sensible defaults anyway. */ + kernel_rx PT_LOAD; + kernel_rw PT_LOAD; + kernel_ro PT_LOAD; +} + +SECTIONS { + /* Chainloader linked at 8M, payload at 16M */ + .text 8M : AT(8M) ALIGN(4K) + { + KEEP(*(.multiboot_header)); + *(.text .text.*) + } : kernel_rx + + .rodata : + { + *(.rodata .rodata.*) + } : kernel_ro + + .data : + { + *(.data .data.*) + } : kernel_rw + + .bss : + { + *(COMMON) + *(.bss .bss.*) + } : kernel_rw + + /DISCARD/ : + { + *(.eh_frame .eh_frame*) + } +} diff --git a/integration-test/bins/multiboot2_chainloader/src/loader.rs b/integration-test/bins/multiboot2_chainloader/src/loader.rs new file mode 100644 index 00000000..816f08c1 --- /dev/null +++ b/integration-test/bins/multiboot2_chainloader/src/loader.rs @@ -0,0 +1,97 @@ +use elf_rs::{ElfFile, ProgramHeaderEntry, ProgramType}; +use multiboot2::builder::InformationBuilder; +use multiboot2::{ + BootLoaderNameTag, CommandLineTag, MemoryArea, MemoryAreaType, MemoryMapTag, ModuleTag, +}; + +/// Loads the first module into memory. Assumes that the module is a ELF file. +/// The handoff is performed according to the Multiboot2 spec. +pub fn load_module(mut modules: multiboot::information::ModuleIter) -> ! { + // Load the ELF from the Multiboot1 boot module. + let elf_mod = modules.next().expect("Should have payload"); + let elf_bytes = unsafe { + core::slice::from_raw_parts( + elf_mod.start as *const u64 as *const u8, + (elf_mod.end - elf_mod.start) as usize, + ) + }; + let elf = elf_rs::Elf32::from_bytes(elf_bytes).expect("Should be valid ELF"); + + // Check if a header is present. + { + let hdr = multiboot2_header::Multiboot2Header::find_header(elf_bytes) + .unwrap() + .expect("Should have Multiboot2 header"); + let hdr = + unsafe { multiboot2_header::Multiboot2Header::load(hdr.0.as_ptr().cast()) }.unwrap(); + log::info!("Multiboot2 header:\n{hdr:#?}"); + } + + + // Map the load segments into memory (at their corresponding link). + { + let elf = elf_rs::Elf32::from_bytes(elf_bytes).expect("Should be valid ELF"); + elf.program_header_iter() + .filter(|ph| ph.ph_type() == ProgramType::LOAD) + .for_each(|ph| { + map_memory(ph); + }); + } + + // Currently, the MBI is not enriched with "real" information as requested. + // Subject here is not to write a feature-complete bootloader but to test + // that the basic data structures are usable. + + // build MBI + let mbi = { + let mut mbi_builder: InformationBuilder = multiboot2::builder::InformationBuilder::new(); + mbi_builder.bootloader_name_tag(BootLoaderNameTag::new("mb2_integrationtest_chainloader")); + mbi_builder.command_line_tag(CommandLineTag::new("chainloaded YEAH")); + // random non-sense memory map + mbi_builder.memory_map_tag(MemoryMapTag::new(&[MemoryArea::new( + 0, + 0xffffffff, + MemoryAreaType::Reserved, + )])); + mbi_builder.add_module_tag(ModuleTag::new( + elf_mod.start as u32, + elf_mod.end as u32, + elf_mod.string.unwrap(), + )); + + mbi_builder.build() + }; + + log::info!( + "Handing over to ELF: {}", + elf_mod.string.unwrap_or("") + ); + + // handoff + unsafe { + core::arch::asm!( + "jmp *%ecx", + in("eax") multiboot2::MAGIC, + in("ebx") mbi.as_ptr() as u32, + in("ecx") elf.entry_point() as u32, + options(noreturn, att_syntax)); + } +} + +/// Blindly copies the LOAD segment content at its desired address in physical +/// address space. The loader assumes that the addresses to not clash with the +/// loader (or anything else). +fn map_memory(ph: ProgramHeaderEntry) { + log::debug!("Mapping LOAD segment {ph:#?}"); + let dest_ptr = ph.vaddr() as *mut u8; + let content = ph.content().expect("Should have content"); + unsafe { core::ptr::copy(content.as_ptr(), dest_ptr, content.len()) }; + let dest_ptr = unsafe { dest_ptr.add(ph.filesz() as usize) }; + + // Zero .bss memory + for _ in 0..(ph.memsz() - ph.filesz()) { + unsafe { + core::ptr::write(dest_ptr, 0); + } + } +} diff --git a/integration-test/bins/multiboot2_chainloader/src/main.rs b/integration-test/bins/multiboot2_chainloader/src/main.rs new file mode 100644 index 00000000..59c2940c --- /dev/null +++ b/integration-test/bins/multiboot2_chainloader/src/main.rs @@ -0,0 +1,28 @@ +#![no_main] +#![no_std] +#![feature(error_in_core)] + +mod loader; +mod multiboot; + +extern crate alloc; + +#[macro_use] +extern crate util; + +use util::init_environment; + +core::arch::global_asm!(include_str!("start.S"), options(att_syntax)); + +/// Entry into the Rust code from assembly using the x86 SystemV calling +/// convention. +#[no_mangle] +fn rust_entry(multiboot_magic: u32, multiboot_hdr: *const u32) -> ! { + init_environment(); + let x = 0.12 + 0.56; + log::debug!("{x}"); + log::debug!("multiboot_hdr={multiboot_hdr:x?}, multiboot_magic=0x{multiboot_magic:x?}"); + let mbi = multiboot::get_mbi(multiboot_magic, multiboot_hdr as u32).unwrap(); + let module_iter = mbi.modules().expect("Should provide modules"); + loader::load_module(module_iter); +} diff --git a/integration-test/bins/multiboot2_chainloader/src/multiboot.rs b/integration-test/bins/multiboot2_chainloader/src/multiboot.rs new file mode 100644 index 00000000..31aa0052 --- /dev/null +++ b/integration-test/bins/multiboot2_chainloader/src/multiboot.rs @@ -0,0 +1,41 @@ +//! Parsing the Multiboot information. Glue code for the [`multiboot`] code. + +use anyhow::anyhow; +use core::slice; +pub use multiboot::information::ModuleIter; +pub use multiboot::information::Multiboot as Mbi; +use multiboot::information::{MemoryManagement, Multiboot, PAddr, SIGNATURE_EAX}; + +static mut MEMORY_MANAGEMENT: Mem = Mem; + +/// Returns an object to access the fields of the Multiboot information +/// structure. +pub fn get_mbi<'a>(magic: u32, ptr: u32) -> anyhow::Result> { + if magic != SIGNATURE_EAX { + return Err(anyhow!("Unknown Multiboot signature {magic:x}")); + } + unsafe { Multiboot::from_ptr(ptr as u64, &mut MEMORY_MANAGEMENT) }.ok_or(anyhow!( + "Can't read Multiboot boot information from pointer" + )) +} + +/// Glue object between the global allocator and the multiboot crate. +struct Mem; + +impl MemoryManagement for Mem { + unsafe fn paddr_to_slice(&self, addr: PAddr, size: usize) -> Option<&'static [u8]> { + let ptr = addr as *const u64 as *const u8; + Some(slice::from_raw_parts(ptr, size)) + } + + // If you only want to read fields, you can simply return `None`. + unsafe fn allocate(&mut self, _length: usize) -> Option<(PAddr, &mut [u8])> { + None + } + + unsafe fn deallocate(&mut self, addr: PAddr) { + if addr != 0 { + unimplemented!() + } + } +} diff --git a/integration-test/bins/multiboot2_chainloader/src/start.S b/integration-test/bins/multiboot2_chainloader/src/start.S new file mode 100644 index 00000000..f296887f --- /dev/null +++ b/integration-test/bins/multiboot2_chainloader/src/start.S @@ -0,0 +1,63 @@ +# Symbol from main.rs +.extern rust_entry + +.code32 + +.section .multiboot_header, "a", @progbits + +/* + * Multiboot v1 Header. + * Required so that we can be booted by QEMU via the "-kernel" parameter. + */ +.align 8 +.global multiboot_header +multiboot_header: + .long 0x1badb002 + .long 0x0 + .long -0x1badb002 + +.section .text + +.global start +start: + # Prepare Multiboot2-handoff parameters for Rust + mov %eax, %edi + mov %ebx, %esi + + # Prepare stack + align it to 16 byte (for SSE registers) + mov $stack_end, %eax + sub $16, %eax + # x86 quirk: stack is n-aligned at address x when %esp+$8 is n-aligned + add $8, %eax + + # Set stack + mov %eax, %esp + mov %eax, %ebp + + # Enable SSE. + # Strictly speaking, this is not necessary, but I activated SSE in the + # compiler spec json file. Rustc/LLVM produces SSE coe for example from the + # core::fmt code. + mov %cr0, %eax + and $0xFFFB, %ax # clear coprocessor emulation CR0.EM + or $0x2, %ax # set coprocessor monitoring CR0.MP + mov %eax, %cr0 + mov %cr4, %eax + or $(3 << 9), %ax # set CR4.OSFXSR and CR4.OSXMMEXCPT + mov %eax, %cr4 + + push %ebp + mov %esp, %ebp + # x86 SystemV calling convention: Push arguments in reverse order to stack + push %esi + push %edi + call rust_entry + ud2 + +.section .data + +# 16K natural-aligned stack. +.align 16384 +stack_begin: + .zero 16384 +stack_end: diff --git a/integration-test/bins/multiboot2_payload/Cargo.toml b/integration-test/bins/multiboot2_payload/Cargo.toml new file mode 100644 index 00000000..29d6b7db --- /dev/null +++ b/integration-test/bins/multiboot2_payload/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "multiboot2_payload" +description = "Multiboot2 integration test" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +multiboot2 = { path = "../../../multiboot2", features = ["builder", "unstable"] } +util = { path = "../util" } +anyhow = { version = "1.0.69", default-features = false } +good_memory_allocator = "0.1.7" +log = { version = "0.4.17", default-features = false } +x86 = "0.52.0" diff --git a/integration-test/bins/multiboot2_payload/build.rs b/integration-test/bins/multiboot2_payload/build.rs new file mode 100644 index 00000000..13b5d578 --- /dev/null +++ b/integration-test/bins/multiboot2_payload/build.rs @@ -0,0 +1,5 @@ +fn main() { + let linker_script = "multiboot2_payload/link.ld"; + println!("cargo:rerun-if-changed={linker_script}"); + println!("cargo:rustc-link-arg=-T{linker_script}"); +} diff --git a/integration-test/bins/multiboot2_payload/link.ld b/integration-test/bins/multiboot2_payload/link.ld new file mode 100644 index 00000000..4891d4e6 --- /dev/null +++ b/integration-test/bins/multiboot2_payload/link.ld @@ -0,0 +1,40 @@ +ENTRY(start) + +PHDRS +{ + /* PT_LOAD FLAGS (page table permissions) not necessary, as we perform + legacy boot. Linker probably sets sensible defaults anyway. */ + kernel_rx PT_LOAD; + kernel_rw PT_LOAD; + kernel_ro PT_LOAD; +} + +SECTIONS { + /* Chainloader linked at 8M, payload at 16M */ + .text 16M : AT(16M) ALIGN(4K) + { + *(.multiboot2_header) + *(.text .text.*) + } : kernel_rx + + .rodata : + { + *(.rodata .rodata.*) + } : kernel_ro + + .data : + { + *(.data .data.*) + } : kernel_rw + + .bss : + { + *(COMMON) + *(.bss .bss.*) + } : kernel_rw + + /DISCARD/ : + { + *(.eh_frame .eh_frame*) + } +} diff --git a/integration-test/bins/multiboot2_payload/src/main.rs b/integration-test/bins/multiboot2_payload/src/main.rs new file mode 100644 index 00000000..4599bc35 --- /dev/null +++ b/integration-test/bins/multiboot2_payload/src/main.rs @@ -0,0 +1,37 @@ +#![no_main] +#![no_std] +#![feature(error_in_core)] + +extern crate alloc; + +#[macro_use] +extern crate util; + +core::arch::global_asm!(include_str!("start.S"), options(att_syntax)); +core::arch::global_asm!(include_str!("multiboot2_header.S")); + +use multiboot2::BootInformation; +use util::{init_environment, qemu_exit_success}; + +mod verify; + +/// Entry into the Rust code from assembly. +#[no_mangle] +fn rust_entry(multiboot2_magic: u32, multiboot2_hdr: u32) -> ! { + main(multiboot2_magic, multiboot2_hdr).expect("Should run multiboot2 integration test"); + log::info!("Integration test finished successfully"); + qemu_exit_success() +} + +/// Executes the main logic. +fn main(multiboot2_magic: u32, multiboot2_hdr: u32) -> anyhow::Result<()> { + init_environment(); + if multiboot2_magic != multiboot2::MAGIC { + Err(anyhow::Error::msg("Invalid bootloader magic"))? + } + log::debug!("multiboot2_hdr={multiboot2_hdr:x?}, multiboot2_magic=0x{multiboot2_magic:x?}"); + + let mbi_ptr = (multiboot2_hdr as *const u8).cast(); + let mbi = unsafe { BootInformation::load(mbi_ptr) }.map_err(anyhow::Error::msg)?; + verify::run(&mbi) +} diff --git a/integration-test/bins/multiboot2_payload/src/multiboot2_header.S b/integration-test/bins/multiboot2_payload/src/multiboot2_header.S new file mode 100644 index 00000000..84ac5773 --- /dev/null +++ b/integration-test/bins/multiboot2_payload/src/multiboot2_header.S @@ -0,0 +1,66 @@ +# Multiboot2 Header definition. +# The assembly code uses the GNU Assembly (GAS) flavor with Intel noprefix +# syntax. + +# Symbol from main.rs +.EXTERN start + +.code32 +.align 8 +.section .multiboot2_header + + mb2_header_start: + .long 0xe85250d6 # magic number + .long 0 # architecture 0 (protected mode i386) + .long mb2_header_end - mb2_header_start # header length + # checksum + .long 0x100000000 - (0xe85250d6 + 0 + (mb2_header_end - mb2_header_start)) + + # OPTIONAL MULTIBOOT2 TAGS (additional to required END TAG) + # ------------------------------------------------------------------------------------ + .align 8 + .Lmb2_header_tag_information_request_start: + .word 1 # type (16bit) + .word 0 # flags (16bit) + .long .Lmb2_header_tag_information_request_end - .Lmb2_header_tag_information_request_start # size (32bit) + .long 1 + .long 2 + .long 3 + .long 4 + .long 5 + .long 6 + .long 7 + .long 8 + .long 9 + .long 10 + .long 11 + .long 12 + # .long 13 GRUB reports: not supported + .long 14 + .long 15 + .long 16 + .long 17 + .long 18 + .long 19 + .long 20 + .long 21 + .long + .Lmb2_header_tag_information_request_end: + + .align 8 + .Lmb2_header_tag_module_alignment_start: + .word 7 # type (16bit) + .word 0 # flags (16bit) + .long .Lmb2_header_tag_module_alignment_end - .Lmb2_header_tag_module_alignment_start # size (32bit) + .long start + .Lmb2_header_tag_module_alignment_end: + # ------------------------------------------------------------------------------------ + + # REQUIRED END TAG + .align 8 + .Lmb2_header_tag_end_start: + .word 0 # type (16bit) + .word 0 # flags (16bit) + .long .Lmb2_header_tag_end_end - .Lmb2_header_tag_end_start # size (32bit) + .Lmb2_header_tag_end_end: + mb2_header_end: diff --git a/integration-test/bins/multiboot2_payload/src/start.S b/integration-test/bins/multiboot2_payload/src/start.S new file mode 100644 index 00000000..cb42c66e --- /dev/null +++ b/integration-test/bins/multiboot2_payload/src/start.S @@ -0,0 +1,52 @@ +# Symbol from main.rs +.extern rust_entry + +.code32 +.align 8 +.section .text + +.global start +start: + # Prepare Multiboot2-handoff parameters for Rust + mov %eax, %edi + mov %ebx, %esi + + # Prepare stack + align it to 16 byte (for SSE registers) + mov $stack_end, %eax + sub $16, %eax + # x86 quirk: stack is n-aligned at address x when %esp+$8 is n-aligned + add $8, %eax + + # Set stack + mov %eax, %esp + mov %eax, %ebp + + # Enable SSE. + # Strictly speaking, this is not necessary, but I activated SSE in the + # compiler spec json file. Rustc/LLVM produces SSE coe for example from the + # core::fmt code. + mov %cr0, %eax + and $0xFFFB, %ax # clear coprocessor emulation CR0.EM + or $0x2, %ax # set coprocessor monitoring CR0.MP + mov %eax, %cr0 + mov %cr4, %eax + or $(3 << 9), %ax # set CR4.OSFXSR and CR4.OSXMMEXCPT + mov %eax, %cr4 + + # x86 (i386) calling convention: + # 1. prepare stackframe pointer + # 2. push arguments on stack in reverse order + push %ebp + mov %esp , %ebp + push %esi + push %edi + call rust_entry + ud2 + +.section .data + +# 16K natural-aligned stack. +.align 16384 +stack_begin: + .zero 16384 +stack_end: diff --git a/integration-test/bins/multiboot2_payload/src/verify/chainloader.rs b/integration-test/bins/multiboot2_payload/src/verify/chainloader.rs new file mode 100644 index 00000000..ef2ab7ed --- /dev/null +++ b/integration-test/bins/multiboot2_payload/src/verify/chainloader.rs @@ -0,0 +1,31 @@ +use crate::verify::{print_memory_map, print_module_info}; +use multiboot2::BootInformation; + +pub fn run(mbi: &BootInformation) -> anyhow::Result<()> { + basic_sanity_checks(mbi)?; + print_memory_map(mbi)?; + print_module_info(mbi)?; + // print_elf_info(mbi)?; + Ok(()) +} + +fn basic_sanity_checks(mbi: &BootInformation) -> anyhow::Result<()> { + // Some basic sanity checks + let bootloader_name = mbi + .boot_loader_name_tag() + .ok_or("No bootloader tag") + .map_err(anyhow::Error::msg)? + .name() + .map_err(anyhow::Error::msg)?; + let cmdline = mbi + .command_line_tag() + .ok_or("No cmdline tag") + .map_err(anyhow::Error::msg)? + .cmdline() + .map_err(anyhow::Error::msg)?; + + assert_eq!(bootloader_name, "mb2_integrationtest_chainloader"); + assert_eq!(cmdline, "chainloaded YEAH"); + + Ok(()) +} diff --git a/integration-test/bins/multiboot2_payload/src/verify/grub.rs b/integration-test/bins/multiboot2_payload/src/verify/grub.rs new file mode 100644 index 00000000..df71c93d --- /dev/null +++ b/integration-test/bins/multiboot2_payload/src/verify/grub.rs @@ -0,0 +1,30 @@ +use crate::verify::{print_elf_info, print_memory_map, print_module_info}; +use multiboot2::BootInformation; + +pub fn run(mbi: &BootInformation) -> anyhow::Result<()> { + basic_sanity_checks(mbi)?; + print_memory_map(mbi)?; + print_module_info(mbi)?; + print_elf_info(mbi)?; + Ok(()) +} + +fn basic_sanity_checks(mbi: &BootInformation) -> anyhow::Result<()> { + // Some basic sanity checks + let bootloader_name = mbi + .boot_loader_name_tag() + .ok_or("No bootloader tag") + .map_err(anyhow::Error::msg)? + .name() + .map_err(anyhow::Error::msg)?; + let cmdline = mbi + .command_line_tag() + .ok_or("No cmdline tag") + .map_err(anyhow::Error::msg)? + .cmdline() + .map_err(anyhow::Error::msg)?; + assert_eq!(bootloader_name, "GRUB 2.06"); + assert_eq!(cmdline, "some commandline arguments"); + + Ok(()) +} diff --git a/integration-test/bins/multiboot2_payload/src/verify/mod.rs b/integration-test/bins/multiboot2_payload/src/verify/mod.rs new file mode 100644 index 00000000..1bc4b6fc --- /dev/null +++ b/integration-test/bins/multiboot2_payload/src/verify/mod.rs @@ -0,0 +1,106 @@ +mod chainloader; +mod grub; + +use alloc::format; +use alloc::vec::Vec; +use multiboot2::BootInformation; + +pub fn run(mbi: &BootInformation) -> anyhow::Result<()> { + println!("{mbi:#?}"); + println!(); + + let bootloader = mbi + .boot_loader_name_tag() + .ok_or("No bootloader tag") + .map_err(anyhow::Error::msg)? + .name() + .map_err(anyhow::Error::msg)?; + + if bootloader.to_lowercase().contains("grub") { + log::info!("loaded by grub"); + grub::run(mbi)?; + } else { + log::info!("loaded by chainloader"); + chainloader::run(mbi)?; + } + + Ok(()) +} + +pub(self) fn print_memory_map(mbi: &BootInformation) -> anyhow::Result<()> { + let memmap = mbi + .memory_map_tag() + .ok_or("Should have memory map") + .map_err(anyhow::Error::msg)?; + println!("Memory Map:"); + memmap.memory_areas().iter().for_each(|e| { + println!( + " 0x{:010x} - 0x{:010x} ({:.3} MiB {:?})", + e.start_address(), + e.end_address(), + e.size() as f32 / 1024.0 / 1024.0, + e.typ() + ); + }); + println!(); + Ok(()) +} + +pub(self) fn print_elf_info(mbi: &BootInformation) -> anyhow::Result<()> { + let sections_iter = mbi + .elf_sections() + .ok_or("Should have elf sections") + .map_err(anyhow::Error::msg)?; + println!("ELF sections:"); + for s in sections_iter { + let typ = format!("{:?}", s.section_type()); + let flags = format!("{:?}", s.flags()); + let name = s.name().map_err(anyhow::Error::msg)?; + println!( + " {:<13} {:<17} {:<22} 0x{:010x} 0x{:010x} {:>5.2} MiB align={}", + name, + typ, + flags, + s.start_address(), + s.end_address(), + s.size() as f32 / 1024.0, + s.addralign(), + ); + } + println!(); + Ok(()) +} + +pub(self) fn print_module_info(mbi: &BootInformation) -> anyhow::Result<()> { + let modules = mbi.module_tags().collect::>(); + if modules.len() != 1 { + Err(anyhow::Error::msg("Should have exactly one boot module"))? + } + let module = modules.first().unwrap(); + let module_cmdline = module.cmdline().map_err(anyhow::Error::msg)?; + println!("Modules:"); + println!( + " 0x{:010x} - 0x{:010x} ({} B, cmdline='{}')", + module.start_address(), + module.end_address(), + module.module_size(), + module_cmdline + ); + println!(" grub cfg passed as boot module:"); + let grup_cfg_ptr = module.start_address() as *const u32 as *const u8; + let grub_cfg = + unsafe { core::slice::from_raw_parts(grup_cfg_ptr, module.module_size() as usize) }; + + // In the GRUB bootflow case, we pass the config as module with it. This is + // not done for the chainloaded case. + if let Ok(str) = core::str::from_utf8(grub_cfg) { + println!("=== file begin ==="); + for line in str.lines() { + println!(" > {line}"); + } + println!("=== file end ==="); + println!(); + } + + Ok(()) +} diff --git a/integration-test/bins/rust-toolchain.toml b/integration-test/bins/rust-toolchain.toml new file mode 100644 index 00000000..0f7ffde2 --- /dev/null +++ b/integration-test/bins/rust-toolchain.toml @@ -0,0 +1,6 @@ +[toolchain] +channel = "nightly-2023-06-22" +profile = "minimal" +components = [ + "rust-src" +] diff --git a/integration-test/bins/util/Cargo.toml b/integration-test/bins/util/Cargo.toml new file mode 100644 index 00000000..4baea136 --- /dev/null +++ b/integration-test/bins/util/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "util" +description = "Util library" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +good_memory_allocator = "0.1.7" +log = { version = "0.4.17", default-features = false } +# Wait for release > 3.0.1 +qemu-exit = { git = "https://github.com/rust-embedded/qemu-exit.git", rev = "3cee0efb5c1842b5261850c57b3b4d608542ff03" } diff --git a/integration-test/bins/util/src/allocator.rs b/integration-test/bins/util/src/allocator.rs new file mode 100644 index 00000000..017f3c2e --- /dev/null +++ b/integration-test/bins/util/src/allocator.rs @@ -0,0 +1,17 @@ +use good_memory_allocator::SpinLockedAllocator; + +#[repr(align(0x4000))] +struct Align16K(T); + +/// 16 KiB naturally aligned backing storage for heap. +static mut HEAP: Align16K<[u8; 0x4000]> = Align16K([0; 0x4000]); + +#[global_allocator] +static ALLOCATOR: SpinLockedAllocator = SpinLockedAllocator::empty(); + +/// Initializes the allocator. Call only once. +pub fn init() { + unsafe { + ALLOCATOR.init(HEAP.0.as_ptr().cast::() as _, HEAP.0.len()); + } +} diff --git a/integration-test/bins/util/src/debugcon.rs b/integration-test/bins/util/src/debugcon.rs new file mode 100644 index 00000000..6fdb97ae --- /dev/null +++ b/integration-test/bins/util/src/debugcon.rs @@ -0,0 +1,68 @@ +//! Driver for QEMU's debugcon device. + +use core::fmt::{Arguments, Write}; +use log::{LevelFilter, Log, Metadata, Record}; + +static LOGGER: DebugconLogger = DebugconLogger; + +struct Debugcon; + +/// Internal API for the `println!` macro. +pub fn _print(args: Arguments) { + Debugcon.write_fmt(args).unwrap(); +} + +impl Debugcon { + /// I/O port of QEMUs debugcon device on x86. + const IO_PORT: u16 = 0xe9; + + pub fn write_byte(byte: u8) { + unsafe { + core::arch::asm!( + "outb %al, %dx", + in("al") byte, + in("dx") Self::IO_PORT, + options(att_syntax, nomem, nostack, preserves_flags) + ) + } + } +} + +impl Write for Debugcon { + fn write_str(&mut self, s: &str) -> core::fmt::Result { + for &byte in s.as_bytes() { + Debugcon::write_byte(byte); + } + Ok(()) + } +} + +pub struct DebugconLogger; + +impl DebugconLogger { + pub fn init() { + // Ignore, as we can't do anything about it here. + let _ = log::set_logger(&LOGGER); + log::set_max_level(LevelFilter::Trace); + } +} + +impl Log for DebugconLogger { + fn enabled(&self, _metadata: &Metadata) -> bool { + true + } + + fn log(&self, record: &Record) { + // Ignore result as we can't do anything about it. + let _ = writeln!( + Debugcon, + "[{:>5}: {}@{}]: {}", + record.level(), + record.file().unwrap_or(""), + record.line().unwrap_or(0), + record.args() + ); + } + + fn flush(&self) {} +} diff --git a/integration-test/bins/util/src/lib.rs b/integration-test/bins/util/src/lib.rs new file mode 100644 index 00000000..779dd244 --- /dev/null +++ b/integration-test/bins/util/src/lib.rs @@ -0,0 +1,41 @@ +#![no_std] + +#[macro_use] +extern crate alloc; + +use core::panic::PanicInfo; +use log::error; +use qemu_exit::QEMUExit; + +static QEMU_EXIT: qemu_exit::X86 = qemu_exit::X86::new(QEMU_EXIT_PORT, QEMU_EXIT_SUCCESS); + +#[macro_use] +pub mod macros; +pub mod allocator; +pub mod debugcon; + +const QEMU_EXIT_PORT: u16 = 0xf4; +/// Custom error code to report success. +const QEMU_EXIT_SUCCESS: u32 = 73; + +/// Initializes the environment. +pub fn init_environment() { + debugcon::DebugconLogger::init(); + log::info!("Logger initialized!"); + allocator::init(); + log::info!("Allocator initialized! {:?}", vec![1, 2, 3]); +} + +#[panic_handler] +fn panic_handler(info: &PanicInfo) -> ! { + error!("PANIC! {}", info); + qemu_exit_failure() +} + +pub fn qemu_exit_success() -> ! { + QEMU_EXIT.exit_success() +} + +pub fn qemu_exit_failure() -> ! { + QEMU_EXIT.exit_failure() +} diff --git a/integration-test/bins/util/src/macros.rs b/integration-test/bins/util/src/macros.rs new file mode 100644 index 00000000..8c1418f2 --- /dev/null +++ b/integration-test/bins/util/src/macros.rs @@ -0,0 +1,10 @@ +#[macro_export] +macro_rules! println { + () => { + $crate::println!("") + }; + ($($arg:tt)*) => { + $crate::debugcon::_print(format_args!($($arg)*)); + $crate::debugcon::_print(format_args!("\n")); + }; +} diff --git a/integration-test/bins/x86-unknown-none.json b/integration-test/bins/x86-unknown-none.json new file mode 100644 index 00000000..3e5359f6 --- /dev/null +++ b/integration-test/bins/x86-unknown-none.json @@ -0,0 +1,15 @@ +{ + "llvm-target": "i686-unknown-none", + "data-layout": "e-m:e-i32:32-f80:128-n8:16:32-S128-p:32:32", + "arch": "x86", + "target-endian": "little", + "target-pointer-width": "32", + "target-c-int-width": "32", + "os": "none", + "executables": true, + "linker-flavor": "ld.lld", + "linker": "rust-lld", + "panic-strategy": "abort", + "disable-redzone": true, + "features": "+sse" +} diff --git a/integration-test/nix/sources.json b/integration-test/nix/sources.json new file mode 100644 index 00000000..f5049a96 --- /dev/null +++ b/integration-test/nix/sources.json @@ -0,0 +1,14 @@ +{ + "nixpkgs": { + "branch": "nixos-23.05", + "description": "Nix Packages collection", + "homepage": null, + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ad157fe26e74211e7dde0456cb3fd9ab78b6e552", + "sha256": "0l5gimzlbzq1svw48p4h3wf24ry21icl9198jk5x4xqvs6k2gffx", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/ad157fe26e74211e7dde0456cb3fd9ab78b6e552.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + } +} diff --git a/integration-test/nix/sources.nix b/integration-test/nix/sources.nix new file mode 100644 index 00000000..fe3dadf7 --- /dev/null +++ b/integration-test/nix/sources.nix @@ -0,0 +1,198 @@ +# This file has been generated by Niv. + +let + + # + # The fetchers. fetch_ fetches specs of type . + # + + fetch_file = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchurl { inherit (spec) url sha256; name = name'; } + else + pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; + + fetch_tarball = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchTarball { name = name'; inherit (spec) url sha256; } + else + pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; + + fetch_git = name: spec: + let + ref = + spec.ref or ( + if spec ? branch then "refs/heads/${spec.branch}" else + if spec ? tag then "refs/tags/${spec.tag}" else + abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!" + ); + submodules = spec.submodules or false; + submoduleArg = + let + nixSupportsSubmodules = builtins.compareVersions builtins.nixVersion "2.4" >= 0; + emptyArgWithWarning = + if submodules + then + builtins.trace + ( + "The niv input \"${name}\" uses submodules " + + "but your nix's (${builtins.nixVersion}) builtins.fetchGit " + + "does not support them" + ) + { } + else { }; + in + if nixSupportsSubmodules + then { inherit submodules; } + else emptyArgWithWarning; + in + builtins.fetchGit + ({ url = spec.repo; inherit (spec) rev; inherit ref; } // submoduleArg); + + fetch_local = spec: spec.path; + + fetch_builtin-tarball = name: throw + ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=tarball -a builtin=true''; + + fetch_builtin-url = name: throw + ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=file -a builtin=true''; + + # + # Various helpers + # + + # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 + sanitizeName = name: + ( + concatMapStrings (s: if builtins.isList s then "-" else s) + ( + builtins.split "[^[:alnum:]+._?=-]+" + ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) + ) + ); + + # The set of packages used when specs are fetched using non-builtins. + mkPkgs = sources: system: + let + sourcesNixpkgs = + import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; + hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; + hasThisAsNixpkgsPath = == ./.; + in + if builtins.hasAttr "nixpkgs" sources + then sourcesNixpkgs + else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then + import { } + else + abort + '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; + + # The actual fetching function. + fetch = pkgs: name: spec: + + if ! builtins.hasAttr "type" spec then + abort "ERROR: niv spec ${name} does not have a 'type' attribute" + else if spec.type == "file" then fetch_file pkgs name spec + else if spec.type == "tarball" then fetch_tarball pkgs name spec + else if spec.type == "git" then fetch_git name spec + else if spec.type == "local" then fetch_local spec + else if spec.type == "builtin-tarball" then fetch_builtin-tarball name + else if spec.type == "builtin-url" then fetch_builtin-url name + else + abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; + + # If the environment variable NIV_OVERRIDE_${name} is set, then use + # the path directly as opposed to the fetched source. + replace = name: drv: + let + saneName = stringAsChars (c: if (builtins.match "[a-zA-Z0-9]" c) == null then "_" else c) name; + ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; + in + if ersatz == "" then drv else + # this turns the string into an actual Nix path (for both absolute and + # relative paths) + if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; + + # Ports of functions for older nix versions + + # a Nix version of mapAttrs if the built-in doesn't exist + mapAttrs = builtins.mapAttrs or ( + f: set: with builtins; + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) + ); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 + range = first: last: if first > last then [ ] else builtins.genList (n: first + n) (last - first + 1); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 + stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 + stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); + concatMapStrings = f: list: concatStrings (map f list); + concatStrings = builtins.concatStringsSep ""; + + # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 + optionalAttrs = cond: as: if cond then as else { }; + + # fetchTarball version that is compatible between all the versions of Nix + builtins_fetchTarball = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchTarball; + in + if lessThan nixVersion "1.12" then + fetchTarball ({ inherit url; } // (optionalAttrs (name != null) { inherit name; })) + else + fetchTarball attrs; + + # fetchurl version that is compatible between all the versions of Nix + builtins_fetchurl = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchurl; + in + if lessThan nixVersion "1.12" then + fetchurl ({ inherit url; } // (optionalAttrs (name != null) { inherit name; })) + else + fetchurl attrs; + + # Create the final "sources" from the config + mkSources = config: + mapAttrs + ( + name: spec: + if builtins.hasAttr "outPath" spec + then + abort + "The values in sources.json should not have an 'outPath' attribute" + else + spec // { outPath = replace name (fetch config.pkgs name spec); } + ) + config.sources; + + # The "config" used by the fetchers + mkConfig = + { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null + , sources ? if sourcesFile == null then { } else builtins.fromJSON (builtins.readFile sourcesFile) + , system ? builtins.currentSystem + , pkgs ? mkPkgs sources system + }: rec { + # The sources, i.e. the attribute set of spec name to spec + inherit sources; + + # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers + inherit pkgs; + }; + +in +mkSources (mkConfig { }) // { __functor = _: settings: mkSources (mkConfig settings); } diff --git a/integration-test/run.sh b/integration-test/run.sh new file mode 100755 index 00000000..2f24d09a --- /dev/null +++ b/integration-test/run.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +DIR=$(dirname "$(realpath "$0")") +cd "$DIR" || exit + +# Helper script that runs the actual script in a Nix shell. +nix-shell --run ./.run.sh diff --git a/integration-test/shell.nix b/integration-test/shell.nix new file mode 100644 index 00000000..0140ee1e --- /dev/null +++ b/integration-test/shell.nix @@ -0,0 +1,12 @@ +let + sources = import ./nix/sources.nix; + pkgs = import sources.nixpkgs {}; +in +pkgs.mkShell rec { + nativeBuildInputs = with pkgs; [ + grub2 + qemu + rustup + xorriso + ]; +} diff --git a/integration-test/tests/README.md b/integration-test/tests/README.md new file mode 100644 index 00000000..862ad49c --- /dev/null +++ b/integration-test/tests/README.md @@ -0,0 +1,4 @@ +# Integrationtest Definitions + +This directory contains relevant definitions and shell scripts to start and run +individual tests. diff --git a/integration-test/tests/multiboot2-header/README.md b/integration-test/tests/multiboot2-header/README.md new file mode 100644 index 00000000..60d63f47 --- /dev/null +++ b/integration-test/tests/multiboot2-header/README.md @@ -0,0 +1,7 @@ +# multiboot2-header - Integration Test + +This integration test loads the `multiboot2_chainloader` binary as Multiboot1 +payload using QEMU. The `multiboot2_payload` binary is passed as boot module. +The `multiboot2_chainloader` behaves as bootloader and eventually loads +`multiboot2_payload` into the memory. `multiboot2_payload` figures out during +runtime whether it was loaded by GRUB or by the chainloader. diff --git a/integration-test/tests/multiboot2-header/run_qemu.sh b/integration-test/tests/multiboot2-header/run_qemu.sh new file mode 100755 index 00000000..d7bbfbc3 --- /dev/null +++ b/integration-test/tests/multiboot2-header/run_qemu.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# This script starts a bootable image in QEMU using legacy BIOS boot. + +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +DIR=$(dirname "$(realpath "$0")") +cd "$DIR" || exit + +BINS_DIR="../../bins/target/x86-unknown-none/release" +CHAINLOADER="$BINS_DIR/multiboot2_chainloader" +PAYLOAD="$BINS_DIR/multiboot2_payload" +# add "-d int \" to debug CPU exceptions +# "-display none" is necessary for the CI but locally the display and the +# combat monitor are really helpful + +set +e +qemu-system-x86_64 \ + -kernel "$CHAINLOADER" \ + -append "chainloader" \ + -initrd "$PAYLOAD multiboot2 payload" \ + -m 24m \ + -debugcon stdio \ + -no-reboot \ + -device isa-debug-exit,iobase=0xf4,iosize=0x04 \ + -display none `# relevant for the CI` + +EXIT_CODE=$? +# Custom exit code used by the integration test to report success. +QEMU_EXIT_SUCCESS=73 + +echo "#######################################" +if [[ $EXIT_CODE -eq $QEMU_EXIT_SUCCESS ]]; then + echo "SUCCESS - Integration Test 'multiboot2-header'" + exit 0 +else + echo "FAILED - Integration Test 'multiboot2-header'" + exit "$EXIT_CODE" +fi diff --git a/integration-test/tests/multiboot2/.gitignore b/integration-test/tests/multiboot2/.gitignore new file mode 100644 index 00000000..526859d0 --- /dev/null +++ b/integration-test/tests/multiboot2/.gitignore @@ -0,0 +1,2 @@ +.vol +grub_boot.img diff --git a/integration-test/tests/multiboot2/README.md b/integration-test/tests/multiboot2/README.md new file mode 100644 index 00000000..9ebf2a7e --- /dev/null +++ b/integration-test/tests/multiboot2/README.md @@ -0,0 +1,5 @@ +# multiboot2 - Integration Test + +This integration test uses GRUB as Multiboot2 bootloader and loads the +`multiboot2_payload` binary into the memory. The MBI is read at runtime and +certain checks are performed. diff --git a/integration-test/tests/multiboot2/build_img.sh b/integration-test/tests/multiboot2/build_img.sh new file mode 100755 index 00000000..ff528557 --- /dev/null +++ b/integration-test/tests/multiboot2/build_img.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# This script builds a bootable image. It bundles the test binary into a GRUB +# installation. The GRUB installation is configured to chainload the binary +# via Multiboot2. + +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +DIR=$(dirname "$(realpath "$0")") +cd "$DIR" || exit + +MULTIBOOT2_PAYLOAD_DIR="../../bins" +MULTIBOOT2_PAYLOAD_PATH="$MULTIBOOT2_PAYLOAD_DIR/target/x86-unknown-none/release/multiboot2_payload" + +echo "Verifying that the binary is a multiboot2 binary..." +grub-file --is-x86-multiboot2 "$MULTIBOOT2_PAYLOAD_PATH" + +# Delete previous state. +rm -rf .vol + +mkdir -p .vol/boot/grub +cp grub.cfg .vol/boot/grub +cp "$MULTIBOOT2_PAYLOAD_PATH" .vol + +# Create a GRUB image with the files in ".vol" being embedded. +grub-mkrescue -o "grub_boot.img" ".vol" 2>/dev/null diff --git a/integration-test/tests/multiboot2/grub.cfg b/integration-test/tests/multiboot2/grub.cfg new file mode 100644 index 00000000..ab810423 --- /dev/null +++ b/integration-test/tests/multiboot2/grub.cfg @@ -0,0 +1,13 @@ +# GRUB 2 configuration that boots the integration test binary via Multiboot2. + +set timeout=0 +set default=0 +# set debug=all + +menuentry "Integration Test" { + # The leading slash is very important. + multiboot2 /multiboot2_payload some commandline arguments + # Pass some module + command line. + module2 /boot/grub/grub.cfg grub-config + boot +} diff --git a/integration-test/tests/multiboot2/run_qemu.sh b/integration-test/tests/multiboot2/run_qemu.sh new file mode 100755 index 00000000..099308b1 --- /dev/null +++ b/integration-test/tests/multiboot2/run_qemu.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# This script starts a bootable image in QEMU using legacy BIOS boot. + +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +DIR=$(dirname "$(realpath "$0")") +cd "$DIR" || exit + +BOOT_IMAGE="grub_boot.img" + +# add "-d int \" to debug CPU exceptions +# "-display none" is necessary for the CI but locally the display and the +# combat monitor are really helpful + +set +e +qemu-system-x86_64 \ + -boot d \ + -cdrom "$BOOT_IMAGE" \ + -m 24m \ + -debugcon stdio \ + -no-reboot \ + -device isa-debug-exit,iobase=0xf4,iosize=0x04 \ + -display none `# relevant for the CI` + +EXIT_CODE=$? +# Custom exit code used by the integration test to report success. +QEMU_EXIT_SUCCESS=73 + + +echo "#######################################" +if [[ $EXIT_CODE -eq $QEMU_EXIT_SUCCESS ]]; then + echo "SUCCESS - Integration Test 'multiboot2'" + exit 0 +else + echo "FAILED - Integration Test 'multiboot2'" + exit "$EXIT_CODE" +fi From 76cad2d1ce33d25fd581dfec4abf3be4fc5c2220 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 22 Jun 2023 17:54:44 +0200 Subject: [PATCH 2/4] ci: typos fix --- .typos.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/.typos.toml b/.typos.toml index db059eed..6947ea6e 100644 --- a/.typos.toml +++ b/.typos.toml @@ -7,6 +7,7 @@ extend-exclude = [ [default.extend-words] Rela = "Rela" +grup = "grup" [default.extend-identifiers] # FOOBAR = "FOOBAR" From f4419fb1ce7ea960be4f4b093f60d1fbe45118f6 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Fri, 23 Jun 2023 10:36:58 +0200 Subject: [PATCH 3/4] ci/nix: don't force Nix shell on everyone + fix CI --- .github/workflows/rust.yml | 11 +++++++++-- integration-test/.run.sh | 35 ----------------------------------- integration-test/README.md | 12 +++++++++--- integration-test/run.sh | 32 ++++++++++++++++++++++++++++++-- integration-test/shell.nix | 5 +++++ 5 files changed, 53 insertions(+), 42 deletions(-) delete mode 100755 integration-test/.run.sh diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d53a0fc0..ab025f33 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -129,7 +129,7 @@ jobs: features: builder,unstable integrationtest: - name: multiboot2 integrationtest + name: integrationtest needs: - build_nightly - build_nostd_nightly @@ -138,4 +138,11 @@ jobs: - name: Check out uses: actions/checkout@v3 - uses: cachix/install-nix-action@v20 - - run: integration-test/run.sh + with: + # This channel is only required to invoke "nix-shell". + # Everything inside that nix-shell will use a pinned version of + # nixpkgs. + nix_path: nixpkgs=channel:nixos-23.05 + - run: | + cd integration-test + nix-shell --run ./run.sh diff --git a/integration-test/.run.sh b/integration-test/.run.sh deleted file mode 100755 index 318557b5..00000000 --- a/integration-test/.run.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -# http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail -IFS=$'\n\t' - -DIR=$(dirname "$(realpath "$0")") -cd "$DIR" || exit - -function fn_main() { - fn_build_rust_bins - fn_multiboot2_integrationtest - fn_multiboot2_header_integrationtest -} - -function fn_build_rust_bins() { - cd "bins" - cargo build --release - cd "$DIR" -} - -function fn_multiboot2_integrationtest() { - cd tests/multiboot2 - ./build_img.sh - ./run_qemu.sh - cd "$DIR" -} - -function fn_multiboot2_header_integrationtest() { - cd tests/multiboot2-header - ./run_qemu.sh - cd "$DIR" -} - -fn_main diff --git a/integration-test/README.md b/integration-test/README.md index e7e82479..6639368d 100644 --- a/integration-test/README.md +++ b/integration-test/README.md @@ -10,8 +10,14 @@ directory contains test definitions, run scripts, and other relevant files. The main entry to run all tests is `./run.sh` in this directory. ## TL;DR: -- `$ ./run.sh` to execute the integration tests +- `$ nix-shell --run ./run.sh` to execute the integration tests with Nix (recommended) +- `$ ./run.sh` to execute the integration tests (you have to install dependencies manually) ## Prerequisites -The tests rely on [`nix`](https://nixos.org/) being installed / `nix-shell` -being available to get the relevant tools. +The tests are executed best when using [`nix`](https://nixos.org/)/`nix-shell` +to get the relevant tools. Otherwise, please make sure the following packages +are available: +- grub helper tools +- rustup +- QEMU +- xorriso diff --git a/integration-test/run.sh b/integration-test/run.sh index 2f24d09a..318557b5 100755 --- a/integration-test/run.sh +++ b/integration-test/run.sh @@ -1,7 +1,35 @@ #!/usr/bin/env bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + DIR=$(dirname "$(realpath "$0")") cd "$DIR" || exit -# Helper script that runs the actual script in a Nix shell. -nix-shell --run ./.run.sh +function fn_main() { + fn_build_rust_bins + fn_multiboot2_integrationtest + fn_multiboot2_header_integrationtest +} + +function fn_build_rust_bins() { + cd "bins" + cargo build --release + cd "$DIR" +} + +function fn_multiboot2_integrationtest() { + cd tests/multiboot2 + ./build_img.sh + ./run_qemu.sh + cd "$DIR" +} + +function fn_multiboot2_header_integrationtest() { + cd tests/multiboot2-header + ./run_qemu.sh + cd "$DIR" +} + +fn_main diff --git a/integration-test/shell.nix b/integration-test/shell.nix index 0140ee1e..1871b0df 100644 --- a/integration-test/shell.nix +++ b/integration-test/shell.nix @@ -9,4 +9,9 @@ pkgs.mkShell rec { rustup xorriso ]; + + # To invoke "nix-shell" in the CI-runner, we need a global Nix channel. + # For better reproducibility inside the Nix shell, we override this channel + # with the pinned nixpkgs version. + NIX_PATH = "nixpkgs=${sources.nixpkgs}"; } From c994e7b95ca2cc89670006b70e4edbe28b57497f Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Fri, 23 Jun 2023 10:41:10 +0200 Subject: [PATCH 4/4] ci: use cargo cache for integration test --- .github/workflows/rust.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ab025f33..996416ba 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -143,6 +143,21 @@ jobs: # Everything inside that nix-shell will use a pinned version of # nixpkgs. nix_path: nixpkgs=channel:nixos-23.05 - - run: | - cd integration-test - nix-shell --run ./run.sh + - name: Set up cargo cache + uses: actions/cache@v3 + continue-on-error: false + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + integration-test/bins/target/ + # Hash over Cargo.toml and Cargo.lock, as this might be copied to + # projects that do not have a Cargo.lock in their repository tree! + key: ${{ runner.os }}-${{ github.job }}-${{ hashFiles('integration-test/**/Cargo.toml', 'integration-test/**/Cargo.lock', 'integration-test/bins/rust-toolchain.toml') }} + # Have all the "copying into Nix store" messages in a dedicated step for + # better log visibility. + - run: cd integration-test && nix-shell --run "echo OK" && cd .. + # Now, run the actual test. + - run: cd integration-test && nix-shell --run ./run.sh && cd ..