From 3f4191b7819cf81a34fea29ba4f4cbe4e04297f0 Mon Sep 17 00:00:00 2001 From: James Martin Date: Tue, 21 Jul 2020 15:38:21 -0700 Subject: [PATCH] Add a ton of comments to main.rs and update README/timeline docs. --- Cargo.toml | 8 +- README.md | 99 +++++++++++++++++++---- docs/timeline.md | 23 ++---- src/main.rs | 200 ++++++++++++++++++++++++++++++++++++++--------- 4 files changed, 259 insertions(+), 71 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3f76380..be1c865 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,8 +9,6 @@ publish = false [dependencies] compiler_builtins = { git = "https://github.com/rust-lang/compiler-builtins" } -uefi = "0.4.7" -x86_64 = "0.11.1" [dependencies.log] version = "0.4.11" @@ -19,3 +17,9 @@ default-features = false [dependencies.num-integer] version = "0.1.36" default-features = false + +[target.'cfg(target_os = "uefi")'.dependencies] +uefi = "0.4.7" + +[target.'cfg(target_arch = "x86_64")'.dependencies] +x86_64 = "0.11.1" diff --git a/README.md b/README.md index 4e2d385..b63bc77 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,93 @@ # bootproof -Messing around with UEFI apps. +A hobby x86_64 operating system written in Rust. -I don't have a specific goal here. -My general direction is to work towards a bootable programming language environment, -preferably one where security and allocation etc. are handled through the programming language -rather than through a traditional operating system. -I don't seriously expect to ever accomplish that, so for now I'm probably just going to... -make a forth or something. +## Installation +bootproof runs on x86_64 and expects to be loaded by UEFI. +You can either boot it using an emulator or on your own computer. -## System Requirements -Other configurations may work, but only these systems are regularly tested. -* CPU: x86_64 QEMU, OVMF UEFI. -* Memory: 128 MB. (64 MB appears to be the minimum required to load OVMF at all. Real hardware might require less?) +### Building +You'll need to the Rust nightly toolchain installed +because bootproof relies heavily on Rust nightly features +(most of which are directly necessary for OS development). -## Running -bootproof runs on x86_64 UEFI. You may either boot the program directly on your own computer or use an emulator. +You'll also need the `cargo-xbuild` crate installed +so that you can compile for the `x86_64-unknown-uefi` target. -Make sure you have the `cargo-xbuild` crate installed and nightly Rust so you can compile to the UEFI target. +Building bootproof is pretty straightforward: -First, build with: ``` cargo xbuild --target x86_64-unknown-uefi ``` -And to run, `./run.sh` will launch bootproof in QEMU. +You can add the `--release` flag for a release-profile build. + +This will produce an executable, +`target/x86_64-unknown-efi/{profile}/bootproof.efi`. + +### Running +#### With QEMU +You will need QEMU, and OVMF, which provides a UEFI implementation for QEMU. +On Debian derivatives, you can install these dependencies with: + +``` +apt install qemu-system-x86 ovmf +``` + +*After you have built the crate with `cargo xbuild`*, +you can use `./run.sh $profile` to run QEMU with some good presets. +`$profile` may be either `debug` or `release`, depending on which you built. +If you don't specify, it defaults to `debug`, just like `cargo`. + +The VM's serial port will be mapped to stdio, +which you can use to interact with the OS. + +#### With real hardware +I would strongly recommend against doing this. + +Copy `bootproof.efi` to your system EFI partition in the EFI folder. +You may put it wherever you'd like and select it while booting. +Alternatively, you can name it `/EFI/Boot/BootX64.efi`, +and it will be loaded automatically, *instead of your regular bootloader or OS*. + +You do *not* need a bootloader to run bootproof. The UEFI is all you need. + +## Goals +1. **Have fun.** Ultimately, I'm doing this *because I want to*. + Operating system development can be very difficult and tedious at times, + but if I've turned this project into work, I've failed. + +2. **Gain experience**, in particular with Rust, large-scale projects, + and low-level programming in general. I should always be learning + and becoming a better programmer. + +3. **Show off.** I want to demonstrate my skills as a programmer, + both to employers and to other programmers in general + (because at least in my opinion, writing your own operating system + gives you some serious cred!) + +4. **Make something I'd want to use.** + I should always be working towards an operating system + that directly addresses my use cases and supports my hardware, + so that if I ever managed to get far enough along, + I'd actually *want* to use the OS that I ended up making. + +## Philosophy +1. **Simplicity.** + Getting a lot of stuff done in a simple way + is better than getting very little done in an ideal way, + especially with a scope as large as an entire operating system. + +2. **Maintainability.** + Operating system codebases are large, complex, and long-lived. + In the long term, good maintainability is absolutely necessary. + +3. **Forward-thinking.** + Focus on what you'll need tomorrow, not what you need today. + By the time you have it, tomorrow will be today and today will be yesterday. + +4. **Iterate quickly.** + There's a lot I don't know, and ultimately the best way to learn it + is to explore the space through programming. + It's *okay* to write some crappy code + if that means my next attempt will be much better-- + as long as I don't let it build up and interfere with maintainability. diff --git a/docs/timeline.md b/docs/timeline.md index 9d550a6..949ddfc 100644 --- a/docs/timeline.md +++ b/docs/timeline.md @@ -1,20 +1,9 @@ # Timeline A tentative short-term timeline for what to do next. -## Leaving UEFI -* Transition from UEFI stdout to UEFI GOP -* Transition from UEFI GOP to real graphics drivers: - * VirtIO GPU - * Intel HD Graphics 630 -* Transition from UEFI stdin to a PS/2 keyboard -* Transition from UEFI allocation to a custom allocator -* RTC support continues even after exiting UEFI boot services. - -## Multiprocessing -* Support for basic relocatable executables -* Single-processor scheduler -* Multi-processor scheduling - -## Accessing storage -* NVMe driver -* FAT32 support +1. Set up the APIC. +2. Write a PS/2 keyboard driver. +3. Get a framebuffer working so I can use my graphics mode again. +4. Write an NVMe driver. +5. Implement the FAT32 file system. +6. ??? diff --git a/src/main.rs b/src/main.rs index 9df2c7d..8ce7e9a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,18 @@ +// Kernels cannot use the standard library +// because it depends on features like IO and memory allocation being available, +// but those features are *defined* by the kernel. +// The core and alloc crates form a subset of the standard library which you still can use; +// core only includes basic definitions which do not require anything special, +// and alloc only requires that you define your own global allocator, which we do. #![no_std] +// The entry point to a UEFI application is called `efi_main`, not `main`. #![no_main] +// Used because this is a UEFI application so we need to use its ABI to make calls. #![feature(abi_efiapi)] +// Required by nightly when defining a global allocator. #![feature(alloc_error_handler)] #![feature(asm)] +// Used to conveniently define x86 interrupt handling routines. #![feature(abi_x86_interrupt)] #![feature(generic_associated_types)] extern crate alloc; @@ -16,52 +26,152 @@ mod logger; use alloc::vec::Vec; use uefi::prelude::*; +// # Why did you choose to make bootproof a UEFI application? +// +// There are three major ways an operating system can choose to be loaded: +// +// 1. As a UEFI (Unified Extensible Firmware Interface) application. +// UEFI will set up most hardware and the CPU in a simple sane way +// (long mode, identity paged, etc.), and then load the UEFI app. +// It also supports some decent drivers and features like a page allocator, +// which are useful during booting but are not something you can depend on long-term +// for reasons I'll get into later. +// UEFI is implemented in firmware, which means it requires no installation or configuration; +// just install your OS on your EFI partition, and it'll get loaded and work. +// +// 2. Through a bootloader, such as GRUB, in particular as a multiboot kernel. +// This will also set up things in a sane way and load your OS, +// and is generally more configurable and portable than UEFI +// (in particular because it works on systems which do not support UEFI, +// such as legacy, BIOS-only systems, and architectures which do not use it). +// +// 3. Through the BIOS. The BIOS will load your application however the CPU just happens to be, +// i.e. 16-bit real mode, and provides a bunch of outdated drivers and information, +// which are often missing so that multiple strategies are needed, +// and are generally only available in 16-bit real mode. +// It will only load a few KiB of your OS and beyond that you're on your own. +// +// The BIOS is a legacy P.o.S. system which is deprecated and likely to eventually be removed, +// so I have no interest in supporting it. +// Furthermore, I do not need the legacy compatibility or configurability of a bootloader, +// and its portability with initial system setup, and I'd have to do substantial work to port +// this OS to other platforms anyway, so I have chosen instead to use the +// native, zero-installation, zero-configuration solution of being loaded directly from UEFI. +// +// Support for being loaded as a multiboot kernel is +// something I'd be willing to have in the future, +// but it's not something I'm going to do until it becomes neccesary, if it ever even does. #[entry] fn efi_main(handle: Handle, st_boot: SystemTable) -> Status { + // UEFI applications are, from the perspective of the operating system, split into two phases: + // a booting phase, where the UEFI boot services are available, + // and the runtime phase, where they are not (although a few "runtime" services + // are available in either phase). + // (From the perspective of UEFI itself, there are more phases, + // before and after the UEFI application's lifetime, but these are not relevant to us). + // The UEFI boot services are the stuff like the allocator and various drivers. + // + // So if these UEFI boot services are so great, why should you leave them? + // Because they interfere with you writing your own drivers and so forth. + // The UEFI drivers are great if you're writing an application like a bootloader + // or something that specifically needs full system control like a hardware tester, + // but when you're trying to make a fully-featured operating system runtime, + // you will need to use hardware in more sophisticated ways than UEFI allows. + // There is no reliable way to make UEFI boot services continue to work + // when you try to take control over memory, or interrupts, or anything else, + // which makes it impossible to write your own drivers without disabling it. + // + // In our case, we already *have* been loaded by the UEFI, so we won't need *any* + // of the UEFI's boot services (except for those which are inherently necessary to leave it, + // i.e. the UEFI allocator), so our goal should be to leave it as quickly as possible, + // so we can get on with setting up our hardware we'll actually need it. + + // First, we'll need to set Rust's global allocator to use the UEFI page allocator, + // so we can allocate stuff until we get our own allocator working + // (which we can't do until we've left boot services). + // + // This allocator is necessary for two reasons: + // + // 1. So we can allocate somewhere to store the UEFI memory maps, + // which is necessary to leave UEFI boot services, + // 2. So we can allocate space for our runtime allocator's data structures. + // + // It has the additional benefit of allowing us to use the `println!` macro for debugging, + // which depends on the `format!` macro, which allocates `String`s. + // + // I really wish I *didn't* depend on the UEFI allocator, + // but I haven't found a good way around it so far. + // (There *are* ways I can think of, but they're difficult enough to not be worth it.) use crate::memory::allocator::{ALLOCATOR, GlobalAllocator}; unsafe { - // Generally speaking, we want to depend on UEFI as little as possible, - // so the need for a UEFI allocator may seem a bit strange. - // However, there's this awkward time during booting when we need - // to allocate space for our "real" allocator's data structures and the UEFI memory maps, - // the result being that we need an allocator for our allocator. - // In theory there are probably ways to get around it, but why bother? - // Just taking advantage of the UEFI allocator briefly is a lot easier. - // (This also lets us use `println!` prior to our main allocator being set up.) use crate::memory::allocator::uefi::UefiAllocator; - // ABSOLUTELY DO NOT FORGET TO DISABLE THIS AFTER LEAVING UEFI BOOT SERVICES. - // ALL ALLOCATIONS MUST BE STATIC OR BE FREED BEFORE BOOT SERVICES EXITS. - // If the're not, Rust still try to free UEFI-allocated data using the new allocator, - // which is undefined behavior. + // The allocator must be global and have a static lifetime because of how Rust works; + // we do not have scoped allocators, only an application-wide global one. + // We work around this by using a mutable global variable which we set to + // use whichever implementation (boot or runtime) is available at the time. + // The UEFI allocator needs to be able to reference the boot services table + // so it can make allocations, and, being a global variable, it needs to be an owned copy. + // This is unfortunate because the safety of the use of the boot table + // depends on lifetimes, so that you cannot own a copy of the boot table + // after you exit boot services, and cloning it violates that safety. + // Trying to use the UEFI allocator after exiting boot services would be bad, + // so I have to make this unsafe disclaimer: + // + // **DO NOT FORGET TO DISABLE THIS ALLOCATOR AFTER LEAVING UEFI BOOT SERVICES.** + // + // Furthermore, Rust doesn't *know* that which allocator I've used has changed + // so it might try to free data which was allocated by a different allocator. + // The runtime allocator has knowledge of what memory the UEFI boot services allocated + // through the memory map the UEFI provides when you exit boot services. + // However, I don't to require the allocator to keep track of that + // in conjunction with the memory that it allocated itself, + // so instead I'll make a second unsafe disclaimer: + // + // **ALL ALLOCATIONS MADE BY THE UEFI ALLOCATOR + // MUST BE STATIC OR FREED BEFORE BOOT SERVICES EXITS.** ALLOCATOR = GlobalAllocator::Uefi(UefiAllocator::new(st_boot.unsafe_clone())); + } + unsafe { + // For now, I use a serial device for logging kernel debug output. + // Although serial ports don't physically exist on modern devices, + // they're still supported by emulators, and they're extremely useful for debugging + // thanks to their simplicity. + // For QEMU, you can set `-serial stdio` and kernel output will be logged to STDOUT. use crate::driver::tty::serial::{COM1_PORT, SerialTty}; logger::set_tty(SerialTty::new(COM1_PORT)); logger::init().unwrap(); } - // Our first task is to exit the UEFI boot services. - // UEFI provides a whole bunch of useful device drivers, - // but they're unusable after you exit the boot services, - // and you absolutely *have* to exit boot services to do most OS-related things. + // Next we have to set up our runtime allocator and exit UEFI boot services. + // These must be done simultaneously because our runtime allocator + // depends on the UEFI memory map, which we get as a result of exiting boot services. + // The memory map describes where a bunch of important stuff lies in memory, + // and most importantly to us, describes what memory is free for allocation + // what memory is currently in use by the kernel and UEFI runtime services, + // and what memory is e.g. reserved by the CPU or contains memory-mapped devices. - // When we exit boot services, UEFI provides us with a memory map, - // which describes where a bunch of important stuff lies in memory - // (e.g. memory-mapped devices and the ACPI tables), - // and what memory is available for us to use safely. - - // We can't let the memory map be de-allocated because it is allocated using the UEFI allocator, - // but would end up being freed using the standard allocator, which is undefined behavior. // We must provide a buffer (mmap_buf) for UEFI to write the memory map to. + // We can't let it be de-allocated because it is allocated using the UEFI allocator, + // for the reasons described above. let mut mmap_buf = Vec::new(); let (_mmap, st) = { let bs = st_boot.boot_services(); - // More allocations can happen between the allocation of the buffer and the buffer being filled, - // so add space for 32 more memory descriptors (an arbitrary number) just to make sure there's enough. + // A lot of allocations can happen between the buffer being allocated + // and the buffer being populated when the boot services exit + // (both by us and the UEFI's own processes; + // in fact, reserving space is necessary even when you *immediately* load the memory map), + // so we have to leave extra space in the memory map for those allocations. + // 1024 is a number that I came up with by repeatedly testing numbers + // until the kernel stopped crashing. mmap_buf.resize(bs.memory_map_size() + 1024, 0); - // HACK: I hate having to use the UEFI allocator just to set up another allocator! - // There's got to be a better way. + // First we read the memory map so that the runtime allocator + // can decide how much space it needs to allocate for its own data structures + // using the UEFI allocator, which needs to be done before exiting UEFI boot services. + // Between now and exiting boot services, only kernel and boot services will be made, + // not changes to reserved memory and so forth (or at least I hope not! + // so the amount of physical memory the allocator needs to keep track of will not change. use crate::memory::allocator::standard::StandardAllocator; let mut allocator; { @@ -70,40 +180,58 @@ fn efi_main(handle: Handle, st_boot: SystemTable) -> Status { allocator = StandardAllocator::new(&mut mmap); } - // Finally, we actually exit the UEFI boot services. + // Actually exit UEFI boot services! let (st, mut mmap) = st_boot.exit_boot_services(handle, mmap_buf.as_mut_slice()) .expect_success("Failed to exit the UEFI boot services."); + // We now populate the allocator with the final memory map. + // Before we were just allocating space for data structures, + // but the actual memory used wasn't set in stone; now it is. + // Since we don't distinguish boot services memory + // from unallocated memory after exiting boot services, + // perhaps we could just populate it from the original memory map and ignore this entirely? + // I'm already making the assumption that reserved/runtime memory won't change, + // and I don't make any new kernel allocations between then and now. allocator.populate(&mut mmap); unsafe { ALLOCATOR = GlobalAllocator::Standard(allocator); } (mmap, st) }; - // Set up the stuff I need to handle interrupts, which is necessary to write drivers for most devices. + // Now that UEFI is no longer handling interrupts, + // we want them disabled until we set up our own handler, + // which we will do... also right now. + // Interrupt handling is necessary to write drivers for most devices. use x86_64::instructions::interrupts; - use crate::arch::x86_64::{gdt, idt}; - interrupts::disable(); - // TODO: I've actually found that resetting the GDT isn't necessary in the emulator. - // However, I'm not sure if that's true in general, and at worst it seems harmless, so it stays for now. + + use crate::arch::x86_64::{gdt, idt}; + // TODO: Resetting the GDT hasn't actually proven to be necessary in the emulator. + // However, I'm not sure if that's true in general, + // and at worst it seems harmless, so it stays for now. // That said, further research is needed. gdt::load(); idt::load(); + // We now have our own interrupt handler so we can re-enable them now. + // That said, we still need to set up APIC to recieve interrupts for devices, + // which isn't something that I've programmed yet. I'm working on it, though! interrupts::enable(); // Everything up to this point has been setting up the CPU state, drivers, etc. // Now we begin running actual programs - // (or in this case, since we don't support actual programs yet, whatever debug stuff I want to run). + // (or in this case, since we don't support actual programs yet, + // whatever debug stuff I want to run). main(st) } -fn main(_st: SystemTable) -> ! { +fn main(st: SystemTable) -> ! { // Put whatever code you want for debugging/testing purposes here... arch::x86_64::breakpoint(); - // There's nothing left for us to do at this point, because there are no meaningful programs to run. + // There's nothing left for us to do at this point, + // because there are no meaningful programs to run. // Instead, we'll just spin forever until the computer is turned off. + // We don't want to shut down so we can continue displaying any debug output. // We do *not* disable interrupts to allow for testing the interrupt handlers. loop { x86_64::instructions::hlt(); } }