Initial commit

code/ap_boot_copy.cpp

@@ -0,0 +1,13 @@
+// Allocate physical memory for copying the startup routine
+auto &memoryService = System::getService<MemoryService>();
+void *startupCodeMemory = memoryService.mapIO(0x8000, Util::PAGESIZE);
+
+// Identity map the allocated physical memory to the kernel address space
+memoryService.unmap(reinterpret_cast<uint32_t>(startupCodeMemory));
+memoryService.mapPhysicalAddress(0x8000, 0x8000,
+                                 Kernel::Paging::PRESENT | Kernel::Paging::READ_WRITE);
+
+// Copy the startup routine and prepared variables to the identity mapped page
+auto startupCode = Util::Address<uint32_t>(reinterpret_cast<uint32_t>(&boot_ap));
+auto destination = Util::Address<uint32_t>(0x8000);
+destination.copyRange(startupCode, boot_ap_size);

code/ap_boot_post.cpp

@@ -0,0 +1,4 @@
+// Excerpt from the smpEntry function
+Apic::initializeCurrentLocalApic();
+Apic::enableCurrentErrorHandler();
+Apic::startCurrentTimer();

code/ap_boot_protected_ap.asm

@@ -0,0 +1,25 @@
+; Continuing boot_ap_32:
+    ; Get the local APIC ID of this AP, to locate GDT and stack
+    mov eax, 0x1
+    cpuid
+    shr ebx, 0x18
+    mov edi, ebx ; Now the ID is in EDI
+
+    ; Load the prepared AP GDT
+    mov ebx, [boot_ap_gdts - boot_ap + startup_address]
+    mov eax, [ebx + edi * 0x4]
+    lgdt [eax]
+
+    ; Load the TSS
+    mov ax, 0x28
+    ltr ax
+
+    ; Load the correct stack for this AP
+    mov ebx, [boot_ap_stacks - boot_ap + startup_address]
+    mov esp, [ebx + edi * 0x4]
+    add esp, stack_size ; Stack starts at the bottom
+    mov ebp, esp
+
+    ; Call the entry function
+    push edi
+    call [boot_ap_entry - boot_ap + startup_address]

code/ap_boot_protected_bsp.asm

@@ -0,0 +1,12 @@
+; This section has to be compiled for 32-bit protected mode
+boot_ap_32:
+    ; Set cr3, cr0 and cr4 to the BSP's values for paging
+    mov eax, [boot_ap_cr3 - boot_ap + startup_address]
+    mov cr3, eax
+    mov eax, [boot_ap_cr0 - boot_ap + startup_address]
+    mov cr0, eax
+    mov eax, [boot_ap_cr4 - boot_ap + startup_address]
+    mov cr4, eax
+
+    ; Load the system IDT
+    lidt [boot_ap_idtr - boot_ap + startup_address]

code/ap_boot_real.asm

@@ -0,0 +1,16 @@
+; Continuing boot_ap:
+    ; Enable Protected Mode, executed from an identity mapped page.
+    mov eax, cr0
+    or al, 0x1 ; Set PE bit
+    mov cr0, eax
+
+    ; Setup the protected mode segments
+    mov ax, 0x10
+    mov ds, ax ; Data segment register
+    mov es, ax ; Extra segment register
+    mov ss, ax ; Stack segment register
+    mov fs, ax ; General purpose segment register
+    mov gs, ax ; General purpose segment register
+
+    ; Far jump to protected mode, set code segment register
+    jmp dword 0x8:boot_ap_32 - boot_ap + 0x8000

code/ap_boot_real_prepare.asm

@@ -0,0 +1,13 @@
+; This section has to be compiled for 16-bit real mode
+boot_ap:
+    ; Disable interrupts
+    cli
+
+    ; Enable A20 address line
+    in al, 0x92
+    or al, 2
+    out 0x92, al
+
+    ; Load the temporary GDT required for the far jump into protected mode.
+    lgdt [tmp_gdt_desc - boot_ap + 0x8000]
+

code/ap_boot_usa.cpp

@@ -0,0 +1,19 @@
+// Excerpt from the Universal Startup Algorithm
+for (uint8_t cpu = 0; cpu < cpuCount; ++cpu) {
+    if (cpu == LocalApic::getId()) { continue; } // Skip the BSP
+
+    LocalApic::clearErrors();
+    LocalApic::sendInitIpi(cpu, ICREntry::Level::ASSERT);
+    LocalApic::waitForIpiDispatch();
+    LocalApic::sendInitIpi(cpu, ICREntry::Level::DEASSERT);
+    LocalApic::waitForIpiDispatch();
+    Pit::earlyDelay(10'000); // 10 milliseconds
+    for (uint8_t i = 0; i < 2; ++i) {
+        LocalApic::clearErrors();
+        LocalApic::sendStartupIpi(cpu, apStartupAddress);
+        LocalApic::waitForIpiDispatch();
+        Pit::earlyDelay(200); // 200 microseconds
+    }
+
+    while (!(runningAPs & (1 << cpu))) {} // Wait until the AP is running
+}

code/ap_boot_variables.cpp

@@ -0,0 +1,15 @@
+// These variables are located in the startup routine's TEXT section
+// and exported from smp_boot.asm.
+
+// Use IDTR from the BSP
+asm volatile("sidt %0" : "=m"(boot_ap_idtr));
+
+// Use cr0, cr3 and cr4 from the BSP
+asm volatile("mov %%cr0, %%eax;" : "=a"(boot_ap_cr0));
+asm volatile("mov %%cr3, %%eax;" : "=a"(boot_ap_cr3));
+asm volatile("mov %%cr4, %%eax;" : "=a"(boot_ap_cr4));
+
+// Set the address of the pre-allocated GDTs, stacks and the entry function
+boot_ap_gdts = reinterpret_cast<uint32_t>(apGdts);
+boot_ap_stacks = reinterpret_cast<uint32_t>(apStacks);
+boot_ap_entry = reinterpret_cast<uint32_t>(&smpEntry);

code/apic_allow.cpp

@@ -0,0 +1,10 @@
+void Apic::allow(InterruptRequest interruptRequest) {
+    IoApic::IrqOverride *override = IoApic::getOverride(interruptRequest);
+    if (override == nullptr) {
+        // If no override is specified, the IRQ is identity mapped to the GSI
+        IoApic::allow(static_cast<GlobalSystemInterrupt>(interruptRequest));
+    } else {
+        // If an override is specified, lookup which GSI the IRQ is mapped to
+        IoApic::allow(override->target);
+    }
+}

code/apicerror_trigger.cpp

@@ -0,0 +1,8 @@
+void ApicErrorHandler::trigger(const InterruptFrame &frame) {
+    // Write/read register: Write first, then read
+    // Writing the ESR updates its contents and arms the interrupt again
+    LocalApic::writeDoubleWord(LocalApic::ESR, 0);
+    uint32_t errors = LocalApic::readDoubleWord(LocalApic::ESR);
+
+    log.error("APIC error on core [%d]: [0x%x]!", LocalApic::getId(), errors);
+}

code/apictimer_calibrate.cpp

@@ -0,0 +1,11 @@
+// Excerpt from the ApicTimer::calibrate function
+uint32_t ApicTimer::calibrate() {
+    // Start the timer with a large counter
+    LocalApic::writeDoubleWord(LocalApic::TIMER_INITIAL, 0xFFFFFFFF);
+
+    // Wait a little
+    Pit::earlyDelay(10'000);
+
+    // Calculate how often the timer ticked in one millisecond
+    return (0xFFFFFFFF - LocalApic::readDoubleWord(LocalApic::TIMER_CURRENT)) / 10;
+}

code/apictimer_trigger.cpp

@@ -0,0 +1,10 @@
+// Excerpt from the APIC timer interrupt handler
+void ApicTimer::trigger(const InterruptFrame &frame) {
+    if (cpuId != LocalApic::getId()) {
+        // Abort if the handler doesn't belong to the current CPU
+        return;
+    }
+
+    // Increase the "core-local" time
+    time.addNanoseconds(timerInterval * 1'000'000); // Interval is in milliseconds
+}

code/interruptdispatcher_assign.cpp

@@ -0,0 +1,8 @@
+// Excerpt from the "InterruptDispatcher" class
+void InterruptDispatcher::assign(uint8_t slot, InterruptHandler &isr) {
+    if (handler[slot] == nullptr) {
+        handler[slot] = new Util::ArrayList<InterruptHandler*>;
+    }
+
+    handler[slot]->add(&isr); // Register an interrupt handler to an interrupt vector
+}

code/interruptdispatcher_check_spurious.cpp

@@ -0,0 +1,4 @@
+// Excerpt from the "checkSpuriousInterrupt" function
+bool InterruptService::checkSpuriousInterrupt(InterruptVector interrupt) {
+    return interrupt == InterruptVector::SPURIOUS;
+}

code/interruptdispatcher_dispatch.cpp

@@ -0,0 +1,12 @@
+// Excerpt from the "dispatch" function
+interruptDepthWrapper.inc();
+interruptService.sendEndOfInterrupt(slot); // Signal interrupt servicing
+asm volatile("sti");                       // Allow cascaded interrupts
+
+uint32_t size = handlerList->size();
+for (uint32_t i = 0; i < size; i++) {
+    handlerList->get(i)->trigger(frame);   // Call registered interrupt handlers
+}
+
+asm volatile("cli");
+interruptDepthWrapper.dec();

code/interruptdispatcher_ignore_spurious.cpp

@@ -0,0 +1,5 @@
+// Excerpt from the "dispatch" function
+if (interruptService.checkSpuriousInterrupt(slot)) {
+    spuriousCounterWrapper.inc();
+    return; // Early return to skip the calling of any handlers
+}

code/interrupthandler_def.cpp

@@ -0,0 +1,4 @@
+// Excerpt from the "InterruptHandler" class definition
+public:
+    virtual void plugin() = 0;                             // Register the handler
+    virtual void trigger(const InterruptFrame &frame) = 0; // Handle an interrupt

code/interruptservice_after.cpp

@@ -0,0 +1,7 @@
+void InterruptService::allowHardwareInterrupt(InterruptRequest interrupt) {
+    if (Apic::isEnabled()) {
+        Apic::allow(interrupt);
+    } else {
+        Pic::allow(interrupt);
+    }
+}

code/ioapic_irqoverride.cpp

@@ -0,0 +1,6 @@
+struct IrqOverride {
+    InterruptRequest source;
+    GlobalSystemInterrupt target;
+    REDTBLEntry::PinPolarity polarity;
+    REDTBLEntry::TriggerMode trigger;
+};

code/ioapic_redtbl_entry.cpp

@@ -0,0 +1,13 @@
+struct REDTBLEntry {
+    InterruptVector vector;
+    DeliveryMode    deliveryMode;
+    DestinationMode destinationMode;
+    DeliveryStatus  deliveryStatus;
+    PinPolarity     pinPolarity;
+    TriggerMode     triggerMode;
+    bool            isMasked;
+    uint8_t         destination;
+
+    explicit REDTBLEntry(uint64_t registerValue);
+    explicit operator uint64_t() const;
+};

code/ioapic_redtbl_example.cpp

@@ -0,0 +1,26 @@
+// Excerpt from the IoAPic::initializeREDTBL function
+void IoApic::initializeREDTBL() {
+    // GSI2 belongs to the PIT in many systems
+    auto gsi = GlobalSystemInterrupt(2);
+    IrqOverride *override = getOverride(gsi);
+
+    REDTBLEntry redtblEntry{};
+    redtblEntry.deliveryMode    = REDTBLEntry::DeliveryMode::FIXED;
+    redtblEntry.destinationMode = REDTBLEntry::DestinationMode::PHYSICAL;
+    redtblEntry.isMasked        = true;
+    redtblEntry.destination     = LocalApic::getId(); // Redirect to BSP
+
+    if (override != nullptr) {
+        // Apply any information provided by an interrupt override
+        redtblEntry.vector      = override->source + 32;
+        redtblEntry.pinPolarity = override->polarity;
+        redtblEntry.triggerMode = override->trigger;
+    } else {
+        // Apply PC/AT compatible ISA bus defaults
+        redtblEntry.vector      = gsi + 32;
+        redtblEntry.pinPolarity = REDTBLEntry::PinPolarity::HIGH;
+        redtblEntry.triggerMode = REDTBLEntry::TriggerMode::EDGE;
+    }
+
+    writeREDTBL(gsi, redtblEntry);
+}

code/ioapic_write_indirect.cpp

@@ -0,0 +1,4 @@
+void IoApic::writeIndirectRegister(uint32_t reg, uint32_t val) {
+    writeMMIORegister<uint8_t>(0x00, reg);  // Write the index register
+    writeMMIORegister<uint32_t>(0x10, val); // Write the data register
+}

code/ioapic_write_mmio.cpp

@@ -0,0 +1,4 @@
+template<typename T>
+void IoApic::writeMMIORegister(uint32_t reg, T val) {
+    *reinterpret_cast<volatile T *>(mmioAddress + reg) = val;
+}

code/ioapic_write_redtbl.cpp

@@ -0,0 +1,7 @@
+void IoApic::writeREDTBL(GlobalSystemInterrupt gsi, const REDTBLEntry &redtbl) {
+    auto val = static_cast<uint64_t>(redtbl);
+    redtblLock.acquire(); // Synchronized in case of multiple APs
+    writeIndirectRegister(0x10 + 2 * gsi, val & 0xFFFFFFFF); // Low DW
+    writeIndirectRegister(0x10 + 2 * gsi + 1, val >> 32);    // High DW
+    redtblLock.release();
+}

code/lapic_apr_example.cpp

@@ -0,0 +1,12 @@
+void LocalApic::synchronizeArbitrationIds() {
+    ICREntry icrEntry{};
+    icrEntry.vector               = 0;
+    icrEntry.deliveryMode         = ICREntry::DeliveryMode::INIT;
+    icrEntry.destinationMode      = ICREntry::DestinationMode::PHYSICAL;
+    icrEntry.level                = ICREntry::Level::DEASSERT;
+    icrEntry.triggerMode          = ICREntry::TriggerMode::LEVEL;
+    icrEntry.destinationShorthand = ICREntry::DestinationShorthand::ALL;
+    icrEntry.destination          = 0;
+    writeICR(icrEntry);
+    waitForIpiDispatch();
+}

code/lapic_icr_entry.cpp

@@ -0,0 +1,13 @@
+struct ICREntry {
+    InterruptVector      vector;
+    DeliveryMode         deliveryMode;
+    DestinationMode      destinationMode;
+    DeliveryStatus       deliveryStatus;
+    Level                level;
+    TriggerMode          triggerMode;
+    DestinationShorthand destinationShorthand;
+    uint8_t              destination;
+
+    explicit ICREntry(uint64_t registerValue);
+    explicit operator uint64_t() const;
+};

code/lapic_imcr.cpp

@@ -0,0 +1,7 @@
+void LocalApic::disablePicMode() {
+    IoPort registerSelectorPort = IoPort(0x22);
+    IoPort registerDataPort = IoPort(0x23);
+
+    registerSelectorPort.writeByte(0x70); // IMCR address is 0x70
+    registerDataPort.writeByte(0x01);     // 0x01 disconnects PIC
+}

code/lapic_initipi_example.cpp

@@ -0,0 +1,11 @@
+void LocalApic::issueINIT(uint8_t cpuId, ICREntry::Level level) {
+    ICREntry icrEntry{};
+    icrEntry.vector               = 0;
+    icrEntry.deliveryMode         = ICREntry::DeliveryMode::INIT;
+    icrEntry.destinationMode      = ICREntry::DestinationMode::PHYSICAL;
+    icrEntry.level                = level; // ASSERT or DEASSERT
+    icrEntry.triggerMode          = ICREntry::TriggerMode::LEVEL;
+    icrEntry.destinationShorthand = ICREntry::DestinationShorthand::NO;
+    icrEntry.destination          = cpuId;
+    writeICR(icrEntry); // Writing the ICR issues IPI
+}

code/lapic_lvt_entry.cpp

@@ -0,0 +1,12 @@
+struct LVTEntry {
+    InterruptVector vector;
+    DeliveryMode    deliveryMode;
+    DeliveryStatus  deliveryStatus;
+    PinPolarity     pinPolarity;
+    TriggerMode     triggerMode;
+    bool            isMasked;
+    TimerMode       timerMode;
+
+    explicit LVTEntry(uint32_t registerValue);
+    explicit operator uint32_t() const;
+};

code/lapic_lvt_example.cpp

@@ -0,0 +1,7 @@
+LVTEntry lvtEntry{};
+lvtEntry.deliveryMode = LVTEntry::DeliveryMode::FIXED;
+lvtEntry.pinPolarity  = LVTEntry::PinPolarity::HIGH;
+lvtEntry.triggerMode  = LVTEntry::TriggerMode::EDGE;
+lvtEntry.isMasked     = true;
+lvtEntry.vector       = InterruptVector::LINT0;
+writeLVT(LINT0, lvtEntry);

code/lapic_mmio_alloc.cpp

@@ -0,0 +1,9 @@
+uint32_t allocateMMIORegion() {
+    // The apicBaseAddress is obtained via ACPI
+    auto &memoryService = System::getService<MemoryService>();
+    void *virtAddressPtr = memoryService.mapIO(apicBaseAddress, Util::PAGESIZE);
+
+    // Account for possible misalignment, as mapIO returns a page-aligned pointer
+    uint32_t pageOffset = apicBaseAddress % Util::PAGESIZE;
+    return reinterpret_cast<uint32_t>(virtAddressPtr) + pageOffset;
+}

code/lapic_mmio_write.cpp

@@ -0,0 +1,4 @@
+void LocalApic::writeDoubleWord(uint32_t reg, uint32_t val) {
+    // Use volatile to prevent compiletime caching and code elimination
+    *reinterpret_cast<volatile uint32_t *>(virtAddress + reg) = val;
+}

code/lapic_msr_entry.cpp

@@ -0,0 +1,9 @@
+struct BaseMSREntry {
+    bool     isBSP;
+    bool     isX2Apic;
+    bool     isXApic;
+    uint32_t baseField;
+
+    explicit BaseMSREntry(uint64_t registerValue);
+    explicit operator uint64_t() const;
+};

code/lapic_sipi_example.cpp

@@ -0,0 +1,11 @@
+void LocalApic::issueSIPI(uint8_t cpuId, uint32_t page) {
+    ICREntry icrEntry{};
+    icrEntry.vector               = page;
+    icrEntry.deliveryMode         = ICREntry::DeliveryMode::STARTUP;
+    icrEntry.destinationMode      = ICREntry::DestinationMode::PHYSICAL;
+    icrEntry.level                = ICREntry::Level::ASSERT;
+    icrEntry.triggerMode          = ICREntry::TriggerMode::EDGE;
+    icrEntry.destinationShorthand = ICREntry::DestinationShorthand::NO;
+    icrEntry.destination          = cpuId;
+    writeICR(icrEntry); // Writing the ICR issues an IPI
+}

code/lapic_svr_entry.cpp

@@ -0,0 +1,9 @@
+struct SVREntry {
+    InterruptVector vector;
+    bool            isSWEnabled;
+    bool            hasFocusProcessorChecking;
+    bool            suppressEoiBroadcasting;
+
+    explicit SVREntry(uint32_t registerValue);
+    explicit operator uint32_t() const;
+};

code/lapic_svr_example.cpp

@@ -0,0 +1,4 @@
+SVREntry svrEntry{};
+svrEntry.vector      = InterruptVector::SPURIOUS;
+svrEntry.isSWEnabled = true; // Keep the APIC software enabled
+writeSVR(svrEntry);

code/lapic_write_icr.cpp

@@ -0,0 +1,7 @@
+void LocalApic::writeICR(const ICREntry &icrEntry) {
+    auto val = static_cast<uint64_t>(icrEntry);
+    icrLock.acquire(); // Synchronized in case of multiple APs
+    writeDoubleWord(0x310, val >> 32);
+    writeDoubleWord(0x300, val & 0xFFFFFFFF); // Writing the low DW sends the IPI
+    icrLock.release();
+}

code/lapic_write_lvt.cpp

@@ -0,0 +1,3 @@
+void LocalApic::writeLVT(uint32_t reg, const LVTEntry &lvtEntry) {
+    writeDoubleWord(reg, static_cast<uint32_t>(lvtEntry));
+}

code/lapic_write_msr.cpp

@@ -0,0 +1,4 @@
+void LocalApic::writeBaseMSR(const MSREntry &msrEntry) {
+    ModelSpecificRegister baseMSR = ModelSpecificRegister(0x1B);
+    baseMSR.writeQuadWord(static_cast<uint64_t>(msrEntry)); // Atomic write
+}

code/lapic_write_svr.cpp

@@ -0,0 +1,3 @@
+void LocalApic::writeSVR(const SVREntry &svrEntry) {
+    writeDoubleWord(0xF0, static_cast<uint32_t>(svrEntry));
+}

code/pit_after.cpp

@@ -0,0 +1,12 @@
+void Pit::trigger(const InterruptFrame &frame) {
+    time.addNanoseconds(timerInterval); // Increase system time
+
+    // Don't use PIT for scheduling when the APIC timer is enabled
+    if (Apic::isEnabled()) {
+        return;
+    }
+
+    if (time.toMilliseconds() % yieldInterval == 0) {
+        System::getService<SchedulerService>().yield(); // Trigger preemption
+    }
+}

code/pit_trigger.cpp

@@ -0,0 +1,8 @@
+// Excerpt from the "Pit" interrupt handler
+void Pit::trigger(const InterruptFrame &frame) {
+    time.addNanoseconds(timerInterval); // Increase system time
+
+    if (time.toMilliseconds() % yieldInterval == 0) {
+        System::getService<SchedulerService>().yield(); // Trigger preemption
+    }
+}

code/smp_entry.cpp

@@ -0,0 +1,4 @@
+// Excerpt from the smpEntry function
+void smpEntry(uint8_t cpuId) {
+    runningAPs |= (1 << cpuId); // Mark that this AP is running
+}

code/system_after.cpp

@@ -0,0 +1,9 @@
+// Excerpt from System::initializeSystem(). Located before interrupts
+// are enabled and any devices have registered their handlers.
+if (Apic::isSupported()) {
+    Apic::enable();
+
+    if (Apic::isSmpSupported()) {
+        Apic::startupSmp();
+    }
+}