Add some documentation

core/elf.c

@@ -0,0 +1,182 @@
+#include "elf.h"
+#include "kernel.h"
+#include "klibc.h"
+#include "mem.h"
+#include "paging.h"
+
+/**
+ * Make sure the program is in a valid ELF format, map it into memory,
+ * and return the address of its entry point (ie _start function)
+ *
+ * @return 0 when the program is not a valid ELF
+ */
+
+uaddr_t loadElfProg(const char *prog)
+{
+    int i;
+
+    /**
+     * Typedefs, constants and structure definitions as given by the ELF
+     * standard specifications.
+     */
+    typedef unsigned long Elf32_Addr;
+    typedef unsigned long Elf32_Word;
+    typedef unsigned short Elf32_Half;
+    typedef unsigned long Elf32_Off;
+    // typedef signed long Elf32_Sword;
+
+    /* Elf identification */
+
+#define EI_NIDENT 16
+    typedef struct {
+        unsigned char e_ident[EI_NIDENT];
+        Elf32_Half e_type;
+        Elf32_Half e_machine;
+        Elf32_Word e_version;
+        Elf32_Addr e_entry;
+        Elf32_Off e_phoff;
+        Elf32_Off e_shoff;
+        Elf32_Word e_flags;
+        Elf32_Half e_ehsize;
+        Elf32_Half e_phentsize;
+        Elf32_Half e_phnum;
+        Elf32_Half e_shentsize;
+        Elf32_Half e_shnum;
+        Elf32_Half e_shstrndx;
+    } __attribute__((packed)) Elf32_Ehdr_t;
+
+/* e_ident value */
+#define ELFMAG0 0x7f
+#define ELFMAG1 'E'
+#define ELFMAG2 'L'
+#define ELFMAG3 'F'
+
+/* e_ident offsets */
+#define EI_MAG0 0
+#define EI_MAG1 1
+#define EI_MAG2 2
+#define EI_MAG3 3
+#define EI_CLASS 4
+#define EI_DATA 5
+#define EI_VERSION 6
+#define EI_PAD 7
+
+/* e_ident[EI_CLASS] */
+#define ELFCLASSNONE 0
+#define ELFCLASS32 1
+#define ELFCLASS64 2
+
+/* e_ident[EI_DATA] */
+#define ELFDATANONE 0
+#define ELFDATA2LSB 1
+#define ELFDATA2MSB 2
+
+/* e_type */
+#define ET_NONE 0        /* No file type       */
+#define ET_REL 1         /* Relocatable file   */
+#define ET_EXEC 2        /* Executable file    */
+#define ET_DYN 3         /* Shared object file */
+#define ET_CORE 4        /* Core file          */
+#define ET_LOPROC 0xff00 /* Processor-specific */
+#define ET_HIPROC 0xffff /* Processor-specific */
+
+/* e_machine */
+#define EM_NONE 0  /* No machine     */
+#define EM_M32 1   /* AT&T WE 32100  */
+#define EM_SPARC 2 /* SPARC          */
+#define EM_386 3   /* Intel 80386    */
+#define EM_68K 4   /* Motorola 68000 */
+#define EM_88K 5   /* Motorola 88000 */
+#define EM_860 7   /* Intel 80860    */
+#define EM_MIPS 8  /* MIPS RS3000    */
+
+/* e_version */
+#define EV_NONE 0    /* invalid version */
+#define EV_CURRENT 1 /* current version */
+
+    typedef struct {
+        Elf32_Word p_type;
+        Elf32_Off p_offset;
+        Elf32_Addr p_vaddr;
+        Elf32_Addr p_paddr;
+        Elf32_Word p_filesz;
+        Elf32_Word p_memsz;
+        Elf32_Word p_flags;
+        Elf32_Word p_align;
+    } __attribute__((packed)) Elf32_Phdr_t;
+
+/* Reserved segment types p_type */
+#define PT_NULL 0
+#define PT_LOAD 1
+#define PT_DYNAMIC 2
+#define PT_INTERP 3
+#define PT_NOTE 4
+#define PT_SHLIB 5
+#define PT_PHDR 6
+#define PT_LOPROC 0x70000000
+#define PT_HIPROC 0x7fffffff
+
+/* p_flags */
+#define PF_X 1
+#define PF_W 2
+#define PF_R 4
+
+    Elf32_Ehdr_t *elf_hdr = (Elf32_Ehdr_t *)prog;
+    Elf32_Phdr_t *elf_phdrs;
+
+    /* Macro to check expected values for some fields in the ELF header */
+#define ELF_CHECK(hdr, field, expected_value)                                                 \
+    ({                                                                                        \
+        if ((hdr)->field != (expected_value)) {                                               \
+            printf("ELF prog : for %s, expected %x, got %x\n", #field,                        \
+                   (unsigned)(expected_value), (unsigned)(hdr)->field);                       \
+            return 0;                                                                         \
+        }                                                                                     \
+    })
+
+    ELF_CHECK(elf_hdr, e_ident[EI_MAG0], ELFMAG0);
+    ELF_CHECK(elf_hdr, e_ident[EI_MAG1], ELFMAG1);
+    ELF_CHECK(elf_hdr, e_ident[EI_MAG2], ELFMAG2);
+    ELF_CHECK(elf_hdr, e_ident[EI_MAG3], ELFMAG3);
+    ELF_CHECK(elf_hdr, e_ident[EI_CLASS], ELFCLASS32);
+    ELF_CHECK(elf_hdr, e_ident[EI_DATA], ELFDATA2LSB);
+    ELF_CHECK(elf_hdr, e_type, ET_EXEC);
+    ELF_CHECK(elf_hdr, e_version, EV_CURRENT);
+
+    /* Get the begining of the program header table */
+    elf_phdrs = (Elf32_Phdr_t *)(prog + elf_hdr->e_phoff);
+
+    /* Map the program segment in R/W mode. To make things clean, we
+       should iterate over the sections, not the program header */
+    for (i = 0; i < elf_hdr->e_phnum; i++) {
+        uaddr_t uaddr;
+
+        /* Ignore the empty program headers that are not marked "LOAD" */
+        if (elf_phdrs[i].p_type != PT_LOAD) {
+            continue;
+        }
+
+        if (elf_phdrs[i].p_vaddr < PAGING_BASE_USER_ADDRESS) {
+            printf("User program has an incorrect address 0x%x\n", elf_phdrs[i].p_vaddr);
+            return (uaddr_t)NULL;
+        }
+
+        /* Map pages of physical memory into user space */
+        for (uaddr = ALIGN_DOWN(elf_phdrs[i].p_vaddr, PAGE_SIZE);
+             uaddr < elf_phdrs[i].p_vaddr + elf_phdrs[i].p_memsz; uaddr += PAGE_SIZE) {
+            paddr_t ppage;
+            ppage = allocPhyPage(1);
+
+            assert(pageMap(uaddr, ppage,
+                           PAGING_MEM_USER | PAGING_MEM_WRITE | PAGING_MEM_READ) == 0);
+
+            unrefPhyPage(ppage);
+        }
+
+        /* Copy segment into memory */
+        memcpy((void *)elf_phdrs[i].p_vaddr, (void *)(prog + elf_phdrs[i].p_offset),
+               elf_phdrs[i].p_filesz);
+    }
+
+    return elf_hdr->e_entry;
+}

core/elf.h

@@ -0,0 +1,4 @@
+#pragma once
+
+#include "types.h"
+uaddr_t loadElfProg(const char *prog);

core/main.c

@@ -1,6 +1,7 @@
 #include "alloc.h"
 #include "allocArea.h"
 #include "ata.h"
+#include "elf.h"
 #include "exception.h"
 #include "gdt.h"
 #include "idt.h"
@@ -39,181 +40,6 @@ void idleThread(void *arg)
     }
 }
 
-/**
- * Make sure the program is in a valid ELF format, map it into memory,
- * and return the address of its entry point (ie _start function)
- *
- * @return 0 when the program is not a valid ELF
- */
-static uaddr_t loadElfProg(const char *prog)
-{
-    int i;
-
-    /**
-     * Typedefs, constants and structure definitions as given by the ELF
-     * standard specifications.
-     */
-    typedef unsigned long Elf32_Addr;
-    typedef unsigned long Elf32_Word;
-    typedef unsigned short Elf32_Half;
-    typedef unsigned long Elf32_Off;
-    // typedef signed long Elf32_Sword;
-
-    /* Elf identification */
-
-#define EI_NIDENT 16
-    typedef struct {
-        unsigned char e_ident[EI_NIDENT];
-        Elf32_Half e_type;
-        Elf32_Half e_machine;
-        Elf32_Word e_version;
-        Elf32_Addr e_entry;
-        Elf32_Off e_phoff;
-        Elf32_Off e_shoff;
-        Elf32_Word e_flags;
-        Elf32_Half e_ehsize;
-        Elf32_Half e_phentsize;
-        Elf32_Half e_phnum;
-        Elf32_Half e_shentsize;
-        Elf32_Half e_shnum;
-        Elf32_Half e_shstrndx;
-    } __attribute__((packed)) Elf32_Ehdr_t;
-
-/* e_ident value */
-#define ELFMAG0 0x7f
-#define ELFMAG1 'E'
-#define ELFMAG2 'L'
-#define ELFMAG3 'F'
-
-/* e_ident offsets */
-#define EI_MAG0 0
-#define EI_MAG1 1
-#define EI_MAG2 2
-#define EI_MAG3 3
-#define EI_CLASS 4
-#define EI_DATA 5
-#define EI_VERSION 6
-#define EI_PAD 7
-
-/* e_ident[EI_CLASS] */
-#define ELFCLASSNONE 0
-#define ELFCLASS32 1
-#define ELFCLASS64 2
-
-/* e_ident[EI_DATA] */
-#define ELFDATANONE 0
-#define ELFDATA2LSB 1
-#define ELFDATA2MSB 2
-
-/* e_type */
-#define ET_NONE 0        /* No file type       */
-#define ET_REL 1         /* Relocatable file   */
-#define ET_EXEC 2        /* Executable file    */
-#define ET_DYN 3         /* Shared object file */
-#define ET_CORE 4        /* Core file          */
-#define ET_LOPROC 0xff00 /* Processor-specific */
-#define ET_HIPROC 0xffff /* Processor-specific */
-
-/* e_machine */
-#define EM_NONE 0  /* No machine     */
-#define EM_M32 1   /* AT&T WE 32100  */
-#define EM_SPARC 2 /* SPARC          */
-#define EM_386 3   /* Intel 80386    */
-#define EM_68K 4   /* Motorola 68000 */
-#define EM_88K 5   /* Motorola 88000 */
-#define EM_860 7   /* Intel 80860    */
-#define EM_MIPS 8  /* MIPS RS3000    */
-
-/* e_version */
-#define EV_NONE 0    /* invalid version */
-#define EV_CURRENT 1 /* current version */
-
-    typedef struct {
-        Elf32_Word p_type;
-        Elf32_Off p_offset;
-        Elf32_Addr p_vaddr;
-        Elf32_Addr p_paddr;
-        Elf32_Word p_filesz;
-        Elf32_Word p_memsz;
-        Elf32_Word p_flags;
-        Elf32_Word p_align;
-    } __attribute__((packed)) Elf32_Phdr_t;
-
-/* Reserved segment types p_type */
-#define PT_NULL 0
-#define PT_LOAD 1
-#define PT_DYNAMIC 2
-#define PT_INTERP 3
-#define PT_NOTE 4
-#define PT_SHLIB 5
-#define PT_PHDR 6
-#define PT_LOPROC 0x70000000
-#define PT_HIPROC 0x7fffffff
-
-/* p_flags */
-#define PF_X 1
-#define PF_W 2
-#define PF_R 4
-
-    Elf32_Ehdr_t *elf_hdr = (Elf32_Ehdr_t *)prog;
-    Elf32_Phdr_t *elf_phdrs;
-
-    /* Macro to check expected values for some fields in the ELF header */
-#define ELF_CHECK(hdr, field, expected_value)                                                 \
-    ({                                                                                        \
-        if ((hdr)->field != (expected_value)) {                                               \
-            printf("ELF prog : for %s, expected %x, got %x\n", #field,                        \
-                   (unsigned)(expected_value), (unsigned)(hdr)->field);                       \
-            return 0;                                                                         \
-        }                                                                                     \
-    })
-
-    ELF_CHECK(elf_hdr, e_ident[EI_MAG0], ELFMAG0);
-    ELF_CHECK(elf_hdr, e_ident[EI_MAG1], ELFMAG1);
-    ELF_CHECK(elf_hdr, e_ident[EI_MAG2], ELFMAG2);
-    ELF_CHECK(elf_hdr, e_ident[EI_MAG3], ELFMAG3);
-    ELF_CHECK(elf_hdr, e_ident[EI_CLASS], ELFCLASS32);
-    ELF_CHECK(elf_hdr, e_ident[EI_DATA], ELFDATA2LSB);
-    ELF_CHECK(elf_hdr, e_type, ET_EXEC);
-    ELF_CHECK(elf_hdr, e_version, EV_CURRENT);
-
-    /* Get the begining of the program header table */
-    elf_phdrs = (Elf32_Phdr_t *)(prog + elf_hdr->e_phoff);
-
-    /* Map the program segment in R/W mode. To make things clean, we
-       should iterate over the sections, not the program header */
-    for (i = 0; i < elf_hdr->e_phnum; i++) {
-        uaddr_t uaddr;
-
-        /* Ignore the empty program headers that are not marked "LOAD" */
-        if (elf_phdrs[i].p_type != PT_LOAD) {
-            continue;
-        }
-
-        if (elf_phdrs[i].p_vaddr < PAGING_BASE_USER_ADDRESS) {
-            printf("User program has an incorrect address 0x%x\n", elf_phdrs[i].p_vaddr);
-            return (uaddr_t)NULL;
-        }
-
-        /* Map pages of physical memory into user space */
-        for (uaddr = ALIGN_DOWN(elf_phdrs[i].p_vaddr, PAGE_SIZE);
-             uaddr < elf_phdrs[i].p_vaddr + elf_phdrs[i].p_memsz; uaddr += PAGE_SIZE) {
-            paddr_t ppage;
-            ppage = allocPhyPage(1);
-
-            assert(pageMap(uaddr, ppage,
-                           PAGING_MEM_USER | PAGING_MEM_WRITE | PAGING_MEM_READ) == 0);
-
-            unrefPhyPage(ppage);
-        }
-
-        /* Copy segment into memory */
-        memcpy((void *)elf_phdrs[i].p_vaddr, (void *)(prog + elf_phdrs[i].p_offset),
-               elf_phdrs[i].p_filesz);
-    }
-
-    return elf_hdr->e_entry;
-}
 
 #define FILE_HEADER_SIZE   16
 #define FILE_MAX_SIZE   64

docs/1_system_startup.md

@@ -0,0 +1,8 @@
+# System Startup
+
+The Matos operating system is supposed to be started unsing a bootloader that is [multiboot](https://www.gnu.org/software/grub/manual/multiboot/multiboot.html) compatible.
+As such, it can receive some information about the host such as memory size, layout, video mode supported ...
+
+When the booloader is ready, it will call the `_start` function as it is described as the entry point by the `linker.ld`.
+The `_start` function is defined in ASM under the `arch/$ARCH/boot/boot.S` directory.
+It will setup the stack and few other things like the bss section before calling the `kmain()` function from `core/main.c`

docs/2_physical_mem_setup.md

@@ -0,0 +1,26 @@
+# Physical memory setup
+
+The Physical memory definition stand in the `mem.h` file.
+To keep track of it, the memory is split in chunk of `PAGE_SIZE`. The chunks are then represented by the `struct phyMemDesc` structure.
+
+This structure is a double-linked list allowing us to hold some information on every page like its address and the number of time it is used. Yes, the same physical page could be shared by several different sub-system. 
+
+The system keep track of 2 such linked list. One with the allocated/used (`phyUsedPage`) page. One with the free one (`phyFreePage`).
+
+## Initial `struct phyMemDesc` allocation and setup
+
+Just after system startup, there is no memory allocator setup, and yet we have to store the information about all the physical pages.
+
+For example, if the system was made of 4 MB of memory, we have `4MB/PAGE_SIZE` to store.
+Thanks to `linker.ld` script we have a symbol in C (`__ld_kernel_end`) defining the last address of memory used by the code. So we can compute the needed size to store the structures and book the space after `__ld_kernel_end` for this.
+So from `__ld_kernel_end` to `__ld_kernel_end + MEM_SIZE/PAGE_SIZE` is an array of `struct phyMemDesc` describing all the memory of the computer by chunk of `PAGE_SIZE`
+
+
+We can setup the pages between `__ld_kernel_begin` and `__ld_kernel_end +MEM_SIZE/PAGE_SIZE` as being used and read the information from the bootloader to setup the free pages.
+We will keep one exception for x86 and keep page bellow 0x100000 as used because they are physically mapped to some functions like VGA.
+
+## Allocator
+
+Having lists of used or free physical page make it easy to create a physical page allocator that consist of:
+* allocation: Get a `phyMemDesc` from `phyFreePage` and put it in `phyUsedPage`
+* free: make sure the page is not used anymore and put it in the `phyFreePage` 

docs/3_virtual_mem.md

@@ -0,0 +1,65 @@
+# Virtual Memory
+
+## Introduction
+
+The virtual memory is a memory management system that translate, on the fly, some "ideal" memory address to some real physical memory address.
+It's well described on, for example, [wikipedia](https://en.wikipedia.org/wiki/Virtual_memory).
+
+Here we choose to implement Virtual memory using pagination only. So we keep segmentation configured as a single `flat` model (c.f. `core/gdt.c`).
+
+The paging is mainly implemented in `arch/ARCH/paging.c`.
+
+## x86 and mirroring
+
+We need a way to access and modify the record doing the translation `virtual memory -> physical memory` for the MMU.
+On x86, this register is called the `PD` or Page Directory.
+Once the paging is enabled on the CPU, we use a trick call `mirroring` to access and modify the PD and its content.
+So now, we have a way to map a physical memory page to any given virtual page.
+
+### PD implementation details
+
+In a virtual addr(Vaddr), 10 first bit (MSB) are the index in the Page Directory.
+A Page Directory Entry point to a Page Table. The 10 next bits are then an index in this Page Table. A Page Table entry then point to a physical address at which is added the remaining 12 bits. 
+So they are 1024 entry in the PD, each of them pointing to a PT of 1024 entry.
+Each PTE pointing to 4K page.
+First address (up to pageDesc from mem.c) are mapped such as Paddr == Vaddr. To make PD always accessible a (x86?) trick is used : The mirroring. 
+A given entry N in the PD point to the PD (this is possible because PDE very looks like PTE in x86). So N << (10 + 12 = 4Mo) point to the Paddr of PD.
+Then, accessing N * 4Mo + I * 4Ko is accessing the PT of the Ieme entry in the PD (as MMU take the PD pointed by the PDE number N like a PT).
+More particularly, accessing N * 4Mo + N * 4ko is accessing the PD.
+
+PD is at Vaddr N * 4Mo and take 4ko. Each PT are allocated dynamically.
+Just make sure that N have not been used by identity mapping
+
+## Virtual memory allocators
+
+We will setup 2 different virtual memory allocator:
+* `allocArea` for large memory area of several PAGE_SIZE size
+* `alloc` for object of any size
+
+With each needing the other (`allocArea` need `alloc` to allocate `struct memArea` to keep track of the area and `alloc` need `allocArea` to allocate large memory size) we need to make sure that there is no forever loop.
+
+### `AllocArea`
+
+An area is represented by `struct memArea` and basically consist of a virtual memory address and a size if number of page.
+This is a simple allocator keeping 2 linked list of used/free area.
+
+Allocating a new area (thanks to `areaAlloc()`) consist of:
+1. finding an area big enough in the free list.
+2. split it if the found area is too large.
+3. put the found area in the user area list.
+4. optionally map the area to physical page
+
+Freeing an area (thank to `areaFree()`) consist of trying to find adjacent free area and merge them with this one or just adding the area to the free one.
+
+### `Alloc`
+
+
+For large object allocation ( > PAGE_SIZE), this a basically a call to `areaAlloc()`.
+For smaller object, this is a more complex allocator based on the concept of slab.
+
+The allocator is configured at startup to maintain slab for allocating object of a given size.
+So a slab is a linked list of `struct slabEntry` having each of the entries pointing to one or several memory area that is divided in chunk of the slab configured size.
+It's a linked list so it is each to add a new `struct slabEntry` when one is full.
+Inside a `struct slabEntry` the `freeEl` attribute point to the next free chunk (of the slab configured size) in the allocated page(s). At this address is also a pointer to the next free area in this area. 
+
+