Commit 2c750557 authored by Linus Torvalds's avatar Linus Torvalds

Merge git://git.kernel.org/pub/scm/linux/kernel/git/rusty/linux-2.6-lguest

* git://git.kernel.org/pub/scm/linux/kernel/git/rusty/linux-2.6-lguest:
  lguest: documentation update
  lguest: Add to maintainers file.
  lguest: build fix
  lguest: clean up lguest_launcher.h
  lguest: remove unused "wake" element from struct lguest
  lguest: use defines from x86 headers instead of magic numbers
  lguest: example launcher header cleanup.
parents fc42dabe e1e72965
This diff is collapsed.
...@@ -2259,6 +2259,13 @@ L: legousb-devel@lists.sourceforge.net ...@@ -2259,6 +2259,13 @@ L: legousb-devel@lists.sourceforge.net
W: http://legousb.sourceforge.net/ W: http://legousb.sourceforge.net/
S: Maintained S: Maintained
LGUEST
P: Rusty Russell
M: rusty@rustcorp.com.au
L: lguest@ozlabs.org
W: http://lguest.ozlabs.org/
S: Maintained
LINUX FOR IBM pSERIES (RS/6000) LINUX FOR IBM pSERIES (RS/6000)
P: Paul Mackerras P: Paul Mackerras
M: paulus@au.ibm.com M: paulus@au.ibm.com
......
...@@ -56,6 +56,7 @@ ...@@ -56,6 +56,7 @@
#include <linux/lguest.h> #include <linux/lguest.h>
#include <linux/lguest_launcher.h> #include <linux/lguest_launcher.h>
#include <linux/virtio_console.h> #include <linux/virtio_console.h>
#include <linux/pm.h>
#include <asm/paravirt.h> #include <asm/paravirt.h>
#include <asm/param.h> #include <asm/param.h>
#include <asm/page.h> #include <asm/page.h>
...@@ -98,7 +99,7 @@ static cycle_t clock_base; ...@@ -98,7 +99,7 @@ static cycle_t clock_base;
* When lazy_mode is set, it means we're allowed to defer all hypercalls and do * When lazy_mode is set, it means we're allowed to defer all hypercalls and do
* them as a batch when lazy_mode is eventually turned off. Because hypercalls * them as a batch when lazy_mode is eventually turned off. Because hypercalls
* are reasonably expensive, batching them up makes sense. For example, a * are reasonably expensive, batching them up makes sense. For example, a
* large mmap might update dozens of page table entries: that code calls * large munmap might update dozens of page table entries: that code calls
* paravirt_enter_lazy_mmu(), does the dozen updates, then calls * paravirt_enter_lazy_mmu(), does the dozen updates, then calls
* lguest_leave_lazy_mode(). * lguest_leave_lazy_mode().
* *
...@@ -163,8 +164,8 @@ void async_hcall(unsigned long call, ...@@ -163,8 +164,8 @@ void async_hcall(unsigned long call,
/*:*/ /*:*/
/*G:033 /*G:033
* Here are our first native-instruction replacements: four functions for * After that diversion we return to our first native-instruction
* interrupt control. * replacements: four functions for interrupt control.
* *
* The simplest way of implementing these would be to have "turn interrupts * The simplest way of implementing these would be to have "turn interrupts
* off" and "turn interrupts on" hypercalls. Unfortunately, this is too slow: * off" and "turn interrupts on" hypercalls. Unfortunately, this is too slow:
...@@ -183,7 +184,7 @@ static unsigned long save_fl(void) ...@@ -183,7 +184,7 @@ static unsigned long save_fl(void)
return lguest_data.irq_enabled; return lguest_data.irq_enabled;
} }
/* "restore_flags" just sets the flags back to the value given. */ /* restore_flags() just sets the flags back to the value given. */
static void restore_fl(unsigned long flags) static void restore_fl(unsigned long flags)
{ {
lguest_data.irq_enabled = flags; lguest_data.irq_enabled = flags;
...@@ -356,7 +357,7 @@ static void lguest_cpuid(unsigned int *eax, unsigned int *ebx, ...@@ -356,7 +357,7 @@ static void lguest_cpuid(unsigned int *eax, unsigned int *ebx,
* it. The Host needs to know when the Guest wants to change them, so we have * it. The Host needs to know when the Guest wants to change them, so we have
* a whole series of functions like read_cr0() and write_cr0(). * a whole series of functions like read_cr0() and write_cr0().
* *
* We start with CR0. CR0 allows you to turn on and off all kinds of basic * We start with cr0. cr0 allows you to turn on and off all kinds of basic
* features, but Linux only really cares about one: the horrifically-named Task * features, but Linux only really cares about one: the horrifically-named Task
* Switched (TS) bit at bit 3 (ie. 8) * Switched (TS) bit at bit 3 (ie. 8)
* *
...@@ -371,8 +372,7 @@ static void lguest_cpuid(unsigned int *eax, unsigned int *ebx, ...@@ -371,8 +372,7 @@ static void lguest_cpuid(unsigned int *eax, unsigned int *ebx,
static unsigned long current_cr0, current_cr3; static unsigned long current_cr0, current_cr3;
static void lguest_write_cr0(unsigned long val) static void lguest_write_cr0(unsigned long val)
{ {
/* 8 == TS bit. */ lazy_hcall(LHCALL_TS, val & X86_CR0_TS, 0, 0);
lazy_hcall(LHCALL_TS, val & 8, 0, 0);
current_cr0 = val; current_cr0 = val;
} }
...@@ -387,10 +387,10 @@ static unsigned long lguest_read_cr0(void) ...@@ -387,10 +387,10 @@ static unsigned long lguest_read_cr0(void)
static void lguest_clts(void) static void lguest_clts(void)
{ {
lazy_hcall(LHCALL_TS, 0, 0, 0); lazy_hcall(LHCALL_TS, 0, 0, 0);
current_cr0 &= ~8U; current_cr0 &= ~X86_CR0_TS;
} }
/* CR2 is the virtual address of the last page fault, which the Guest only ever /* cr2 is the virtual address of the last page fault, which the Guest only ever
* reads. The Host kindly writes this into our "struct lguest_data", so we * reads. The Host kindly writes this into our "struct lguest_data", so we
* just read it out of there. */ * just read it out of there. */
static unsigned long lguest_read_cr2(void) static unsigned long lguest_read_cr2(void)
...@@ -398,7 +398,7 @@ static unsigned long lguest_read_cr2(void) ...@@ -398,7 +398,7 @@ static unsigned long lguest_read_cr2(void)
return lguest_data.cr2; return lguest_data.cr2;
} }
/* CR3 is the current toplevel pagetable page: the principle is the same as /* cr3 is the current toplevel pagetable page: the principle is the same as
* cr0. Keep a local copy, and tell the Host when it changes. */ * cr0. Keep a local copy, and tell the Host when it changes. */
static void lguest_write_cr3(unsigned long cr3) static void lguest_write_cr3(unsigned long cr3)
{ {
...@@ -411,7 +411,7 @@ static unsigned long lguest_read_cr3(void) ...@@ -411,7 +411,7 @@ static unsigned long lguest_read_cr3(void)
return current_cr3; return current_cr3;
} }
/* CR4 is used to enable and disable PGE, but we don't care. */ /* cr4 is used to enable and disable PGE, but we don't care. */
static unsigned long lguest_read_cr4(void) static unsigned long lguest_read_cr4(void)
{ {
return 0; return 0;
...@@ -432,7 +432,7 @@ static void lguest_write_cr4(unsigned long val) ...@@ -432,7 +432,7 @@ static void lguest_write_cr4(unsigned long val)
* maps virtual addresses to physical addresses using "page tables". We could * maps virtual addresses to physical addresses using "page tables". We could
* use one huge index of 1 million entries: each address is 4 bytes, so that's * use one huge index of 1 million entries: each address is 4 bytes, so that's
* 1024 pages just to hold the page tables. But since most virtual addresses * 1024 pages just to hold the page tables. But since most virtual addresses
* are unused, we use a two level index which saves space. The CR3 register * are unused, we use a two level index which saves space. The cr3 register
* contains the physical address of the top level "page directory" page, which * contains the physical address of the top level "page directory" page, which
* contains physical addresses of up to 1024 second-level pages. Each of these * contains physical addresses of up to 1024 second-level pages. Each of these
* second level pages contains up to 1024 physical addresses of actual pages, * second level pages contains up to 1024 physical addresses of actual pages,
...@@ -440,7 +440,7 @@ static void lguest_write_cr4(unsigned long val) ...@@ -440,7 +440,7 @@ static void lguest_write_cr4(unsigned long val)
* *
* Here's a diagram, where arrows indicate physical addresses: * Here's a diagram, where arrows indicate physical addresses:
* *
* CR3 ---> +---------+ * cr3 ---> +---------+
* | --------->+---------+ * | --------->+---------+
* | | | PADDR1 | * | | | PADDR1 |
* Top-level | | PADDR2 | * Top-level | | PADDR2 |
...@@ -498,8 +498,7 @@ static void lguest_set_pmd(pmd_t *pmdp, pmd_t pmdval) ...@@ -498,8 +498,7 @@ static void lguest_set_pmd(pmd_t *pmdp, pmd_t pmdval)
* *
* ... except in early boot when the kernel sets up the initial pagetables, * ... except in early boot when the kernel sets up the initial pagetables,
* which makes booting astonishingly slow. So we don't even tell the Host * which makes booting astonishingly slow. So we don't even tell the Host
* anything changed until we've done the first page table switch. * anything changed until we've done the first page table switch. */
*/
static void lguest_set_pte(pte_t *ptep, pte_t pteval) static void lguest_set_pte(pte_t *ptep, pte_t pteval)
{ {
*ptep = pteval; *ptep = pteval;
...@@ -720,10 +719,10 @@ static void lguest_time_init(void) ...@@ -720,10 +719,10 @@ static void lguest_time_init(void)
/* Set up the timer interrupt (0) to go to our simple timer routine */ /* Set up the timer interrupt (0) to go to our simple timer routine */
set_irq_handler(0, lguest_time_irq); set_irq_handler(0, lguest_time_irq);
/* Our clock structure look like arch/i386/kernel/tsc.c if we can use /* Our clock structure looks like arch/x86/kernel/tsc_32.c if we can
* the TSC, otherwise it's a dumb nanosecond-resolution clock. Either * use the TSC, otherwise it's a dumb nanosecond-resolution clock.
* way, the "rating" is initialized so high that it's always chosen * Either way, the "rating" is set so high that it's always chosen over
* over any other clocksource. */ * any other clocksource. */
if (lguest_data.tsc_khz) if (lguest_data.tsc_khz)
lguest_clock.mult = clocksource_khz2mult(lguest_data.tsc_khz, lguest_clock.mult = clocksource_khz2mult(lguest_data.tsc_khz,
lguest_clock.shift); lguest_clock.shift);
...@@ -749,7 +748,7 @@ static void lguest_time_init(void) ...@@ -749,7 +748,7 @@ static void lguest_time_init(void)
* to work. They're pretty simple. * to work. They're pretty simple.
*/ */
/* The Guest needs to tell the host what stack it expects traps to use. For /* The Guest needs to tell the Host what stack it expects traps to use. For
* native hardware, this is part of the Task State Segment mentioned above in * native hardware, this is part of the Task State Segment mentioned above in
* lguest_load_tr_desc(), but to help hypervisors there's this special call. * lguest_load_tr_desc(), but to help hypervisors there's this special call.
* *
...@@ -850,13 +849,16 @@ static __init char *lguest_memory_setup(void) ...@@ -850,13 +849,16 @@ static __init char *lguest_memory_setup(void)
return "LGUEST"; return "LGUEST";
} }
/* Before virtqueues are set up, we use LHCALL_NOTIFY on normal memory to /* We will eventually use the virtio console device to produce console output,
* produce console output. */ * but before that is set up we use LHCALL_NOTIFY on normal memory to produce
* console output. */
static __init int early_put_chars(u32 vtermno, const char *buf, int count) static __init int early_put_chars(u32 vtermno, const char *buf, int count)
{ {
char scratch[17]; char scratch[17];
unsigned int len = count; unsigned int len = count;
/* We use a nul-terminated string, so we have to make a copy. Icky,
* huh? */
if (len > sizeof(scratch) - 1) if (len > sizeof(scratch) - 1)
len = sizeof(scratch) - 1; len = sizeof(scratch) - 1;
scratch[len] = '\0'; scratch[len] = '\0';
...@@ -883,7 +885,7 @@ static __init int early_put_chars(u32 vtermno, const char *buf, int count) ...@@ -883,7 +885,7 @@ static __init int early_put_chars(u32 vtermno, const char *buf, int count)
* Our current solution is to allow the paravirt back end to optionally patch * Our current solution is to allow the paravirt back end to optionally patch
* over the indirect calls to replace them with something more efficient. We * over the indirect calls to replace them with something more efficient. We
* patch the four most commonly called functions: disable interrupts, enable * patch the four most commonly called functions: disable interrupts, enable
* interrupts, restore interrupts and save interrupts. We usually have 10 * interrupts, restore interrupts and save interrupts. We usually have 6 or 10
* bytes to patch into: the Guest versions of these operations are small enough * bytes to patch into: the Guest versions of these operations are small enough
* that we can fit comfortably. * that we can fit comfortably.
* *
...@@ -1015,7 +1017,7 @@ __init void lguest_init(void) ...@@ -1015,7 +1017,7 @@ __init void lguest_init(void)
asm volatile ("mov %0, %%fs" : : "r" (__KERNEL_DS) : "memory"); asm volatile ("mov %0, %%fs" : : "r" (__KERNEL_DS) : "memory");
/* The Host uses the top of the Guest's virtual address space for the /* The Host uses the top of the Guest's virtual address space for the
* Host<->Guest Switcher, and it tells us how much it needs in * Host<->Guest Switcher, and it tells us how big that is in
* lguest_data.reserve_mem, set up on the LGUEST_INIT hypercall. */ * lguest_data.reserve_mem, set up on the LGUEST_INIT hypercall. */
reserve_top_address(lguest_data.reserve_mem); reserve_top_address(lguest_data.reserve_mem);
...@@ -1065,6 +1067,6 @@ __init void lguest_init(void) ...@@ -1065,6 +1067,6 @@ __init void lguest_init(void)
/* /*
* This marks the end of stage II of our journey, The Guest. * This marks the end of stage II of our journey, The Guest.
* *
* It is now time for us to explore the nooks and crannies of the three Guest * It is now time for us to explore the layer of virtual drivers and complete
* devices and complete our understanding of the Guest in "make Drivers". * our understanding of the Guest in "make Drivers".
*/ */
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
#include <asm/processor-flags.h> #include <asm/processor-flags.h>
/*G:020 This is where we begin: head.S notes that the boot header's platform /*G:020 This is where we begin: head.S notes that the boot header's platform
* type field is "1" (lguest), so calls us here. The boot header is in %esi. * type field is "1" (lguest), so calls us here.
* *
* WARNING: be very careful here! We're running at addresses equal to physical * WARNING: be very careful here! We're running at addresses equal to physical
* addesses (around 0), not above PAGE_OFFSET as most code expectes * addesses (around 0), not above PAGE_OFFSET as most code expectes
...@@ -17,13 +17,15 @@ ...@@ -17,13 +17,15 @@
* boot. */ * boot. */
.section .init.text, "ax", @progbits .section .init.text, "ax", @progbits
ENTRY(lguest_entry) ENTRY(lguest_entry)
/* Make initial hypercall now, so we can set up the pagetables. */ /* We make the "initialization" hypercall now to tell the Host about
* us, and also find out where it put our page tables. */
movl $LHCALL_LGUEST_INIT, %eax movl $LHCALL_LGUEST_INIT, %eax
movl $lguest_data - __PAGE_OFFSET, %edx movl $lguest_data - __PAGE_OFFSET, %edx
int $LGUEST_TRAP_ENTRY int $LGUEST_TRAP_ENTRY
/* The Host put the toplevel pagetable in lguest_data.pgdir. The movsl /* The Host put the toplevel pagetable in lguest_data.pgdir. The movsl
* instruction uses %esi implicitly. */ * instruction uses %esi implicitly as the source for the copy we'
* about to do. */
movl lguest_data - __PAGE_OFFSET + LGUEST_DATA_pgdir, %esi movl lguest_data - __PAGE_OFFSET + LGUEST_DATA_pgdir, %esi
/* Copy first 32 entries of page directory to __PAGE_OFFSET entries. /* Copy first 32 entries of page directory to __PAGE_OFFSET entries.
......
...@@ -128,9 +128,12 @@ static void unmap_switcher(void) ...@@ -128,9 +128,12 @@ static void unmap_switcher(void)
__free_pages(switcher_page[i], 0); __free_pages(switcher_page[i], 0);
} }
/*L:305 /*H:032
* Dealing With Guest Memory. * Dealing With Guest Memory.
* *
* Before we go too much further into the Host, we need to grok the routines
* we use to deal with Guest memory.
*
* When the Guest gives us (what it thinks is) a physical address, we can use * When the Guest gives us (what it thinks is) a physical address, we can use
* the normal copy_from_user() & copy_to_user() on the corresponding place in * the normal copy_from_user() & copy_to_user() on the corresponding place in
* the memory region allocated by the Launcher. * the memory region allocated by the Launcher.
......
...@@ -90,6 +90,7 @@ static void do_hcall(struct lguest *lg, struct hcall_args *args) ...@@ -90,6 +90,7 @@ static void do_hcall(struct lguest *lg, struct hcall_args *args)
lg->pending_notify = args->arg1; lg->pending_notify = args->arg1;
break; break;
default: default:
/* It should be an architecture-specific hypercall. */
if (lguest_arch_do_hcall(lg, args)) if (lguest_arch_do_hcall(lg, args))
kill_guest(lg, "Bad hypercall %li\n", args->arg0); kill_guest(lg, "Bad hypercall %li\n", args->arg0);
} }
...@@ -157,7 +158,6 @@ static void do_async_hcalls(struct lguest *lg) ...@@ -157,7 +158,6 @@ static void do_async_hcalls(struct lguest *lg)
* Guest makes a hypercall, we end up here to set things up: */ * Guest makes a hypercall, we end up here to set things up: */
static void initialize(struct lguest *lg) static void initialize(struct lguest *lg)
{ {
/* You can't do anything until you're initialized. The Guest knows the /* You can't do anything until you're initialized. The Guest knows the
* rules, so we're unforgiving here. */ * rules, so we're unforgiving here. */
if (lg->hcall->arg0 != LHCALL_LGUEST_INIT) { if (lg->hcall->arg0 != LHCALL_LGUEST_INIT) {
...@@ -174,7 +174,8 @@ static void initialize(struct lguest *lg) ...@@ -174,7 +174,8 @@ static void initialize(struct lguest *lg)
|| get_user(lg->noirq_end, &lg->lguest_data->noirq_end)) || get_user(lg->noirq_end, &lg->lguest_data->noirq_end))
kill_guest(lg, "bad guest page %p", lg->lguest_data); kill_guest(lg, "bad guest page %p", lg->lguest_data);
/* We write the current time into the Guest's data page once now. */ /* We write the current time into the Guest's data page once so it can
* set its clock. */
write_timestamp(lg); write_timestamp(lg);
/* page_tables.c will also do some setup. */ /* page_tables.c will also do some setup. */
...@@ -182,8 +183,8 @@ static void initialize(struct lguest *lg) ...@@ -182,8 +183,8 @@ static void initialize(struct lguest *lg)
/* This is the one case where the above accesses might have been the /* This is the one case where the above accesses might have been the
* first write to a Guest page. This may have caused a copy-on-write * first write to a Guest page. This may have caused a copy-on-write
* fault, but the Guest might be referring to the old (read-only) * fault, but the old page might be (read-only) in the Guest
* page. */ * pagetable. */
guest_pagetable_clear_all(lg); guest_pagetable_clear_all(lg);
} }
...@@ -220,7 +221,7 @@ void do_hypercalls(struct lguest *lg) ...@@ -220,7 +221,7 @@ void do_hypercalls(struct lguest *lg)
* Normally it doesn't matter: the Guest will run again and * Normally it doesn't matter: the Guest will run again and
* update the trap number before we come back here. * update the trap number before we come back here.
* *
* However, if we are signalled or the Guest sends DMA to the * However, if we are signalled or the Guest sends I/O to the
* Launcher, the run_guest() loop will exit without running the * Launcher, the run_guest() loop will exit without running the
* Guest. When it comes back it would try to re-run the * Guest. When it comes back it would try to re-run the
* hypercall. */ * hypercall. */
......
...@@ -92,8 +92,8 @@ static void set_guest_interrupt(struct lguest *lg, u32 lo, u32 hi, int has_err) ...@@ -92,8 +92,8 @@ static void set_guest_interrupt(struct lguest *lg, u32 lo, u32 hi, int has_err)
/* Remember that we never let the Guest actually disable interrupts, so /* Remember that we never let the Guest actually disable interrupts, so
* the "Interrupt Flag" bit is always set. We copy that bit from the * the "Interrupt Flag" bit is always set. We copy that bit from the
* Guest's "irq_enabled" field into the eflags word: the Guest copies * Guest's "irq_enabled" field into the eflags word: we saw the Guest
* it back in "lguest_iret". */ * copy it back in "lguest_iret". */
eflags = lg->regs->eflags; eflags = lg->regs->eflags;
if (get_user(irq_enable, &lg->lguest_data->irq_enabled) == 0 if (get_user(irq_enable, &lg->lguest_data->irq_enabled) == 0
&& !(irq_enable & X86_EFLAGS_IF)) && !(irq_enable & X86_EFLAGS_IF))
...@@ -124,7 +124,7 @@ static void set_guest_interrupt(struct lguest *lg, u32 lo, u32 hi, int has_err) ...@@ -124,7 +124,7 @@ static void set_guest_interrupt(struct lguest *lg, u32 lo, u32 hi, int has_err)
kill_guest(lg, "Disabling interrupts"); kill_guest(lg, "Disabling interrupts");
} }
/*H:200 /*H:205
* Virtual Interrupts. * Virtual Interrupts.
* *
* maybe_do_interrupt() gets called before every entry to the Guest, to see if * maybe_do_interrupt() gets called before every entry to the Guest, to see if
...@@ -256,19 +256,21 @@ int deliver_trap(struct lguest *lg, unsigned int num) ...@@ -256,19 +256,21 @@ int deliver_trap(struct lguest *lg, unsigned int num)
* bogus one in): if we fail here, the Guest will be killed. */ * bogus one in): if we fail here, the Guest will be killed. */
if (!idt_present(lg->arch.idt[num].a, lg->arch.idt[num].b)) if (!idt_present(lg->arch.idt[num].a, lg->arch.idt[num].b))
return 0; return 0;
set_guest_interrupt(lg, lg->arch.idt[num].a, lg->arch.idt[num].b, has_err(num)); set_guest_interrupt(lg, lg->arch.idt[num].a, lg->arch.idt[num].b,
has_err(num));
return 1; return 1;
} }
/*H:250 Here's the hard part: returning to the Host every time a trap happens /*H:250 Here's the hard part: returning to the Host every time a trap happens
* and then calling deliver_trap() and re-entering the Guest is slow. * and then calling deliver_trap() and re-entering the Guest is slow.
* Particularly because Guest userspace system calls are traps (trap 128). * Particularly because Guest userspace system calls are traps (usually trap
* 128).
* *
* So we'd like to set up the IDT to tell the CPU to deliver traps directly * So we'd like to set up the IDT to tell the CPU to deliver traps directly
* into the Guest. This is possible, but the complexities cause the size of * into the Guest. This is possible, but the complexities cause the size of
* this file to double! However, 150 lines of code is worth writing for taking * this file to double! However, 150 lines of code is worth writing for taking
* system calls down from 1750ns to 270ns. Plus, if lguest didn't do it, all * system calls down from 1750ns to 270ns. Plus, if lguest didn't do it, all
* the other hypervisors would tease it. * the other hypervisors would beat it up at lunchtime.
* *
* This routine indicates if a particular trap number could be delivered * This routine indicates if a particular trap number could be delivered
* directly. */ * directly. */
...@@ -331,7 +333,7 @@ void pin_stack_pages(struct lguest *lg) ...@@ -331,7 +333,7 @@ void pin_stack_pages(struct lguest *lg)
* change stacks on each context switch. */ * change stacks on each context switch. */
void guest_set_stack(struct lguest *lg, u32 seg, u32 esp, unsigned int pages) void guest_set_stack(struct lguest *lg, u32 seg, u32 esp, unsigned int pages)
{ {
/* You are not allowd have a stack segment with privilege level 0: bad /* You are not allowed have a stack segment with privilege level 0: bad
* Guest! */ * Guest! */
if ((seg & 0x3) != GUEST_PL) if ((seg & 0x3) != GUEST_PL)
kill_guest(lg, "bad stack segment %i", seg); kill_guest(lg, "bad stack segment %i", seg);
...@@ -350,7 +352,7 @@ void guest_set_stack(struct lguest *lg, u32 seg, u32 esp, unsigned int pages) ...@@ -350,7 +352,7 @@ void guest_set_stack(struct lguest *lg, u32 seg, u32 esp, unsigned int pages)
* part of the Host: page table handling. */ * part of the Host: page table handling. */
/*H:235 This is the routine which actually checks the Guest's IDT entry and /*H:235 This is the routine which actually checks the Guest's IDT entry and
* transfers it into our entry in "struct lguest": */ * transfers it into the entry in "struct lguest": */
static void set_trap(struct lguest *lg, struct desc_struct *trap, static void set_trap(struct lguest *lg, struct desc_struct *trap,
unsigned int num, u32 lo, u32 hi) unsigned int num, u32 lo, u32 hi)
{ {
...@@ -456,6 +458,18 @@ void copy_traps(const struct lguest *lg, struct desc_struct *idt, ...@@ -456,6 +458,18 @@ void copy_traps(const struct lguest *lg, struct desc_struct *idt,
} }
} }
/*H:200
* The Guest Clock.
*
* There are two sources of virtual interrupts. We saw one in lguest_user.c:
* the Launcher sending interrupts for virtual devices. The other is the Guest
* timer interrupt.
*
* The Guest uses the LHCALL_SET_CLOCKEVENT hypercall to tell us how long to
* the next timer interrupt (in nanoseconds). We use the high-resolution timer
* infrastructure to set a callback at that time.
*
* 0 means "turn off the clock". */
void guest_set_clockevent(struct lguest *lg, unsigned long delta) void guest_set_clockevent(struct lguest *lg, unsigned long delta)
{ {
ktime_t expires; ktime_t expires;
...@@ -466,20 +480,27 @@ void guest_set_clockevent(struct lguest *lg, unsigned long delta) ...@@ -466,20 +480,27 @@ void guest_set_clockevent(struct lguest *lg, unsigned long delta)
return; return;
} }
/* We use wallclock time here, so the Guest might not be running for
* all the time between now and the timer interrupt it asked for. This
* is almost always the right thing to do. */
expires = ktime_add_ns(ktime_get_real(), delta); expires = ktime_add_ns(ktime_get_real(), delta);
hrtimer_start(&lg->hrt, expires, HRTIMER_MODE_ABS); hrtimer_start(&lg->hrt, expires, HRTIMER_MODE_ABS);
} }
/* This is the function called when the Guest's timer expires. */
static enum hrtimer_restart clockdev_fn(struct hrtimer *timer) static enum hrtimer_restart clockdev_fn(struct hrtimer *timer)
{ {
struct lguest *lg = container_of(timer, struct lguest, hrt); struct lguest *lg = container_of(timer, struct lguest, hrt);
/* Remember the first interrupt is the timer interrupt. */
set_bit(0, lg->irqs_pending); set_bit(0, lg->irqs_pending);
/* If the Guest is actually stopped, we need to wake it up. */
if (lg->halted) if (lg->halted)
wake_up_process(lg->tsk); wake_up_process(lg->tsk);
return HRTIMER_NORESTART; return HRTIMER_NORESTART;
} }
/* This sets up the timer for this Guest. */
void init_clockdev(struct lguest *lg) void init_clockdev(struct lguest *lg)
{ {
hrtimer_init(&lg->hrt, CLOCK_REALTIME, HRTIMER_MODE_ABS); hrtimer_init(&lg->hrt, CLOCK_REALTIME, HRTIMER_MODE_ABS);
......
...@@ -74,9 +74,6 @@ struct lguest ...@@ -74,9 +74,6 @@ struct lguest
u32 pgdidx; u32 pgdidx;
struct pgdir pgdirs[4]; struct pgdir pgdirs[4];
/* Cached wakeup: we hold a reference to this task. */
struct task_struct *wake;
unsigned long noirq_start, noirq_end; unsigned long noirq_start, noirq_end;
unsigned long pending_notify; /* pfn from LHCALL_NOTIFY */ unsigned long pending_notify; /* pfn from LHCALL_NOTIFY */
...@@ -103,7 +100,7 @@ int lguest_address_ok(const struct lguest *lg, ...@@ -103,7 +100,7 @@ int lguest_address_ok(const struct lguest *lg,
void __lgread(struct lguest *, void *, unsigned long, unsigned); void __lgread(struct lguest *, void *, unsigned long, unsigned);
void __lgwrite(struct lguest *, unsigned long, const void *, unsigned); void __lgwrite(struct lguest *, unsigned long, const void *, unsigned);
/*L:306 Using memory-copy operations like that is usually inconvient, so we /*H:035 Using memory-copy operations like that is usually inconvient, so we
* have the following helper macros which read and write a specific type (often * have the following helper macros which read and write a specific type (often
* an unsigned long). * an unsigned long).
* *
...@@ -191,7 +188,7 @@ void write_timestamp(struct lguest *lg); ...@@ -191,7 +188,7 @@ void write_timestamp(struct lguest *lg);
* Let's step aside for the moment, to study one important routine that's used * Let's step aside for the moment, to study one important routine that's used
* widely in the Host code. * widely in the Host code.
* *
* There are many cases where the Guest does something invalid, like pass crap * There are many cases where the Guest can do something invalid, like pass crap
* to a hypercall. Since only the Guest kernel can make hypercalls, it's quite * to a hypercall. Since only the Guest kernel can make hypercalls, it's quite
* acceptable to simply terminate the Guest and give the Launcher a nicely * acceptable to simply terminate the Guest and give the Launcher a nicely
* formatted reason. It's also simpler for the Guest itself, which doesn't * formatted reason. It's also simpler for the Guest itself, which doesn't
......
...@@ -53,7 +53,8 @@ struct lguest_device { ...@@ -53,7 +53,8 @@ struct lguest_device {
* Device configurations * Device configurations
* *
* The configuration information for a device consists of a series of fields. * The configuration information for a device consists of a series of fields.
* The device will look for these fields during setup. * We don't really care what they are: the Launcher set them up, and the driver
* will look at them during setup.
* *
* For us these fields come immediately after that device's descriptor in the * For us these fields come immediately after that device's descriptor in the
* lguest_devices page. * lguest_devices page.
...@@ -122,8 +123,8 @@ static void lg_set_status(struct virtio_device *vdev, u8 status) ...@@ -122,8 +123,8 @@ static void lg_set_status(struct virtio_device *vdev, u8 status)
* The other piece of infrastructure virtio needs is a "virtqueue": a way of * The other piece of infrastructure virtio needs is a "virtqueue": a way of
* the Guest device registering buffers for the other side to read from or * the Guest device registering buffers for the other side to read from or
* write into (ie. send and receive buffers). Each device can have multiple * write into (ie. send and receive buffers). Each device can have multiple
* virtqueues: for example the console has one queue for sending and one for * virtqueues: for example the console driver uses one queue for sending and
* receiving. * another for receiving.
* *
* Fortunately for us, a very fast shared-memory-plus-descriptors virtqueue * Fortunately for us, a very fast shared-memory-plus-descriptors virtqueue
* already exists in virtio_ring.c. We just need to connect it up. * already exists in virtio_ring.c. We just need to connect it up.
...@@ -158,7 +159,7 @@ static void lg_notify(struct virtqueue *vq) ...@@ -158,7 +159,7 @@ static void lg_notify(struct virtqueue *vq)
* *
* This is kind of an ugly duckling. It'd be nicer to have a standard * This is kind of an ugly duckling. It'd be nicer to have a standard
* representation of a virtqueue in the configuration space, but it seems that * representation of a virtqueue in the configuration space, but it seems that
* everyone wants to do it differently. The KVM guys want the Guest to * everyone wants to do it differently. The KVM coders want the Guest to
* allocate its own pages and tell the Host where they are, but for lguest it's * allocate its own pages and tell the Host where they are, but for lguest it's
* simpler for the Host to simply tell us where the pages are. * simpler for the Host to simply tell us where the pages are.
* *
...@@ -284,6 +285,8 @@ static void add_lguest_device(struct lguest_device_desc *d) ...@@ -284,6 +285,8 @@ static void add_lguest_device(struct lguest_device_desc *d)
{ {
struct lguest_device *ldev; struct lguest_device *ldev;
/* Start with zeroed memory; Linux's device layer seems to count on
* it. */
ldev = kzalloc(sizeof(*ldev), GFP_KERNEL); ldev = kzalloc(sizeof(*ldev), GFP_KERNEL);
if (!ldev) { if (!ldev) {
printk(KERN_EMERG "Cannot allocate lguest dev %u\n", printk(KERN_EMERG "Cannot allocate lguest dev %u\n",
......
...@@ -8,20 +8,22 @@ ...@@ -8,20 +8,22 @@
#include <linux/fs.h> #include <linux/fs.h>
#include "lg.h" #include "lg.h"
/*L:315 To force the Guest to stop running and return to the Launcher, the /*L:055 When something happens, the Waker process needs a way to stop the
* Waker sets writes LHREQ_BREAK and the value "1" to /dev/lguest. The * kernel running the Guest and return to the Launcher. So the Waker writes
* Launcher then writes LHREQ_BREAK and "0" to release the Waker. */ * LHREQ_BREAK and the value "1" to /dev/lguest to do this. Once the Launcher
* has done whatever needs attention, it writes LHREQ_BREAK and "0" to release
* the Waker. */
static int break_guest_out(struct lguest *lg, const unsigned long __user *input) static int break_guest_out(struct lguest *lg, const unsigned long __user *input)
{ {
unsigned long on; unsigned long on;
/* Fetch whether they're turning break on or off.. */ /* Fetch whether they're turning break on or off. */
if (get_user(on, input) != 0) if (get_user(on, input) != 0)
return -EFAULT; return -EFAULT;
if (on) { if (on) {
lg->break_out = 1; lg->break_out = 1;
/* Pop it out (may be running on different CPU) */ /* Pop it out of the Guest (may be running on different CPU) */
wake_up_process(lg->tsk); wake_up_process(lg->tsk);
/* Wait for them to reset it */ /* Wait for them to reset it */
return wait_event_interruptible(lg->break_wq, !lg->break_out); return wait_event_interruptible(lg->break_wq, !lg->break_out);
...@@ -58,7 +60,7 @@ static ssize_t read(struct file *file, char __user *user, size_t size,loff_t*o) ...@@ -58,7 +60,7 @@ static ssize_t read(struct file *file, char __user *user, size_t size,loff_t*o)
if (!lg) if (!lg)
return -EINVAL; return -EINVAL;
/* If you're not the task which owns the guest, go away. */ /* If you're not the task which owns the Guest, go away. */
if (current != lg->tsk) if (current != lg->tsk)
return -EPERM; return -EPERM;
...@@ -92,8 +94,8 @@ static ssize_t read(struct file *file, char __user *user, size_t size,loff_t*o) ...@@ -92,8 +94,8 @@ static ssize_t read(struct file *file, char __user *user, size_t size,loff_t*o)
* base: The start of the Guest-physical memory inside the Launcher memory. * base: The start of the Guest-physical memory inside the Launcher memory.
* *
* pfnlimit: The highest (Guest-physical) page number the Guest should be * pfnlimit: The highest (Guest-physical) page number the Guest should be
* allowed to access. The Launcher has to live in Guest memory, so it sets * allowed to access. The Guest memory lives inside the Launcher, so it sets
* this to ensure the Guest can't reach it. * this to ensure the Guest can only reach its own memory.
* *
* pgdir: The (Guest-physical) address of the top of the initial Guest * pgdir: The (Guest-physical) address of the top of the initial Guest
* pagetables (which are set up by the Launcher). * pagetables (which are set up by the Launcher).
...@@ -189,7 +191,7 @@ static int initialize(struct file *file, const unsigned long __user *input) ...@@ -189,7 +191,7 @@ static int initialize(struct file *file, const unsigned long __user *input)
} }
/*L:010 The first operation the Launcher does must be a write. All writes /*L:010 The first operation the Launcher does must be a write. All writes
* start with a 32 bit number: for the first write this must be * start with an unsigned long number: for the first write this must be
* LHREQ_INITIALIZE to set up the Guest. After that the Launcher can use * LHREQ_INITIALIZE to set up the Guest. After that the Launcher can use
* writes of other values to send interrupts. */ * writes of other values to send interrupts. */
static ssize_t write(struct file *file, const char __user *in, static ssize_t write(struct file *file, const char __user *in,
...@@ -275,8 +277,7 @@ static int close(struct inode *inode, struct file *file) ...@@ -275,8 +277,7 @@ static int close(struct inode *inode, struct file *file)
* The Launcher is the Host userspace program which sets up, runs and services * The Launcher is the Host userspace program which sets up, runs and services
* the Guest. In fact, many comments in the Drivers which refer to "the Host" * the Guest. In fact, many comments in the Drivers which refer to "the Host"
* doing things are inaccurate: the Launcher does all the device handling for * doing things are inaccurate: the Launcher does all the device handling for
* the Guest. The Guest can't tell what's done by the the Launcher and what by * the Guest, but the Guest can't know that.
* the Host.
* *
* Just to confuse you: to the Host kernel, the Launcher *is* the Guest and we * Just to confuse you: to the Host kernel, the Launcher *is* the Guest and we
* shall see more of that later. * shall see more of that later.
......
This diff is collapsed.
...@@ -12,8 +12,6 @@ ...@@ -12,8 +12,6 @@
#include "lg.h" #include "lg.h"
/*H:600 /*H:600
* We've almost completed the Host; there's just one file to go!
*
* Segments & The Global Descriptor Table * Segments & The Global Descriptor Table
* *
* (That title sounds like a bad Nerdcore group. Not to suggest that there are * (That title sounds like a bad Nerdcore group. Not to suggest that there are
...@@ -55,7 +53,7 @@ static int ignored_gdt(unsigned int num) ...@@ -55,7 +53,7 @@ static int ignored_gdt(unsigned int num)
|| num == GDT_ENTRY_DOUBLEFAULT_TSS); || num == GDT_ENTRY_DOUBLEFAULT_TSS);
} }
/*H:610 Once the GDT has been changed, we fix the new entries up a little. We /*H:630 Once the Guest gave us new GDT entries, we fix them up a little. We
* don't care if they're invalid: the worst that can happen is a General * don't care if they're invalid: the worst that can happen is a General
* Protection Fault in the Switcher when it restores a Guest segment register * Protection Fault in the Switcher when it restores a Guest segment register
* which tries to use that entry. Then we kill the Guest for causing such a * which tries to use that entry. Then we kill the Guest for causing such a
...@@ -84,25 +82,33 @@ static void fixup_gdt_table(struct lguest *lg, unsigned start, unsigned end) ...@@ -84,25 +82,33 @@ static void fixup_gdt_table(struct lguest *lg, unsigned start, unsigned end)
} }
} }
/* This routine is called at boot or modprobe time for each CPU to set up the /*H:610 Like the IDT, we never simply use the GDT the Guest gives us. We keep
* "constant" GDT entries for Guests running on that CPU. */ * a GDT for each CPU, and copy across the Guest's entries each time we want to
* run the Guest on that CPU.
*
* This routine is called at boot or modprobe time for each CPU to set up the
* constant GDT entries: the ones which are the same no matter what Guest we're
* running. */
void setup_default_gdt_entries(struct lguest_ro_state *state) void setup_default_gdt_entries(struct lguest_ro_state *state)
{ {
struct desc_struct *gdt = state->guest_gdt; struct desc_struct *gdt = state->guest_gdt;
unsigned long tss = (unsigned long)&state->guest_tss; unsigned long tss = (unsigned long)&state->guest_tss;
/* The hypervisor segments are full 0-4G segments, privilege level 0 */ /* The Switcher segments are full 0-4G segments, privilege level 0 */
gdt[GDT_ENTRY_LGUEST_CS] = FULL_EXEC_SEGMENT; gdt[GDT_ENTRY_LGUEST_CS] = FULL_EXEC_SEGMENT;
gdt[GDT_ENTRY_LGUEST_DS] = FULL_SEGMENT; gdt[GDT_ENTRY_LGUEST_DS] = FULL_SEGMENT;
/* The TSS segment refers to the TSS entry for this CPU, so we cannot /* The TSS segment refers to the TSS entry for this particular CPU.
* copy it from the Guest. Forgive the magic flags */ * Forgive the magic flags: the 0x8900 means the entry is Present, it's
* privilege level 0 Available 386 TSS system segment, and the 0x67
* means Saturn is eclipsed by Mercury in the twelfth house. */
gdt[GDT_ENTRY_TSS].a = 0x00000067 | (tss << 16); gdt[GDT_ENTRY_TSS].a = 0x00000067 | (tss << 16);
gdt[GDT_ENTRY_TSS].b = 0x00008900 | (tss & 0xFF000000) gdt[GDT_ENTRY_TSS].b = 0x00008900 | (tss & 0xFF000000)
| ((tss >> 16) & 0x000000FF); | ((tss >> 16) & 0x000000FF);
} }
/* This routine is called before the Guest is run for the first time. */ /* This routine sets up the initial Guest GDT for booting. All entries start
* as 0 (unusable). */
void setup_guest_gdt(struct lguest *lg) void setup_guest_gdt(struct lguest *lg)
{ {
/* Start with full 0-4G segments... */ /* Start with full 0-4G segments... */
...@@ -114,13 +120,8 @@ void setup_guest_gdt(struct lguest *lg) ...@@ -114,13 +120,8 @@ void setup_guest_gdt(struct lguest *lg)
lg->arch.gdt[GDT_ENTRY_KERNEL_DS].b |= (GUEST_PL << 13); lg->arch.gdt[GDT_ENTRY_KERNEL_DS].b |= (GUEST_PL << 13);
} }
/* Like the IDT, we never simply use the GDT the Guest gives us. We set up the /*H:650 An optimization of copy_gdt(), for just the three "thead-local storage"
* GDTs for each CPU, then we copy across the entries each time we want to run * entries. */
* a different Guest on that CPU. */
/* A partial GDT load, for the three "thead-local storage" entries. Otherwise
* it's just like load_guest_gdt(). So much, in fact, it would probably be
* neater to have a single hypercall to cover both. */
void copy_gdt_tls(const struct lguest *lg, struct desc_struct *gdt) void copy_gdt_tls(const struct lguest *lg, struct desc_struct *gdt)
{ {
unsigned int i; unsigned int i;
...@@ -129,7 +130,9 @@ void copy_gdt_tls(const struct lguest *lg, struct desc_struct *gdt) ...@@ -129,7 +130,9 @@ void copy_gdt_tls(const struct lguest *lg, struct desc_struct *gdt)
gdt[i] = lg->arch.gdt[i]; gdt[i] = lg->arch.gdt[i];
} }
/* This is the full version */ /*H:640 When the Guest is run on a different CPU, or the GDT entries have
* changed, copy_gdt() is called to copy the Guest's GDT entries across to this
* CPU's GDT. */
void copy_gdt(const struct lguest *lg, struct desc_struct *gdt) void copy_gdt(const struct lguest *lg, struct desc_struct *gdt)
{ {
unsigned int i; unsigned int i;
...@@ -141,7 +144,8 @@ void copy_gdt(const struct lguest *lg, struct desc_struct *gdt) ...@@ -141,7 +144,8 @@ void copy_gdt(const struct lguest *lg, struct desc_struct *gdt)
gdt[i] = lg->arch.gdt[i]; gdt[i] = lg->arch.gdt[i];
} }
/* This is where the Guest asks us to load a new GDT (LHCALL_LOAD_GDT). */ /*H:620 This is where the Guest asks us to load a new GDT (LHCALL_LOAD_GDT).
* We copy it from the Guest and tweak the entries. */
void load_guest_gdt(struct lguest *lg, unsigned long table, u32 num) void load_guest_gdt(struct lguest *lg, unsigned long table, u32 num)
{ {
/* We assume the Guest has the same number of GDT entries as the /* We assume the Guest has the same number of GDT entries as the
...@@ -157,16 +161,22 @@ void load_guest_gdt(struct lguest *lg, unsigned long table, u32 num) ...@@ -157,16 +161,22 @@ void load_guest_gdt(struct lguest *lg, unsigned long table, u32 num)
lg->changed |= CHANGED_GDT; lg->changed |= CHANGED_GDT;
} }
/* This is the fast-track version for just changing the three TLS entries.
* Remember that this happens on every context switch, so it's worth
* optimizing. But wouldn't it be neater to have a single hypercall to cover
* both cases? */
void guest_load_tls(struct lguest *lg, unsigned long gtls) void guest_load_tls(struct lguest *lg, unsigned long gtls)
{ {
struct desc_struct *tls = &lg->arch.gdt[GDT_ENTRY_TLS_MIN]; struct desc_struct *tls = &lg->arch.gdt[GDT_ENTRY_TLS_MIN];
__lgread(lg, tls, gtls, sizeof(*tls)*GDT_ENTRY_TLS_ENTRIES); __lgread(lg, tls, gtls, sizeof(*tls)*GDT_ENTRY_TLS_ENTRIES);
fixup_gdt_table(lg, GDT_ENTRY_TLS_MIN, GDT_ENTRY_TLS_MAX+1); fixup_gdt_table(lg, GDT_ENTRY_TLS_MIN, GDT_ENTRY_TLS_MAX+1);
/* Note that just the TLS entries have changed. */
lg->changed |= CHANGED_GDT_TLS; lg->changed |= CHANGED_GDT_TLS;
} }
/*:*/
/* /*H:660
* With this, we have finished the Host. * With this, we have finished the Host.
* *
* Five of the seven parts of our task are complete. You have made it through * Five of the seven parts of our task are complete. You have made it through
......
...@@ -63,7 +63,7 @@ static struct lguest_pages *lguest_pages(unsigned int cpu) ...@@ -63,7 +63,7 @@ static struct lguest_pages *lguest_pages(unsigned int cpu)
static DEFINE_PER_CPU(struct lguest *, last_guest); static DEFINE_PER_CPU(struct lguest *, last_guest);
/*S:010 /*S:010
* We are getting close to the Switcher. * We approach the Switcher.
* *
* Remember that each CPU has two pages which are visible to the Guest when it * Remember that each CPU has two pages which are visible to the Guest when it
* runs on that CPU. This has to contain the state for that Guest: we copy the * runs on that CPU. This has to contain the state for that Guest: we copy the
...@@ -134,7 +134,7 @@ static void run_guest_once(struct lguest *lg, struct lguest_pages *pages) ...@@ -134,7 +134,7 @@ static void run_guest_once(struct lguest *lg, struct lguest_pages *pages)
* *
* The lcall also pushes the old code segment (KERNEL_CS) onto the * The lcall also pushes the old code segment (KERNEL_CS) onto the
* stack, then the address of this call. This stack layout happens to * stack, then the address of this call. This stack layout happens to
* exactly match the stack of an interrupt... */ * exactly match the stack layout created by an interrupt... */
asm volatile("pushf; lcall *lguest_entry" asm volatile("pushf; lcall *lguest_entry"
/* This is how we tell GCC that %eax ("a") and %ebx ("b") /* This is how we tell GCC that %eax ("a") and %ebx ("b")
* are changed by this routine. The "=" means output. */ * are changed by this routine. The "=" means output. */
...@@ -151,40 +151,46 @@ static void run_guest_once(struct lguest *lg, struct lguest_pages *pages) ...@@ -151,40 +151,46 @@ static void run_guest_once(struct lguest *lg, struct lguest_pages *pages)
} }
/*:*/ /*:*/
/*M:002 There are hooks in the scheduler which we can register to tell when we
* get kicked off the CPU (preempt_notifier_register()). This would allow us
* to lazily disable SYSENTER which would regain some performance, and should
* also simplify copy_in_guest_info(). Note that we'd still need to restore
* things when we exit to Launcher userspace, but that's fairly easy.
*
* The hooks were designed for KVM, but we can also put them to good use. :*/
/*H:040 This is the i386-specific code to setup and run the Guest. Interrupts /*H:040 This is the i386-specific code to setup and run the Guest. Interrupts
* are disabled: we own the CPU. */ * are disabled: we own the CPU. */
void lguest_arch_run_guest(struct lguest *lg) void lguest_arch_run_guest(struct lguest *lg)
{ {
/* Remember the awfully-named TS bit? If the Guest has asked /* Remember the awfully-named TS bit? If the Guest has asked to set it
* to set it we set it now, so we can trap and pass that trap * we set it now, so we can trap and pass that trap to the Guest if it
* to the Guest if it uses the FPU. */ * uses the FPU. */
if (lg->ts) if (lg->ts)
lguest_set_ts(); lguest_set_ts();
/* SYSENTER is an optimized way of doing system calls. We /* SYSENTER is an optimized way of doing system calls. We can't allow
* can't allow it because it always jumps to privilege level 0. * it because it always jumps to privilege level 0. A normal Guest
* A normal Guest won't try it because we don't advertise it in * won't try it because we don't advertise it in CPUID, but a malicious
* CPUID, but a malicious Guest (or malicious Guest userspace * Guest (or malicious Guest userspace program) could, so we tell the
* program) could, so we tell the CPU to disable it before * CPU to disable it before running the Guest. */
* running the Guest. */
if (boot_cpu_has(X86_FEATURE_SEP)) if (boot_cpu_has(X86_FEATURE_SEP))
wrmsr(MSR_IA32_SYSENTER_CS, 0, 0); wrmsr(MSR_IA32_SYSENTER_CS, 0, 0);
/* Now we actually run the Guest. It will pop back out when /* Now we actually run the Guest. It will return when something
* something interesting happens, and we can examine its * interesting happens, and we can examine its registers to see what it
* registers to see what it was doing. */ * was doing. */
run_guest_once(lg, lguest_pages(raw_smp_processor_id())); run_guest_once(lg, lguest_pages(raw_smp_processor_id()));
/* The "regs" pointer contains two extra entries which are not /* Note that the "regs" pointer contains two extra entries which are
* really registers: a trap number which says what interrupt or * not really registers: a trap number which says what interrupt or
* trap made the switcher code come back, and an error code * trap made the switcher code come back, and an error code which some
* which some traps set. */ * traps set. */
/* If the Guest page faulted, then the cr2 register will tell /* If the Guest page faulted, then the cr2 register will tell us the
* us the bad virtual address. We have to grab this now, * bad virtual address. We have to grab this now, because once we
* because once we re-enable interrupts an interrupt could * re-enable interrupts an interrupt could fault and thus overwrite
* fault and thus overwrite cr2, or we could even move off to a * cr2, or we could even move off to a different CPU. */
* different CPU. */
if (lg->regs->trapnum == 14) if (lg->regs->trapnum == 14)
lg->arch.last_pagefault = read_cr2(); lg->arch.last_pagefault = read_cr2();
/* Similarly, if we took a trap because the Guest used the FPU, /* Similarly, if we took a trap because the Guest used the FPU,
...@@ -197,14 +203,15 @@ void lguest_arch_run_guest(struct lguest *lg) ...@@ -197,14 +203,15 @@ void lguest_arch_run_guest(struct lguest *lg)
wrmsr(MSR_IA32_SYSENTER_CS, __KERNEL_CS, 0); wrmsr(MSR_IA32_SYSENTER_CS, __KERNEL_CS, 0);
} }
/*H:130 Our Guest is usually so well behaved; it never tries to do things it /*H:130 Now we've examined the hypercall code; our Guest can make requests.
* isn't allowed to. Unfortunately, Linux's paravirtual infrastructure isn't * Our Guest is usually so well behaved; it never tries to do things it isn't
* quite complete, because it doesn't contain replacements for the Intel I/O * allowed to, and uses hypercalls instead. Unfortunately, Linux's paravirtual
* instructions. As a result, the Guest sometimes fumbles across one during * infrastructure isn't quite complete, because it doesn't contain replacements
* the boot process as it probes for various things which are usually attached * for the Intel I/O instructions. As a result, the Guest sometimes fumbles
* to a PC. * across one during the boot process as it probes for various things which are
* usually attached to a PC.
* *
* When the Guest uses one of these instructions, we get trap #13 (General * When the Guest uses one of these instructions, we get a trap (General
* Protection Fault) and come here. We see if it's one of those troublesome * Protection Fault) and come here. We see if it's one of those troublesome
* instructions and skip over it. We return true if we did. */ * instructions and skip over it. We return true if we did. */
static int emulate_insn(struct lguest *lg) static int emulate_insn(struct lguest *lg)
...@@ -275,43 +282,43 @@ static int emulate_insn(struct lguest *lg) ...@@ -275,43 +282,43 @@ static int emulate_insn(struct lguest *lg)
void lguest_arch_handle_trap(struct lguest *lg) void lguest_arch_handle_trap(struct lguest *lg)
{ {
switch (lg->regs->trapnum) { switch (lg->regs->trapnum) {
case 13: /* We've intercepted a GPF. */ case 13: /* We've intercepted a General Protection Fault. */
/* Check if this was one of those annoying IN or OUT /* Check if this was one of those annoying IN or OUT
* instructions which we need to emulate. If so, we * instructions which we need to emulate. If so, we just go
* just go back into the Guest after we've done it. */ * back into the Guest after we've done it. */
if (lg->regs->errcode == 0) { if (lg->regs->errcode == 0) {
if (emulate_insn(lg)) if (emulate_insn(lg))
return; return;
} }
break; break;
case 14: /* We've intercepted a page fault. */ case 14: /* We've intercepted a Page Fault. */
/* The Guest accessed a virtual address that wasn't /* The Guest accessed a virtual address that wasn't mapped.
* mapped. This happens a lot: we don't actually set * This happens a lot: we don't actually set up most of the
* up most of the page tables for the Guest at all when * page tables for the Guest at all when we start: as it runs
* we start: as it runs it asks for more and more, and * it asks for more and more, and we set them up as
* we set them up as required. In this case, we don't * required. In this case, we don't even tell the Guest that
* even tell the Guest that the fault happened. * the fault happened.
* *
* The errcode tells whether this was a read or a * The errcode tells whether this was a read or a write, and
* write, and whether kernel or userspace code. */ * whether kernel or userspace code. */
if (demand_page(lg, lg->arch.last_pagefault, lg->regs->errcode)) if (demand_page(lg, lg->arch.last_pagefault, lg->regs->errcode))
return; return;
/* OK, it's really not there (or not OK): the Guest /* OK, it's really not there (or not OK): the Guest needs to
* needs to know. We write out the cr2 value so it * know. We write out the cr2 value so it knows where the
* knows where the fault occurred. * fault occurred.
* *
* Note that if the Guest were really messed up, this * Note that if the Guest were really messed up, this could
* could happen before it's done the INITIALIZE * happen before it's done the LHCALL_LGUEST_INIT hypercall, so
* hypercall, so lg->lguest_data will be NULL */ * lg->lguest_data could be NULL */
if (lg->lguest_data && if (lg->lguest_data &&
put_user(lg->arch.last_pagefault, &lg->lguest_data->cr2)) put_user(lg->arch.last_pagefault, &lg->lguest_data->cr2))
kill_guest(lg, "Writing cr2"); kill_guest(lg, "Writing cr2");
break; break;
case 7: /* We've intercepted a Device Not Available fault. */ case 7: /* We've intercepted a Device Not Available fault. */
/* If the Guest doesn't want to know, we already /* If the Guest doesn't want to know, we already restored the
* restored the Floating Point Unit, so we just * Floating Point Unit, so we just continue without telling
* continue without telling it. */ * it. */
if (!lg->ts) if (!lg->ts)
return; return;
break; break;
...@@ -536,9 +543,6 @@ int lguest_arch_init_hypercalls(struct lguest *lg) ...@@ -536,9 +543,6 @@ int lguest_arch_init_hypercalls(struct lguest *lg)
return 0; return 0;
} }
/* Now we've examined the hypercall code; our Guest can make requests. There
* is one other way we can do things for the Guest, as we see in
* emulate_insn(). :*/
/*L:030 lguest_arch_setup_regs() /*L:030 lguest_arch_setup_regs()
* *
...@@ -562,7 +566,7 @@ void lguest_arch_setup_regs(struct lguest *lg, unsigned long start) ...@@ -562,7 +566,7 @@ void lguest_arch_setup_regs(struct lguest *lg, unsigned long start)
* is supposed to always be "1". Bit 9 (0x200) controls whether * is supposed to always be "1". Bit 9 (0x200) controls whether
* interrupts are enabled. We always leave interrupts enabled while * interrupts are enabled. We always leave interrupts enabled while
* running the Guest. */ * running the Guest. */
regs->eflags = 0x202; regs->eflags = X86_EFLAGS_IF | 0x2;
/* The "Extended Instruction Pointer" register says where the Guest is /* The "Extended Instruction Pointer" register says where the Guest is
* running. */ * running. */
...@@ -570,8 +574,8 @@ void lguest_arch_setup_regs(struct lguest *lg, unsigned long start) ...@@ -570,8 +574,8 @@ void lguest_arch_setup_regs(struct lguest *lg, unsigned long start)
/* %esi points to our boot information, at physical address 0, so don't /* %esi points to our boot information, at physical address 0, so don't
* touch it. */ * touch it. */
/* There are a couple of GDT entries the Guest expects when first /* There are a couple of GDT entries the Guest expects when first
* booting. */ * booting. */
setup_guest_gdt(lg); setup_guest_gdt(lg);
} }
...@@ -6,6 +6,37 @@ ...@@ -6,6 +6,37 @@
* are feeling invigorated and refreshed then the next, more challenging stage * are feeling invigorated and refreshed then the next, more challenging stage
* can be found in "make Guest". :*/ * can be found in "make Guest". :*/
/*M:012 Lguest is meant to be simple: my rule of thumb is that 1% more LOC must
* gain at least 1% more performance. Since neither LOC nor performance can be
* measured beforehand, it generally means implementing a feature then deciding
* if it's worth it. And once it's implemented, who can say no?
*
* This is why I haven't implemented this idea myself. I want to, but I
* haven't. You could, though.
*
* The main place where lguest performance sucks is Guest page faulting. When
* a Guest userspace process hits an unmapped page we switch back to the Host,
* walk the page tables, find it's not mapped, switch back to the Guest page
* fault handler, which calls a hypercall to set the page table entry, then
* finally returns to userspace. That's two round-trips.
*
* If we had a small walker in the Switcher, we could quickly check the Guest
* page table and if the page isn't mapped, immediately reflect the fault back
* into the Guest. This means the Switcher would have to know the top of the
* Guest page table and the page fault handler address.
*
* For simplicity, the Guest should only handle the case where the privilege
* level of the fault is 3 and probably only not present or write faults. It
* should also detect recursive faults, and hand the original fault to the
* Host (which is actually really easy).
*
* Two questions remain. Would the performance gain outweigh the complexity?
* And who would write the verse documenting it? :*/
/*M:011 Lguest64 handles NMI. This gave me NMI envy (until I looked at their
* code). It's worth doing though, since it would let us use oprofile in the
* Host when a Guest is running. :*/
/*S:100 /*S:100
* Welcome to the Switcher itself! * Welcome to the Switcher itself!
* *
...@@ -88,7 +119,7 @@ ENTRY(switch_to_guest) ...@@ -88,7 +119,7 @@ ENTRY(switch_to_guest)
// All saved and there's now five steps before us: // All saved and there's now five steps before us:
// Stack, GDT, IDT, TSS // Stack, GDT, IDT, TSS
// And last of all the page tables are flipped. // Then last of all the page tables are flipped.
// Yet beware that our stack pointer must be // Yet beware that our stack pointer must be
// Always valid lest an NMI hits // Always valid lest an NMI hits
...@@ -103,25 +134,25 @@ ENTRY(switch_to_guest) ...@@ -103,25 +134,25 @@ ENTRY(switch_to_guest)
lgdt LGUEST_PAGES_guest_gdt_desc(%eax) lgdt LGUEST_PAGES_guest_gdt_desc(%eax)
// The Guest's IDT we did partially // The Guest's IDT we did partially
// Move to the "struct lguest_pages" as well. // Copy to "struct lguest_pages" as well.
lidt LGUEST_PAGES_guest_idt_desc(%eax) lidt LGUEST_PAGES_guest_idt_desc(%eax)
// The TSS entry which controls traps // The TSS entry which controls traps
// Must be loaded up with "ltr" now: // Must be loaded up with "ltr" now:
// The GDT entry that TSS uses
// Changes type when we load it: damn Intel!
// For after we switch over our page tables // For after we switch over our page tables
// It (as the rest) will be writable no more. // That entry will be read-only: we'd crash.
// (The GDT entry TSS needs
// Changes type when we load it: damn Intel!)
movl $(GDT_ENTRY_TSS*8), %edx movl $(GDT_ENTRY_TSS*8), %edx
ltr %dx ltr %dx
// Look back now, before we take this last step! // Look back now, before we take this last step!
// The Host's TSS entry was also marked used; // The Host's TSS entry was also marked used;
// Let's clear it again, ere we return. // Let's clear it again for our return.
// The GDT descriptor of the Host // The GDT descriptor of the Host
// Points to the table after two "size" bytes // Points to the table after two "size" bytes
movl (LGUEST_PAGES_host_gdt_desc+2)(%eax), %edx movl (LGUEST_PAGES_host_gdt_desc+2)(%eax), %edx
// Clear the type field of "used" (byte 5, bit 2) // Clear "used" from type field (byte 5, bit 2)
andb $0xFD, (GDT_ENTRY_TSS*8 + 5)(%edx) andb $0xFD, (GDT_ENTRY_TSS*8 + 5)(%edx)
// Once our page table's switched, the Guest is live! // Once our page table's switched, the Guest is live!
...@@ -131,7 +162,7 @@ ENTRY(switch_to_guest) ...@@ -131,7 +162,7 @@ ENTRY(switch_to_guest)
// The page table change did one tricky thing: // The page table change did one tricky thing:
// The Guest's register page has been mapped // The Guest's register page has been mapped
// Writable onto our %esp (stack) -- // Writable under our %esp (stack) --
// We can simply pop off all Guest regs. // We can simply pop off all Guest regs.
popl %eax popl %eax
popl %ebx popl %ebx
...@@ -152,16 +183,15 @@ ENTRY(switch_to_guest) ...@@ -152,16 +183,15 @@ ENTRY(switch_to_guest)
addl $8, %esp addl $8, %esp
// The last five stack slots hold return address // The last five stack slots hold return address
// And everything needed to change privilege // And everything needed to switch privilege
// Into the Guest privilege level of 1, // From Switcher's level 0 to Guest's 1,
// And the stack where the Guest had last left it. // And the stack where the Guest had last left it.
// Interrupts are turned back on: we are Guest. // Interrupts are turned back on: we are Guest.
iret iret
// There are two paths where we switch to the Host // We treat two paths to switch back to the Host
// Yet both must save Guest state and restore Host
// So we put the routine in a macro. // So we put the routine in a macro.
// We are on our way home, back to the Host
// Interrupted out of the Guest, we come here.
#define SWITCH_TO_HOST \ #define SWITCH_TO_HOST \
/* We save the Guest state: all registers first \ /* We save the Guest state: all registers first \
* Laid out just as "struct lguest_regs" defines */ \ * Laid out just as "struct lguest_regs" defines */ \
...@@ -194,7 +224,7 @@ ENTRY(switch_to_guest) ...@@ -194,7 +224,7 @@ ENTRY(switch_to_guest)
movl %esp, %eax; \ movl %esp, %eax; \
andl $(~(1 << PAGE_SHIFT - 1)), %eax; \ andl $(~(1 << PAGE_SHIFT - 1)), %eax; \
/* Save our trap number: the switch will obscure it \ /* Save our trap number: the switch will obscure it \
* (The Guest regs are not mapped here in the Host) \ * (In the Host the Guest regs are not mapped here) \
* %ebx holds it safe for deliver_to_host */ \ * %ebx holds it safe for deliver_to_host */ \
movl LGUEST_PAGES_regs_trapnum(%eax), %ebx; \ movl LGUEST_PAGES_regs_trapnum(%eax), %ebx; \
/* The Host GDT, IDT and stack! \ /* The Host GDT, IDT and stack! \
...@@ -210,9 +240,9 @@ ENTRY(switch_to_guest) ...@@ -210,9 +240,9 @@ ENTRY(switch_to_guest)
/* Switch to Host's GDT, IDT. */ \ /* Switch to Host's GDT, IDT. */ \
lgdt LGUEST_PAGES_host_gdt_desc(%eax); \ lgdt LGUEST_PAGES_host_gdt_desc(%eax); \
lidt LGUEST_PAGES_host_idt_desc(%eax); \ lidt LGUEST_PAGES_host_idt_desc(%eax); \
/* Restore the Host's stack where it's saved regs lie */ \ /* Restore the Host's stack where its saved regs lie */ \
movl LGUEST_PAGES_host_sp(%eax), %esp; \ movl LGUEST_PAGES_host_sp(%eax), %esp; \
/* Last the TSS: our Host is complete */ \ /* Last the TSS: our Host is returned */ \
movl $(GDT_ENTRY_TSS*8), %edx; \ movl $(GDT_ENTRY_TSS*8), %edx; \
ltr %dx; \ ltr %dx; \
/* Restore now the regs saved right at the first. */ \ /* Restore now the regs saved right at the first. */ \
...@@ -222,14 +252,15 @@ ENTRY(switch_to_guest) ...@@ -222,14 +252,15 @@ ENTRY(switch_to_guest)
popl %ds; \ popl %ds; \
popl %es popl %es
// Here's where we come when the Guest has just trapped: // The first path is trod when the Guest has trapped:
// (Which trap we'll see has been pushed on the stack). // (Which trap it was has been pushed on the stack).
// We need only switch back, and the Host will decode // We need only switch back, and the Host will decode
// Why we came home, and what needs to be done. // Why we came home, and what needs to be done.
return_to_host: return_to_host:
SWITCH_TO_HOST SWITCH_TO_HOST
iret iret
// We are lead to the second path like so:
// An interrupt, with some cause external // An interrupt, with some cause external
// Has ajerked us rudely from the Guest's code // Has ajerked us rudely from the Guest's code
// Again we must return home to the Host // Again we must return home to the Host
...@@ -238,7 +269,7 @@ deliver_to_host: ...@@ -238,7 +269,7 @@ deliver_to_host:
// But now we must go home via that place // But now we must go home via that place
// Where that interrupt was supposed to go // Where that interrupt was supposed to go
// Had we not been ensconced, running the Guest. // Had we not been ensconced, running the Guest.
// Here we see the cleverness of our stack: // Here we see the trickness of run_guest_once():
// The Host stack is formed like an interrupt // The Host stack is formed like an interrupt
// With EIP, CS and EFLAGS layered. // With EIP, CS and EFLAGS layered.
// Interrupt handlers end with "iret" // Interrupt handlers end with "iret"
...@@ -263,7 +294,7 @@ deliver_to_host: ...@@ -263,7 +294,7 @@ deliver_to_host:
xorw %ax, %ax xorw %ax, %ax
orl %eax, %edx orl %eax, %edx
// Now the address of the handler's in %edx // Now the address of the handler's in %edx
// We call it now: its "iret" takes us home. // We call it now: its "iret" drops us home.
jmp *%edx jmp *%edx
// Every interrupt can come to us here // Every interrupt can come to us here
......
...@@ -18,12 +18,17 @@ ...@@ -18,12 +18,17 @@
#define LHCALL_LOAD_TLS 16 #define LHCALL_LOAD_TLS 16
#define LHCALL_NOTIFY 17 #define LHCALL_NOTIFY 17
#define LGUEST_TRAP_ENTRY 0x1F
#ifndef __ASSEMBLY__
#include <asm/hw_irq.h>
/*G:031 First, how does our Guest contact the Host to ask for privileged /*G:031 First, how does our Guest contact the Host to ask for privileged
* operations? There are two ways: the direct way is to make a "hypercall", * operations? There are two ways: the direct way is to make a "hypercall",
* to make requests of the Host Itself. * to make requests of the Host Itself.
* *
* Our hypercall mechanism uses the highest unused trap code (traps 32 and * Our hypercall mechanism uses the highest unused trap code (traps 32 and
* above are used by real hardware interrupts). Seventeen hypercalls are * above are used by real hardware interrupts). Fifteen hypercalls are
* available: the hypercall number is put in the %eax register, and the * available: the hypercall number is put in the %eax register, and the
* arguments (when required) are placed in %edx, %ebx and %ecx. If a return * arguments (when required) are placed in %edx, %ebx and %ecx. If a return
* value makes sense, it's returned in %eax. * value makes sense, it's returned in %eax.
...@@ -31,20 +36,15 @@ ...@@ -31,20 +36,15 @@
* Grossly invalid calls result in Sudden Death at the hands of the vengeful * Grossly invalid calls result in Sudden Death at the hands of the vengeful
* Host, rather than returning failure. This reflects Winston Churchill's * Host, rather than returning failure. This reflects Winston Churchill's
* definition of a gentleman: "someone who is only rude intentionally". */ * definition of a gentleman: "someone who is only rude intentionally". */
#define LGUEST_TRAP_ENTRY 0x1F
#ifndef __ASSEMBLY__
#include <asm/hw_irq.h>
static inline unsigned long static inline unsigned long
hcall(unsigned long call, hcall(unsigned long call,
unsigned long arg1, unsigned long arg2, unsigned long arg3) unsigned long arg1, unsigned long arg2, unsigned long arg3)
{ {
/* "int" is the Intel instruction to trigger a trap. */ /* "int" is the Intel instruction to trigger a trap. */
asm volatile("int $" __stringify(LGUEST_TRAP_ENTRY) asm volatile("int $" __stringify(LGUEST_TRAP_ENTRY)
/* The call is in %eax (aka "a"), and can be replaced */ /* The call in %eax (aka "a") might be overwritten */
: "=a"(call) : "=a"(call)
/* The other arguments are in %eax, %edx, %ebx & %ecx */ /* The arguments are in %eax, %edx, %ebx & %ecx */
: "a"(call), "d"(arg1), "b"(arg2), "c"(arg3) : "a"(call), "d"(arg1), "b"(arg2), "c"(arg3)
/* "memory" means this might write somewhere in memory. /* "memory" means this might write somewhere in memory.
* This isn't true for all calls, but it's safe to tell * This isn't true for all calls, but it's safe to tell
......
...@@ -12,8 +12,8 @@ ...@@ -12,8 +12,8 @@
#define LG_CLOCK_MAX_DELTA ULONG_MAX #define LG_CLOCK_MAX_DELTA ULONG_MAX
/*G:032 The second method of communicating with the Host is to via "struct /*G:032 The second method of communicating with the Host is to via "struct
* lguest_data". The Guest's very first hypercall is to tell the Host where * lguest_data". Once the Guest's initialization hypercall tells the Host where
* this is, and then the Guest and Host both publish information in it. :*/ * this is, the Guest and Host both publish information in it. :*/
struct lguest_data struct lguest_data
{ {
/* 512 == enabled (same as eflags in normal hardware). The Guest /* 512 == enabled (same as eflags in normal hardware). The Guest
......
#ifndef _ASM_LGUEST_USER #ifndef _LINUX_LGUEST_LAUNCHER
#define _ASM_LGUEST_USER #define _LINUX_LGUEST_LAUNCHER
/* Everything the "lguest" userspace program needs to know. */ /* Everything the "lguest" userspace program needs to know. */
#include <linux/types.h> #include <linux/types.h>
/* They can register up to 32 arrays of lguest_dma. */
#define LGUEST_MAX_DMA 32
/* At most we can dma 16 lguest_dma in one op. */
#define LGUEST_MAX_DMA_SECTIONS 16
/* How many devices? Assume each one wants up to two dma arrays per device. */
#define LGUEST_MAX_DEVICES (LGUEST_MAX_DMA/2)
/* Where the Host expects the Guest to SEND_DMA console output to. */
#define LGUEST_CONSOLE_DMA_KEY 0
/*D:010 /*D:010
* Drivers * Drivers
...@@ -20,7 +10,11 @@ ...@@ -20,7 +10,11 @@
* real devices (think of the damage it could do!) we provide virtual devices. * real devices (think of the damage it could do!) we provide virtual devices.
* We could emulate a PCI bus with various devices on it, but that is a fairly * We could emulate a PCI bus with various devices on it, but that is a fairly
* complex burden for the Host and suboptimal for the Guest, so we have our own * complex burden for the Host and suboptimal for the Guest, so we have our own
* "lguest" bus and simple drivers. * simple lguest bus and we use "virtio" drivers. These drivers need a set of
* routines from us which will actually do the virtual I/O, but they handle all
* the net/block/console stuff themselves. This means that if we want to add
* a new device, we simply need to write a new virtio driver and create support
* for it in the Launcher: this code won't need to change.
* *
* Devices are described by a simplified ID, a status byte, and some "config" * Devices are described by a simplified ID, a status byte, and some "config"
* bytes which describe this device's configuration. This is placed by the * bytes which describe this device's configuration. This is placed by the
...@@ -51,9 +45,9 @@ struct lguest_vqconfig { ...@@ -51,9 +45,9 @@ struct lguest_vqconfig {
/* Write command first word is a request. */ /* Write command first word is a request. */
enum lguest_req enum lguest_req
{ {
LHREQ_INITIALIZE, /* + pfnlimit, pgdir, start, pageoffset */ LHREQ_INITIALIZE, /* + base, pfnlimit, pgdir, start */
LHREQ_GETDMA, /* No longer used */ LHREQ_GETDMA, /* No longer used */
LHREQ_IRQ, /* + irq */ LHREQ_IRQ, /* + irq */
LHREQ_BREAK, /* + on/off flag (on blocks until someone does off) */ LHREQ_BREAK, /* + on/off flag (on blocks until someone does off) */
}; };
#endif /* _ASM_LGUEST_USER */ #endif /* _LINUX_LGUEST_LAUNCHER */
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment