atom feed6 messages in com.xensource.lists.xen-devel[Xen-devel] GFX Passthrough
FromSent OnAttachments
Tobias GeigerMay 10, 2010 7:35 am.Other, .Other, .Other, 2 more
listmailMay 10, 2010 8:39 am 
Tobias GeigerMay 10, 2010 8:57 am 
Weald WindMay 10, 2010 12:25 pm 
Pasi KärkkäinenMay 10, 2010 12:52 pm 
Pasi KärkkäinenMay 10, 2010 12:53 pm 
Subject:[Xen-devel] GFX Passthrough
From:Tobias Geiger (tobi@vido.info)
Date:May 10, 2010 7:35:55 am
List:com.xensource.lists.xen-devel
Attachments:
05_sound-makefile - 0.5k
04_hvmloader - 3k
02_makefile - 0.6k
03_dsdt - 2k
01_pass-through - 4k

Hi List,

many People seem to be interested in the Graphic-Card Passthrough Feature (for more or less obvious reasons).

Official Support is still under development, and i hope not to interfere with it in any (bad) way ...

But i remember my own painfull and timeconsuming research when i wanted this feature to work, so i thought perhaps this spares some time for the unpatient users like me :)

So here is what i did to enable Passthrough of my Secondary PCIE Graphic-Card which is a NVIDIA GT200;

Key thing seems to be the Patches 1 to 4 (Patch #5 is only so that i have alsa-backed sound emulation , for some reason i have only oss-backed if not applied ?!)

Before all that, you need to read out the VGA Bios of the to-be-passthroughed- Graphiccard with a tool; i used nvflash.exe under DOS for this task. The patches assume you saved this file as "vgabios-pt.bin"

1. check out xen unstable: hg clone http://xenbits.xensource.com/xen-unstable.hg 2. go into there and do an "initial build" so that qemu-dm gets checked-out: cd xen-unstable.hg; cd tools; make ; make clean 3. apply the patches from within the "xen-unstable.hg" dir, with patch -p0 4. move the vgabios-pt.bin to "tools/firmware/vgabios/" 4. do a "make install"

Thats it.

Warning: a "make clean" deletes the "vgabios-pt.bin" file , so remember to put it there again before your next "make" ...

After that, and assuming your hardware/bios is VT-D ready and your XEN-PCI Config is correct (kernel-cmdline, xen-configfile), the HVM guest should be able to work with the passedthrough-Graphicscard.

Greetings, Good luck, and THANK YOU XEN-GUYS! Tobias

P.S.: NONE of the patches are my origin. they were gathered from here (xen- devel) and endless google-researches, representing the hard brainwork of much smarter guys than me ;)

--- tools/ioemu-remote/xen-setup.org 2010-03-15 12:07:28.650764455 +0100 +++ tools/ioemu-remote/xen-setup 2010-03-15 12:06:35.217764207 +0100 @@ -16,7 +16,7 @@ if test -z "${XEN_SCRIPT_DIR}"; then XEN_SCRIPT_DIR="/etc/xen/scripts" fi

-./configure --disable-gfx-check --disable-curses --disable-slirp "$@"
--prefix=${PREFIX} +./configure --audio-drv-list=alsa --enable-mixemu --disable-gfx-check
--disable-curses --disable-slirp "$@" --prefix=${PREFIX}

target=i386-dm

--- tools/firmware/hvmloader/hvmloader.c.org 2010-03-15 11:59:29.517930657 +0100 +++ tools/firmware/hvmloader/hvmloader.c 2010-03-15 12:04:59.486764339 +0100 @@ -115,6 +115,9 @@ unsigned long pci_mem_end = PCI_MEM_END;

static enum { VGA_none, VGA_std, VGA_cirrus, VGA_pt } virtual_vga = VGA_none;

+/* virtual BDF of pass-throughed gfx */ +static uint8_t gfx_bdf; + static void init_hypercalls(void) { uint32_t eax, ebx, ecx, edx; @@ -217,6 +220,42 @@ static void pci_setup(void) virtual_vga = VGA_cirrus; else if ( virtual_vga == VGA_none ) virtual_vga = VGA_pt; + gfx_bdf = devfn; + + /* Make vBAR=pBAR */ + printf("Make vBAR = pBAR of assigned gfx\n"); + for ( bar = 0; bar < 7; bar++ ) + { + bar_reg = PCI_BASE_ADDRESS_0 + 4*bar; + if ( bar == 6 ) + bar_reg = PCI_ROM_ADDRESS; + /* When first time read, it will return physical address */ + bar_data = pci_readl(devfn, bar_reg); + pci_writel(devfn, bar_reg, bar_data); + + /* Now enable the memory or I/O mapping. */ + cmd = pci_readw(devfn, PCI_COMMAND); + if ( (bar_reg == PCI_ROM_ADDRESS) || + ((bar_data & PCI_BASE_ADDRESS_SPACE) == + PCI_BASE_ADDRESS_SPACE_MEMORY) ) + cmd |= PCI_COMMAND_MEMORY; + else + cmd |= PCI_COMMAND_IO; + cmd |= PCI_COMMAND_MASTER; + pci_writew(devfn, PCI_COMMAND, cmd); + } + + /* Map the interrupt. */ + pin = pci_readb(devfn, PCI_INTERRUPT_PIN); + if ( pin != 0 ) + { + /* This is the barber's pole mapping used by Xen. */ + link = ((pin - 1) + (devfn >> 3)) & 3; + isa_irq = pci_readb(PCI_ISA_DEVFN, 0x60 + link); + pci_writeb(devfn, PCI_INTERRUPT_LINE, isa_irq); + } + continue; + break; case 0x0680: /* PIIX4 ACPI PM. Special device with special PCI config space. */ @@ -690,8 +729,10 @@ int main(void) break; case VGA_pt: printf("Loading VGABIOS of passthroughed gfx ...\n"); - vgabios_sz = - round_option_rom((*(uint8_t *)(VGABIOS_PHYSICAL_ADDRESS+2)) * 512); + memcpy((void *)VGABIOS_PHYSICAL_ADDRESS, + vgabios_pt, sizeof(vgabios_pt)); + *(uint8_t *)(VGABIOS_PHYSICAL_ADDRESS + sizeof(vgabios_pt)) = gfx_bdf; + vgabios_sz = round_option_rom(sizeof(vgabios_pt) + 1); break; default: printf("No emulated VGA adaptor ...\n");

--- tools/firmware/hvmloader/Makefile.org 2010-03-15 11:59:04.617764082 +0100 +++ tools/firmware/hvmloader/Makefile 2010-03-15 11:58:55.936761608 +0100 @@ -50,6 +50,7 @@ hvmloader: $(OBJS) acpi/acpi.a roms.h: ../rombios/BIOS-bochs-latest ../vgabios/VGABIOS-lgpl-latest.bin \ ../vgabios/VGABIOS-lgpl-latest.cirrus.bin ../etherboot/eb-roms.h sh ./mkhex rombios ../rombios/BIOS-bochs-latest > roms.h + sh ./mkhex vgabios_pt ../vgabios/vgabios-pt.bin >> roms.h sh ./mkhex vgabios_stdvga ../vgabios/VGABIOS-lgpl-latest.bin >> roms.h sh ./mkhex vgabios_cirrusvga \ ../vgabios/VGABIOS-lgpl-latest.cirrus.bin >> roms.h

--- tools/firmware/hvmloader/acpi/dsdt.asl.orig 2010-03-15 06:50:35.791762654
+0100 +++ tools/firmware/hvmloader/acpi/dsdt.asl 2010-03-15 12:03:37.579845691 +0100 @@ -173,6 +173,34 @@ DefinitionBlock ("DSDT.aml", "DSDT", 2, 0x00000000, 0x00020000)

+ /* reserve MMIO BARs of gfx for 1:1 mapping */ + DWordMemory( + ResourceProducer, PosDecode, MinFixed, MaxFixed, + Cacheable, ReadWrite, + 0x00000000, + 0xE0000000, + 0xEFFFFFFF, + 0x00000000, + 0x10000000) + + DWordMemory( + ResourceProducer, PosDecode, MinFixed, MaxFixed, + NonCacheable, ReadWrite, + 0x00000000, + 0xC0000000, + 0xC1FFFFFF, + 0x00000000, + 0x02000000) + + DWordMemory( + ResourceProducer, PosDecode, MinFixed, MaxFixed, + NonCacheable, ReadWrite, + 0x00000000, + 0xC2000000, + 0xC2FFFFFF, + 0x00000000, + 0x01000000) + DWordMemory( ResourceProducer, PosDecode, MinFixed, MaxFixed, Cacheable, ReadWrite,

--- tools/ioemu-remote/hw/pass-through.c.org 2010-05-10 15:24:52.115083489 +0200 +++ tools/ioemu-remote/hw/pass-through.c 2010-05-10 16:02:19.517997970 +0200 @@ -1865,6 +1865,75 @@ return rc; }

+#define PCI_HEADER_TYPE_ADDR 0x0e +#define PCI_BRIDGE_FLAG 0x01 + +#define PCI_CLASS_CODE_ADDR_0 0x09 +#define PCI_CLASS_CODE_ADDR_1 0x0a +#define PCI_CLASS_CODE_ADDR_2 0x0b +#define PCI_CLASS_CODE_DATA_0 0x060000 +#define PCI_CLASS_CODE_DATA_1 0x0600 +#define PCI_CLASS_CODE_DATA_2 0x06 + +#define PCI_SECOND_BUS_NUMBER_ADDR 0x19 + +#define PCI_BRIDGE_CONTROL_ADDR 0x3e +#define PCI_BRIDGE_VGA_ENABLE 0x18 + +#define PCI_GRAPHIC_CONTROL_ADDR 0x52 +#define PCI_HOST_BRIDGE_IGD_VGA_DISABLE 0x02 + +/* + * Claim vga cycle for the graphics card pass-through + */ +static uint32_t gfx_claim_vga_cycle(struct pci_access *pci_access, + uint32_t bus, uint32_t devfn, uint32_t func) +{ + struct pci_dev *pci_dev; + + for ( pci_dev = pci_access->devices; pci_dev != NULL; pci_dev =
pci_dev->next ) + { + /* Check whether this is a ordinary bridge */ + if ( pci_read_byte(pci_dev, PCI_HEADER_TYPE_ADDR) == PCI_BRIDGE_FLAG ) + { + unsigned sec_bus_num = pci_read_byte(pci_dev,
PCI_SECOND_BUS_NUMBER_ADDR); + unsigned ubrg = pci_read_byte(pci_dev, PCI_BRIDGE_CONTROL_ADDR); + + PT_LOG("bridge for bus %d, previous bridge control is %x\n",
sec_bus_num, ubrg); + PT_LOG("bus=0x%d, dev=0x%x,
func=0x%x\n",pci_dev->bus,pci_dev->dev,pci_dev->func); + + if ( sec_bus_num == bus ) /* VGA device's bridge */ + ubrg |= PCI_BRIDGE_VGA_ENABLE; + else /* Other device's bridge */ + ubrg &= ~PCI_BRIDGE_VGA_ENABLE; + + pci_write_byte(pci_dev, PCI_BRIDGE_CONTROL_ADDR, ubrg); + PT_LOG("bridge for bus %d, updated bridge control is %x\n",
sec_bus_num, ubrg); + } + } + + for ( pci_dev = pci_access->devices; pci_dev != NULL; pci_dev =
pci_dev->next ) + { + /* Check host bridge */ + if ( pci_read_word(pci_dev, PCI_CLASS_CODE_ADDR_1) ==
PCI_CLASS_CODE_DATA_1 ) + { + unsigned uigd = pci_read_byte(pci_dev, PCI_GRAPHIC_CONTROL_ADDR); + + PT_LOG("previous igd control is %x\n", uigd); + + if ( bus == 0 ) + uigd &= ~PCI_HOST_BRIDGE_IGD_VGA_DISABLE; + else + uigd |= PCI_HOST_BRIDGE_IGD_VGA_DISABLE; + + pci_write_byte(pci_dev, PCI_GRAPHIC_CONTROL_ADDR, uigd); + PT_LOG("updated igd control is %x\n", uigd); + } + } + + return 0; +} + /* * register VGA resources for the domain with assigned gfx */ @@ -3298,6 +3367,8 @@ }

/* read BAR */ +static int gfx_first_read_BAR[7] = {1, 1, 1, 1, 1, 1, 1}; + static int pt_bar_reg_read(struct pt_dev *ptdev, struct pt_reg_tbl *cfg_entry, uint32_t *value, uint32_t valid_mask) @@ -3320,6 +3391,17 @@ /* use fixed-up value from kernel sysfs */ *value = ptdev->pci_dev->base_addr[index];

+ if ( ptdev->pci_dev->device_class == 0x300 ) + { + if ( gfx_first_read_BAR[index] == 1 ) + { + gfx_first_read_BAR[index] = 0; + PT_LOG("first read BARs of gfx\n"); + return 0; + } + } + + /* set emulate mask depend on BAR flag */ switch (ptdev->bases[index].bar_flag) { @@ -4397,6 +4479,13 @@ } }

+ if ( pci_dev->device_class == 0x0300 ) + { + rc = gfx_claim_vga_cycle(pci_access, r_bus, r_dev, r_func); + if ( rc != 0 ) + return NULL; + } + /* reinitialize each config register to be emulated */ rc = pt_config_init(assigned_device); if ( rc < 0 ) {