android qemu-kvm内存管理和IO映射

为什么内存管理和IO映射要放一起呢？因为IO映射有memory map io(MMIO)和port map io(PMIO)两种，其中MMIO和内存管理有关的。

MMIO和普通内存的访问的汇编指令是相同的；PMIO有自己的汇编指令。

kvm如果执行到了PMIO的指令，那么退出状态是KVM_EXIT_IO。

kvm怎么知道某段内存是MMIO，从而退出状态是KVM_EXIT_MMIO，而某段内存是普通内存，由kvm处理缺页，而不退出呢？这是本文想要探究的一个东西。

PS：kvm退出状态是KVM_EXIT_IO或者KVM_EXIT_MMIO，那么将由qemu进行读写虚拟设备IO端口的模拟，如何模拟？

PMIO的看《android qemu-kvm i8254 pit虚拟设备》，MMIO的看《android emulator虚拟设备分析第一篇之battery》。

阅读本文前，需要对MMU，页表，虚拟内存空间，物理内存空间有一些了解。对于kvm的影子页表的工作原理，kvm的vcpu的执行，以及kvm的ioctl有一些了解。

前提知识：

KVM之内存虚拟化：http://royluo.org/2016/03/13/kvm-mmu-virtualization/，重点看GVA，GPA，HVA，HPA，影子页表的东西

kvm api：https://kernel.org/doc/Documentation/virtual/kvm/api.txt，重点看ioctl的东西，并对kvm的使用有所印象

kvm api使用实例：https://lwn.net/Articles/658511/，https://lwn.net/Articles/658512/ 

QEMU-MEMORY-MANAGEMENT.TXT：https://android.googlesource.com/platform/external/qemu.git/+/master/docs/QEMU-MEMORY-MANAGEMENT.TXT

本文使用的android版本是5.1.0，x86的img；host机器是intel x86_64，ubuntu12.04

先来一个大图：

普通内存的申请(external/qemu/hw/i386/pc.c)：

     /*
      * Allocate a single contiguous RAM so that the goldfish
      * framebuffer can work well especially when the frame buffer is
      * large.
      */
     ram_addr = qemu_ram_alloc(NULL, "pc.ram", below_4g_mem_size);
     cpu_register_physical_memory(0, below_4g_mem_size, ram_addr);

由于kvm可以使用硬件提供的EPT影子页表，所以我们只需要将HVA和GPA的关系通过ioctl告知kvm即可(KVM_SET_USER_MEMORY_REGION)。

先介绍一下qemu内存管理中重要的ram_list和RAMBlock

RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };

 typedef struct RAMList {
     QemuMutex mutex;
     unsigned long *dirty_memory[DIRTY_MEMORY_NUM];
     RAMBlock *mru_block;
     QTAILQ_HEAD(ram, RAMBlock) blocks;
     uint32_t version;
 } RAMList;

 typedef struct RAMBlock {
     uint8_t *host;
     ram_addr_t offset;
     ram_addr_t length;
     uint32_t flags;
     char idstr[256];
     /* Reads can take either the iothread or the ramlist lock.
      * Writes must take both locks.
      */
     QTAILQ_ENTRY(RAMBlock) next;
     int fd;
 } RAMBlock;

ram_list有个锁，是RAMBlock的链表，按照length从大到小排序。RAMBlock的host是HVA，offset是qemu自己引入的介于HVA和GPA之间的一层，length是大小，idstr是名称。每一个RAMBlock对应一个slot，也就是内存插槽。

qemu_ram_alloc直接调用qemu_ram_alloc_from_ptr，主要是申请HVA，求取offset(可放下size的最小空闲空间)，设置RAMBlock结构体并按从大到小顺序插入ram_list:

ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
                                   ram_addr_t size, void *host)
{
    RAMBlock *block, *new_block;
    ram_addr_t old_ram_size, new_ram_size;

    //last_ram_offset求取phys_offset空间已用的大小，old_ram_size为phys_offset空间已用页面个数
    old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;

    //页对齐
    size = TARGET_PAGE_ALIGN(size);
    new_block = g_malloc0(sizeof(*new_block));
    new_block->fd = -1;

    /* This assumes the iothread lock is taken here too.  */
    qemu_mutex_lock_ramlist();
    //new_block->mr = mr;

    //在phys_offset空间中寻找可以存放size的最小的空闲空间的位置
    new_block->offset = find_ram_offset(size);
    if (host) {
        new_block->host = host;
        new_block->flags |= RAM_PREALLOC_MASK;
    }
//...else if...xen... 
    else {
//...if...mem_path...
        if (!new_block->host) {
            //申请HVA，此时未分配HGA，HGA是在kvm发现缺页时再分配的
            new_block->host = phys_mem_alloc(size);
            if (!new_block->host) {
                fprintf(stderr, "Cannot set up guest memory '%s': %s\n",
                        name, strerror(errno));
                exit(1);
            }
//...HAXM...

            memory_try_enable_merging(new_block->host, size);
        }
    }
    new_block->length = size;

//...if...dev..
    // 设置RAMBlock的名字
    pstrcat(new_block->idstr, sizeof(new_block->idstr), name);

    /* Keep the list sorted from biggest to smallest block.  */
    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
        if (block->length < new_block->length) {
            break;
        }
    }
    if (block) {
        QTAILQ_INSERT_BEFORE(block, new_block, next);
    } else {
        QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
    }
    ram_list.mru_block = NULL;

    ram_list.version++;
    qemu_mutex_unlock_ramlist();

    //dirty ram
    new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
    if (new_ram_size > old_ram_size) {
        int i;
        for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
            ram_list.dirty_memory[i] =
                bitmap_zero_extend(ram_list.dirty_memory[i],
                                   old_ram_size, new_ram_size);
       }
    }
    cpu_physical_memory_set_dirty_range(new_block->offset, size);

    //无用
    qemu_ram_setup_dump(new_block->host, size);

    if (kvm_enabled())
        //无用
        kvm_setup_guest_memory(new_block->host, size);

    return new_block->offset;
}

cpu_register_physical_memory最终调用的是cpu_register_physical_memory_log，主要对普通内存KVM_SET_USER_MEMORY_REGION，对MMIO不KVM_SET_USER_MEMORY_REGION。然后设置一下普通内存的phys_offset随着每一页的设置而增大，MMIO的phys_offset和io_index相关，不变：

void cpu_register_physical_memory_log(hwaddr start_addr,
                                         ram_addr_t size,
                                         ram_addr_t phys_offset,
                                         ram_addr_t region_offset,
                                         bool log_dirty)
{
    hwaddr addr, end_addr;
    PhysPageDesc *p;
    CPUState *cpu;
    ram_addr_t orig_size = size;
    subpage_t *subpage;

    if (kvm_enabled())
        // 如果是普通内存，那么KVM_SET_USER_MEMORY_REGION
        // 如果是MMIO，那么修改ram_list和RAMBlock，使对应的内存段没有KVM_SET_USER_MEMORY_REGION，其他内存段不变
        kvm_set_phys_mem(start_addr, size, phys_offset);
//...haxm...

    if (phys_offset == IO_MEM_UNASSIGNED) {
        region_offset = start_addr;
    }
    region_offset &= TARGET_PAGE_MASK;
    // 页对齐
    size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
    end_addr = start_addr + (hwaddr)size;

    addr = start_addr;
    do {
        p = phys_page_find(addr >> TARGET_PAGE_BITS);
        if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
            ram_addr_t orig_memory = p->phys_offset;
            hwaddr start_addr2, end_addr2;
// ...if...subpage...
            else {
                p->phys_offset = phys_offset;
                if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
                    (phys_offset & IO_MEM_ROMD))
                    // 普通内存的phys_offset需要增加的
                    // MMIO的phys_offset和io_index相关，不管申请几页，都不变
                    phys_offset += TARGET_PAGE_SIZE;
            }
        } else {
            p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
            p->phys_offset = phys_offset;
            p->region_offset = region_offset;
            if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
                (phys_offset & IO_MEM_ROMD)) {
                // 普通内存的phys_offset需要增加的
                // MMIO的phys_offset和io_index相关，不管申请几页，都不变
                phys_offset += TARGET_PAGE_SIZE;
            } else {
                hwaddr start_addr2, end_addr2;
// ...if...subpage...
            }
        }
        region_offset += TARGET_PAGE_SIZE;
        addr += TARGET_PAGE_SIZE;
    } while (addr != end_addr);

    /* since each CPU stores ram addresses in its TLB cache, we must
       reset the modified entries */
    /* XXX: slow ! */
    CPU_FOREACH(cpu) {
        tlb_flush(cpu->env_ptr, 1);
    }
}

phys_page_find_alloc中实现了每一个页都有对应的PhysPageDesc，主要记录了该页对应的phys_offset，从而可以找到HVA以及io_index。注意l1_phys_map是一个多级页表，不仅仅是两级的。

kvm_set_phys_mem用于分配slot，重新分配slot，普通内存KVM_SET_USER_MEMORY_REGION，MMIO不KVM_SET_USER_MEMORY_REGION

void kvm_set_phys_mem(hwaddr start_addr,
                      ram_addr_t size,
                      ram_addr_t phys_offset)
{
    KVMState *s = kvm_state;
    ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
    KVMSlot *mem, old;
    int err;

// ...页面不对齐的检测...

    /* KVM does not support read-only slots */
    phys_offset &= ~IO_MEM_ROM;

    while (1) {
        // 寻找内存空间有重合的slot，重新分配slot的
        mem = kvm_lookup_overlapping_slot(s, start_addr, size);
        if (!mem) {
            break;
        }

        // 普通内存，且完全重合，不需要重新分配
        if (flags < IO_MEM_UNASSIGNED && start_addr >= mem->start_addr &&
            (start_addr + size <= mem->start_addr + mem->memory_size) &&
            (phys_offset - start_addr == mem->phys_offset - mem->start_addr)) {
            /* The new slot fits into the existing one and comes with
             * identical parameters - nothing to be done. */
            return;
        }

        old = *mem;

        // 需要重新分配，先注销之前的
        /* unregister the overlapping slot */
        mem->memory_size = 0;
        err = kvm_set_user_memory_region(s, mem);
        if (err) {
            fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
                    __func__, strerror(-err));
            abort();
        }

// ......

        // 重合前的一段，重新分配slot
        /* register prefix slot */
        if (old.start_addr < start_addr) {
            mem = kvm_alloc_slot(s);
            mem->memory_size = start_addr - old.start_addr;
            mem->start_addr = old.start_addr;
            mem->phys_offset = old.phys_offset;
            mem->flags = 0;

            err = kvm_set_user_memory_region(s, mem);
            if (err) {
                fprintf(stderr, "%s: error registering prefix slot: %s\n",
                        __func__, strerror(-err));
                abort();
            }
        }

        // 重合后的一段，重新分配slot
        /* register suffix slot */
        if (old.start_addr + old.memory_size > start_addr + size) {
            ram_addr_t size_delta;

            mem = kvm_alloc_slot(s);
            mem->start_addr = start_addr + size;
            size_delta = mem->start_addr - old.start_addr;
            mem->memory_size = old.memory_size - size_delta;
            mem->phys_offset = old.phys_offset + size_delta;
            mem->flags = 0;

            err = kvm_set_user_memory_region(s, mem);
            if (err) {
                fprintf(stderr, "%s: error registering suffix slot: %s\n",
                        __func__, strerror(-err));
                abort();
            }
        }
    }

    /* in case the KVM bug workaround already "consumed" the new slot */
    if (!size)
        return;

    // 重合的一段，重新分配slot，如果是MMIO，则不分配，且不kvm_set_user_memory_region，将缺页且KVM无法处理，导致KVM_EXIT_MMIO
    /* KVM does not need to know about this memory */
    if (flags >= IO_MEM_UNASSIGNED)
        return;

    mem = kvm_alloc_slot(s);
    mem->memory_size = size;
    mem->start_addr = start_addr;
    mem->phys_offset = phys_offset;
    mem->flags = 0;

    // kvm ioctl KVM_SET_USER_MEMORY_REGION
    err = kvm_set_user_memory_region(s, mem);
    if (err) {
        fprintf(stderr, "%s: error registering slot: %s\n", __func__,
                strerror(-err));
        abort();
    }
}

kvm_set_user_memory_region用于普通内存的KVM_SET_USER_MEMORY_REGION，guest_phys_addr就是GPA，userspace_addr就是HVA，需要根据phys_offset，在ram_list中进行RAMBlock的查找和匹配，然后找到RAMBlock->host，也就是HVA：

static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
{
    struct kvm_userspace_memory_region mem;

    mem.slot = slot->slot;
    mem.guest_phys_addr = slot->start_addr;
    mem.memory_size = slot->memory_size;
    mem.userspace_addr = (unsigned long)qemu_get_ram_ptr(slot->phys_offset);
    mem.flags = slot->flags;
    if (s->migration_log) {
        mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
    }
    return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
}

MMIO内存的申请(external/qemu/hw/android/goldfish/device.c)，申请的MMIO都是页对齐的，基本上都是申请了1页。具体内容需要看《android emulator虚拟设备分析第一篇之battery》：

 int goldfish_device_add(struct goldfish_device *dev,
                        CPUReadMemoryFunc **mem_read,
                        CPUWriteMemoryFunc **mem_write,
                        void *opaque)
 {
     int iomemtype;
     goldfish_add_device_no_io(dev);
     iomemtype = cpu_register_io_memory(mem_read, mem_write, opaque);
     cpu_register_physical_memory(dev->base, dev->size, iomemtype);
     return 0;
 }