理解 QEMU/KVM 和 Ceph（2）：QEMU 的 RBD 块驱动（block driver）

-drive file=/var/lib/nova/instances/cc388037-18dc-4159-896c-2b7180e7dd20/disk,if=none,id=drive-virtio-disk0,format=qcow2,cache=none -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x4,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1

-drive file=rbd:volumes/volume-512c91d8-a4da-4dcf-b5aa-ef43cf25cb3a:id=cinder:key=AQBc4vtV+JywHhAAqX8N+M69PhIJuUzf1mqNAg==:auth_supported=cephx\;none:mon_host=9.115.251.194\:6789\;9.115.251.195\:6789\;9.115.251.218\:6789,if=none,id=drive-virtio-disk1,format=raw,serial=512c91d8-a4da-4dcf-b5aa-ef43cf25cb3a,cache=writeback -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x6,drive=drive-virtio-disk1,id=virtio-disk1 

static BlockDriver bdrv_rbd = {
    .format_name        = "rbd",
    .instance_size      = sizeof(BDRVRBDState),
    .bdrv_needs_filename = true,
    .bdrv_file_open     = qemu_rbd_open,
    .bdrv_close         = qemu_rbd_close,
    .bdrv_create        = qemu_rbd_create,
    .bdrv_has_zero_init = bdrv_has_zero_init_1,
    .bdrv_get_info      = qemu_rbd_getinfo,
    .create_opts        = &qemu_rbd_create_opts,
    .bdrv_getlength     = qemu_rbd_getlength,
    .bdrv_truncate      = qemu_rbd_truncate,
    .protocol_name      = "rbd",
    .bdrv_aio_readv         = qemu_rbd_aio_readv,
    .bdrv_aio_writev        = qemu_rbd_aio_writev,
#ifdef LIBRBD_SUPPORTS_AIO_FLUSH
    .bdrv_aio_flush         = qemu_rbd_aio_flush,
#else
    .bdrv_co_flush_to_disk  = qemu_rbd_co_flush,
#endif
#ifdef LIBRBD_SUPPORTS_DISCARD
    .bdrv_aio_discard       = qemu_rbd_aio_discard,
#endif
    .bdrv_snapshot_create   = qemu_rbd_snap_create,
    .bdrv_snapshot_delete   = qemu_rbd_snap_remove,
    .bdrv_snapshot_list     = qemu_rbd_snap_list,
    .bdrv_snapshot_goto     = qemu_rbd_snap_rollback,
#ifdef LIBRBD_SUPPORTS_INVALIDATE
    .bdrv_invalidate_cache  = qemu_rbd_invalidate_cache,
#endif
};

static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags, Error **errp) # options 参数见下文描述
{
    BDRVRBDState *s = bs->opaque;
    ...
    opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
    qemu_opts_absorb_qdict(opts, options, &local_err);
    ...

    filename = qemu_opt_get(opts, "filename");

    if (qemu_rbd_parsename(filename, pool, sizeof(pool),
                           snap_buf, sizeof(snap_buf),
                           s->name, sizeof(s->name),
                           conf, sizeof(conf), errp) < 0) {
        r = -EINVAL;
        goto failed_opts;
    }

    clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
    r = rados_create(&s->cluster, clientname); #创建一个handle，其中，cluster 是保存 handle 的数据结构，clientname 是访问 ceph 的username
    ...

    s->snap = NULL;
    if (snap_buf[0] != '\0') {
        s->snap = g_strdup(snap_buf);
    }

    /*
     * Fallback to more conservative semantics if setting cache
     * options fails. Ignore errors from setting rbd_cache because the
     * only possible error is that the option does not exist, and
     * librbd defaults to no caching. If write through caching cannot
     * be set up, fall back to no caching.
     */
    if (flags & BDRV_O_NOCACHE) { #当缓存模式为 nocache 时，设置 cluster 中的配置为 '关闭 rbd cache'
        rados_conf_set(s->cluster, "rbd_cache", "false");
    } else { #其它 cache 模式下，设置 cluster handle 中的配置为 '打开 rbd cache'
        rados_conf_set(s->cluster, "rbd_cache", "true");
    }
    if (strstr(conf, "conf=") == NULL) { #当没有制定 ceph 配置文件时，调用 rados_conf_read_file 函数去读取默认的文件来配置 cluster handle。
        /* try default location, but ignore failure */
        rados_conf_read_file(s->cluster, NULL); #默认文件主要为 

• /etc/ceph/ceph.conf

    }
    if (conf[0] != '\0') { 
        r = qemu_rbd_set_conf(s->cluster, conf, errp); #继续将配置保存到 handle cluster，如果包含 'conf=‘，则调用 rados_conf_read_file 函数读取该文件并将其内容保存到 cluster handle
        if (r < 0) {
            goto failed_shutdown;
        }
    }
    r = rados_connect(s->cluster); #使用 cluster handle 连接到 ceph 集群，cluster handle 中的配置只有到此时才得到应用，之前一直在准备它。
    ...
    r = rados_ioctx_create(s->cluster, pool, &s->io_ctx); #创建 ioctx
    ...
    r = rbd_open(s->io_ctx, s->name, &s->image, s->snap); #打开客户机磁盘对应的 Ceph image
    ...
    bs->read_only = (s->snap != NULL); #如果是 snapshot 的，则只读
    qemu_opts_del(opts);
    return 0;
...
}

static int qemu_rbd_set_conf(rados_t cluster, const char *conf, Error **errp)
{
    ...
    buf = g_strdup(conf);
    p = buf;

    while (p) {
        ret = qemu_rbd_next_tok(name, sizeof(name), p,'=', "conf option name", &p, errp);
        ...if (strcmp(name, "conf") == 0) {
            ret = rados_conf_read_file(cluster, value); #如果配置中包括 "conf"，则将其内容读取到 cluster handle。可见，如果配置文件中有 rbd cache 的话，则会覆盖qemu之前所做的设置
            ...
        } else if (strcmp(name, "id") == 0) {
            /* ignore, this is parsed by qemu_rbd_parse_clientname() */
        } else {
            ret = rados_conf_set(cluster, name, value); #将 conf 中的配置保存到 cluster handle
           ...
        }
    }
    g_free(buf);
    return ret;
}

file=rbd:volumes/volume-512c91d8-a4da-4dcf-b5aa-ef43cf25cb3a:id=cinder:key=AQBc4vtV+JywHhAAqX8N+M69PhIJuUzf1mqNAg==:auth_supported=cephx\;none:mon_host=9.115.251.194\:6789\;9.115.251.195\:6789\;9.115.251.218\:6789,if=none,id=drive-virtio-disk1,format=raw,serial=512c91d8-a4da-4dcf-b5aa-ef43cf25cb3a,cache=writeback

$CEPH_CONF (environment variable)
/etc/ceph/ceph.conf
~/.ceph/config
ceph.conf (in the current working directory)

/*设置cache的参数*/ 
    if (flags & BDRV_O_NOCACHE) {
        rados_conf_set(s->cluster, "rbd_cache", "false");
    } else {
        rados_conf_set(s->cluster, "rbd_cache", "true");
    }
    r = rados_connect(s->cluster);     //连接cluster

1: 修改 /etc/ceph/ceph.conf，添加 log file 和 admin socket 
    [global]
    log file = /var/log/ceph/$name.log
    max open files = 131072
    auth cluster required = none
    auth service required = none
    auth client required = none
    rbd cache = true
    debug perfcounter = 20
    admin socket=/var/run/ceph/rbd-$pid.asok

2: 修改 /etc/apparmor.d/abstractions/libvirt-qemu，添加下列行，使得运行 qemu 的用户有权限读写 log 和 admin socket 文件

# for rbd
capability mknod,

# for rbd
/etc/ceph/ceph.conf r,
/var/log/ceph/* rw,
/var/run/ceph/** rw,

3. 重启 libvirt-bin 和 nova-compute 服务
4. boot 一个新的虚机，或者重启一个已经存在的虚机
5. 使用 admin socket: ceph --admin-daemon /var/run/ceph/rbd-12856.asok perf dump

root@compute1:/var/log/ceph# cat /etc/ceph/ceph.conf  | grep 'rbd cache'
rbd cache = true
rbd cache writethrough until flush = true


root@compute1:/var/log/ceph# virsh dumpxml instance-00000068 | grep cache
      <driver name='qemu' type='qcow2' cache='none' discard='unmap'/>
      <driver name='qemu' type='raw' cache='writeback'/>

root@compute1:/var/log/ceph# ceph --admin-daemon rbd-10588.asok config show | grep rbd_cache
    "rbd_cache": "true",
    "rbd_cache_writethrough_until_flush": "true",
    "rbd_cache_size": "33554432",
    "rbd_cache_max_dirty": "25165824",
    "rbd_cache_target_dirty": "16777216",
    "rbd_cache_max_dirty_age": "1",
    "rbd_cache_max_dirty_object": "0", （这是因为 rbd cache writethrough until flush = true 而此时 librbd 还没有收到 flush 操作过）

"rbd_cache_block_writes_upfront": "false",