一、CephFS

ceph FS 即 cephfilesystem，可以实现文件系统共享功能(符合POSIX标准-可移植操作系统接口)，以实现内核中的ceph客户端模块被应用程序调用, 客户端通过ceph协议挂载并使用 ceph 集群作为数据存储服务器。

Ceph FS 需要运行MetaDataServices(MDS)服务，其守护进程为ceph-mds，ceph-mds进程管理与cephFS上存储的文件相关的元数据，并协调对ceph存储集群的访问。

在linux 系统使用ls等操作查看某个目录下的文件的时候，会有保存在磁盘上的分区表记录文件的名称、创建日期、大小、inode及存储位置等元数据信息，在cephfs由于数据是被打散为若干个离散的object进行分布式存储，因此并没有统一保存文件的元数据，而且将文件的元数据保存到一个单独的存储出matedatapool，但是客户端并不能直接访问 matedata pool 中的元数据信息，而是在读写数的时候有MDS(matadataserver)进行处理，读数据的时候有MDS从matedatapool加载元数据然后缓存在内存(用于后期快速响应其它客户端的请求)并返回给客户端，写数据的时候有MDS缓存在内存并同步到matedatapool。

删除部署的时候创建的

root@ceph-node1:~# 
# 将 myfs 文件系统标记为 failed
ceph fs fail myfs
myfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed.
root@ceph-node1:~# 
# 查看 MDS 状态，确认都变成了 failed 或 standby
ceph mds stat
myfs:0/1 2 up:standby, 1 failed

root@ceph-node1:~# 
# 现在应该可以正常删除了
ceph fs rm myfs --yes-i-really-mean-it
If there are active snapshot schedules associated with this file-system, you might see EIO errors in the mgr logs or at the snap-schedule command-line due to the missing file-system. However, these errors are transient and will get auto-resolved.
root@ceph-node1:~# ceph osd pool rm cephfs.myfs.data  cephfs.myfs.data  --yes-i-really-really-mean-it
pool 'cephfs.myfs.data' removed
root@ceph-node1:~# ceph osd pool rm cephfs.myfs.meta  cephfs.myfs.meta --yes-i-really-really-mean-it
pool 'cephfs.myfs.meta' removed
root@ceph-node1:~# ceph orch ls mds
NAME          PORTS  RUNNING  REFRESHED  AGE  PLACEMENT
mds.mycephfs             3/3  7m ago     7m   ceph-node1;ceph-node2;ceph-node3;count:3
mds.myfs                 2/2  7m ago     11d  ceph-node2;ceph-node3;count:2
root@ceph-node1:~# ceph orch rm mds.myfs
Removed service mds.myfs

1.1 创建cephfs

# 创建元数据存储池
root@ceph-node1:~# ceph osd pool create cephfs-metadata 32 32
pool 'cephfs-metadata' created
# 创建数据池
root@ceph-node1:~# ceph osd pool create cephfs-data 32 32
pool 'cephfs-data' created

# 创建新的文件系统使用新池
root@ceph-node1:~# ceph fs new mycephfs cephfs-metadata cephfs-data
  Pool 'cephfs-data' (id '13') has pg autoscale mode 'on' but is not marked as bulk.
  Consider setting the flag by running
    # ceph osd pool set cephfs-data bulk true
new fs with metadata pool 12 and data pool 13

root@ceph-node1:~# ceph fs ls
name: mycephfs, metadata pool: cephfs-metadata, data pools: [cephfs-data ]
root@ceph-node1:~# ceph fs status
mycephfs - 0 clients
========
RANK  STATE            MDS               ACTIVITY     DNS    INOS   DIRS   CAPS
 0    active  myfs.ceph-node2.unogmh  Reqs:    0 /s    10     13     12      0
      POOL         TYPE     USED  AVAIL
cephfs-metadata  metadata   127k  37.8G
  cephfs-data      data       0   37.8G
     STANDBY MDS
myfs.ceph-node3.ydxqjg
MDS version: ceph version 20.2.2 (0fcffee29411e3a38036764817b6e1afc59741cc) tentacle (stable - RelWithDebInfo)

# 部署mds服务
root@ceph-node1:~# ceph orch apply mds mycephfs --placement='3 ceph-node1 ceph-node2 ceph-node3'
Scheduled mds.mycephfs update...


root@ceph-node1:~# ceph fs status mycephfs
mycephfs - 0 clients
========
RANK  STATE              MDS                 ACTIVITY     DNS    INOS   DIRS   CAPS
 0    active  mycephfs.ceph-node3.yttmtf  Reqs:    0 /s    10     13     12      0
      POOL         TYPE     USED  AVAIL
cephfs-metadata  metadata   127k  37.8G
  cephfs-data      data       0   37.8G
       STANDBY MDS
  myfs.ceph-node3.ydxqjg
mycephfs.ceph-node2.wnhrgf
  myfs.ceph-node2.unogmh
mycephfs.ceph-node1.espged
MDS version: ceph version 20.2.2 (0fcffee29411e3a38036764817b6e1afc59741cc) tentacle (stable - RelWithDebInfo)

# 创建subvolumegroup
root@ceph-node1:~# ceph fs subvolumegroup create mycephfs csi

root@ceph-node1:~# ceph fs subvolumegroup ls mycephfs
[
    {
   
        "name": "csi"
    }
]

1.2 创建用户

root@ceph-node1:~# ceph auth add client.k8s mon 'allow r' mds 'allow rw' osd 'allow rwx pool=cephfs-data'
added key for client.k8s
root@ceph-node1:~# ceph auth get client.k8s
[client.k8s]
        key = AQD1oIJqJD6JABAAkTr6cFG4hQnu5a6KKNMfdA==
        caps mds = "allow rw"
        caps mon = "allow r"
        caps osd = "allow rwx pool=cephfs-data"

root@ceph-node1:~# ceph auth get client.k8s -o ceph.client.k8s.keyring
root@ceph-node1:~# ceph auth print-key client.k8s
AQD1oIJqJD6JABAAkTr6cFG4hQnu5a6KKNMfdA==root@ceph-node1:~#
root@ceph-node1:~# ceph auth print-key client.k8s > k8s.key

客户端验证权限

root@client:~# ceph --user k8s -s
  cluster:
    id:     32e044ec-8fb1-11f1-a636-000c29b8ae8b
    health: HEALTH_OK

  services:
    mon: 3 daemons, quorum ceph-node1,ceph-node2,ceph-node3 (age 3h) [leader: ceph-node1]
    mgr: ceph-node1.axywbf(active, since 3h), standbys: ceph-node2.zuwijg
    mds: 1/1 daemons up, 2 standby
    osd: 6 osds: 6 up (since 3h), 6 in (since 11d)
    rgw: 2 daemons active (2 hosts, 1 zones)

  data:
    volumes: 1/1 healthy
    pools:   8 pools, 257 pgs
    objects: 305 objects, 601 KiB
    usage:   325 MiB used, 120 GiB / 120 GiB avail
    pgs:     257 active+clean

root@client:~#

1.3 内核空间挂载ceph-fs

客户端挂载有两种方式，一是内核空间一是用户空间，内核空间挂载需要内核支持ceph模块，用户空间挂载需要安装ceph-fuse

# 客户端通过key挂载
root@client:~# mount -t ceph 10.10.0.230:6789,10.10.0.231:6789,10.10.0.232:6789:/ /data -o name=k8s,secret=AQD1oIJqJD6JABAAkTr6cFG4hQnu5a6KKNMfdA==
root@client:~# df -h
Filesystem                                            Size  Used Avail Use% Mounted on
tmpfs                                                 192M  948K  192M   1% /run
/dev/mapper/ubuntu--vg-ubuntu--lv                      38G  7.3G   29G  21% /
tmpfs                                                 960M     0  960M   0% /dev/shm
tmpfs                                                 5.0M     0  5.0M   0% /run/lock
/dev/sda2                                             2.0G  197M  1.6G  11% /boot
tmpfs                                                 192M   12K  192M   1% /run/user/0
10.10.0.230:6789,10.10.0.231:6789,10.10.0.232:6789:/   38G     0   38G   0% /data

开机挂载

root@client:/etc/ceph# cat /etc/fstab
# /etc/fstab: static file system information.
#
# Use 'blkid' to print the universally unique identifier for a
# device; this may be used with UUID= as a more robust way to name devices
# that works even if disks are added and removed. See fstab(5).
#
# <file system> <mount point>   <type>  <options>       <dump>  <pass>
# / was on /dev/ubuntu-vg/ubuntu-lv during curtin installation
/dev/disk/by-id/dm-uuid-LVM-UcFhhlxaCa8rcob8HkQcZiPIjIw64MyJwyBtJSUOy2QxLhA1Y4DVAWBZlXQZc2cD / ext4 defaults 0 1
# /boot was on /dev/sda2 during curtin installation
/dev/disk/by-uuid/a2b9e3e4-c5d1-49a7-8787-24b6111d7050 /boot ext4 defaults 0 1
10.10.0.230:6789,10.10.0.231:6789,10.10.0.232:6789:/ /data ceph defaults,name=k8s,secretfile=/etc/ceph/k8s.key,_netdev 0 0

二、对象存储网关RadosGW

数据不需要放置在目录层次结构中，而是存在于平面地址空间内的同一级别

应用通过唯一地址来识别每个单独的数据对象

每个对象可包含有助于检索的元数据

通过RESTfulAPI 在应用级别（而非用户级别）进行访问

2.1 RadosGW 对象存储网关简介

RadosGW 是对象存储(OSS,Object Storage Service)的一种访问实现方式，RADOS 网关也称为 Ceph 对象网关、RadosGW、RGW，是一种服务，使客户端能够利用标准对象存储API来访问 Ceph 集群，它支持 AWS S3 和 Swift API，在 ceph 0.8 版本之后使用 Civetweb (https://github.com/civetweb/civetweb ) 的web服务器来响应api请求，客户端使用http/https 协议通过RESTfulAPI 与RGW通信，而RGW则通过librados与ceph集群通信，RGW客户端通过s3或者swiftapi使用RGW用户进行身份验证，然后RGW网关代表用户利用cephx与 ceph 存储进行身份验证。

S3 由Amazon于2006年推出，全称为SimpleStorageService,S3 定义了对象存储，是对象存储事实上的标准，从某种意义上说，S3就是对象存储，对象存储就是S3,它是对象存储市场的霸主，后续的对象存储都是对S3的模仿

2.2 RadosGW 存储特点

通过对象存储网关将数据存储为对象，每个对象除了包含数据，还包含数据自身的元数据。

对象通过ObjectID来检索，无法通过普通文件系统的挂载方式通过文件路径加文件名称操作来直接访问对象，只能通过API来访问，或者第三方客户端（实际上也是对API的封装）。

对象的存储不是垂直的目录树结构，而是存储在扁平的命名空间中，AmazonS3将这个扁平命名空间称为bucket，而swift则将其称为容器。

无论是bucket还是容器，都不能再嵌套(在bucket不能再包含bucket)。

bucket 需要被授权才能访问到，一个帐户可以对多个bucket授权，而权限可以不同。

方便横向扩展、快速检索数据。

不支持客户端挂载,且需要客户端在访问的时候指定文件名称。

不是很适用于文件过于频繁修改及删除的场景。

ceph 使用bucket 作为存储桶(存储空间)，实现对象数据的存储和多用户隔离，数据存储在 bucket 中，用户的权限也是针对bucket进行授权，可以设置用户对不同的bucket拥有不同的权限，以实现权限管理。

2.2.1 bucket 特性

存储空间(bucket)是用于存储对象（Object）的容器，所有的对象都必须隶属于某个存储空间，

可以设置和修改存储空间属性用来控制地域、访问权限、生命周期等，这些属性设置直接作用于该存储空间内所有对象，因此可以通过灵活创建不同的存储空间来完成不同的管理功能。

同一个存储空间的内部是扁平的，没有文件系统的目录等概念，所有的对象都直接隶属于其对应的存储空间。

每个用户可以拥有多个存储空间

存储空间的名称在OSS范围内必须是全局唯一的，一旦创建之后无法修改名称。

存储空间内部的对象数目没有限制。

2.2.2 bucket 命名规范

通用存储桶命名规则 - Amazon Simple Storage Service

只能包括小写字母、数字和短横线（-）。

必须以小写字母或者数字开头和结尾。

长度必须在3-63字节之间。

存储桶名称不能使用用IP地址格式。

Bucket 名称必须全局唯一。

2.3 对象存储访问对比

Amazon S3：提供了user、bucket 和 object 分别表示用户、存储桶和对象，其中bucket隶属于user，可以针对user设置不同bucket的名称空间的访问权限，而且不同用户允许访问相同的bucket。

OpenStack Swift：提供了 user、container 和 object 分别对应于用户、存储桶和对象，不过它还额外为user提供了父级组件account，account用于表示一个项目或租户(OpenStack用户)，因此一个account中可包含一到多个user，它们可共享使用同一组container，并为container 提供名称空间。

RadosGW：提供了user、subuser、bucket 和 object，其中的 user 对应于 S3 的user，而subuser 则对应于Swift的user，不过user和subuser都不支持为bucket提供名称空间，因此，不同用户的存储桶也不允许同名；不过，自Jewel版本起，RadosGW引入了tenant（租户）用于为user 和bucket 提供名称空间，但它是个可选组件，RadosGW基于ACL为不同的用户设置不同的权限控制，如：

Read 读权限

Write 写权限

Readwrite 读写权限

full-control 全部控制权限

查看默认radosgw的存储池信息：
#rgw pool 信息：
.rgw.root： 包含 realm(领域信息)，比如zone和zonegroup。
default.rgw.log： 存储日志信息，用于记录各种log信息。
default.rgw.control： 系统控制池，在有数据更新时，通知其它RGW更新缓存。
default.rgw.meta： 元数据存储池，通过不同的名称空间分别存储不同的rados对象，这些
名称空间包括⽤⼾UID及其bucket映射信息的名称空间users.uid、⽤⼾的密钥名称空间
users.keys、⽤⼾的 email 名称空间 users.email、⽤⼾的 subuser 的名称空间 users.swift，以
及bucket 的名称空间root等。
default.rgw.buckets.index： 存放 bucket 到 object 的索引信息。
default.rgw.buckets.data： 存放对象的数据。
default.rgw.buckets.non-ec #数据的额外信息存储池
default.rgw.users.uid： 存放用户信息的存储池。
default.rgw.data.root： 存放 bucket 的元数据，结构体对应RGWBucketInfo，比如存放桶名、
桶ID、data_pool 等


root@ceph-node1:~# radosgw-admin zone get --rgw-zone=default --rgw-zonegroup=default
{
   
    "id": "e7571b98-2dc3-4d57-87b8-c35a1bd15bd4",
    "name": "default",
    "domain_root": "default.rgw.meta:root",
    "control_pool": "default.rgw.control",
    "dedup_pool": "default.rgw.dedup",
    "gc_pool": "default.rgw.log:gc",
    "lc_pool": "default.rgw.log:lc",
    "log_pool": "default.rgw.log",
    "intent_log_pool": "default.rgw.log:intent",
    "usage_log_pool": "default.rgw.log:usage",
    "roles_pool": "default.rgw.meta:roles",
    "reshard_pool": "default.rgw.log:reshard",
    "user_keys_pool": "default.rgw.meta:users.keys",
    "user_email_pool": "default.rgw.meta:users.email",
    "user_swift_pool": "default.rgw.meta:users.swift",
    "user_uid_pool": "default.rgw.meta:users.uid",
    "otp_pool": "default.rgw.otp",
    "notif_pool": "default.rgw.log:notif",
    "topics_pool": "default.rgw.meta:topics",
    "account_pool": "default.rgw.meta:accounts",
    "group_pool": "default.rgw.meta:groups",
    "bucket_logging_pool": "default.rgw.log:logging",
    "system_key": {
   
        "access_key": "",
        "secret_key": ""
    },
    "placement_pools": [
        {
   
            "key": "default-placement",
            "val": {
   
                "index_pool": "default.rgw.buckets.index",
                "storage_classes": {
   
                    "STANDARD": {
   
                        "data_pool": "default.rgw.buckets.data"
                    }
                },
                "data_extra_pool": "default.rgw.buckets.non-ec",
                "index_type": 0,
                "inline_data": true
            }
        }
    ],
    "realm_id": "0a0b9442-1f77-4ba1-8dc4-f73155fe5539",
    "restore_pool": "default.rgw.log:restore"
}

2.4 RGW 使用部署

删除

ceph orch ls --service_type rgw

ceph orch rm rgw.site-rgw

root@ceph-node1:~# radosgw-admin zone delete --rgw-zone=default
2026-08-18T15:54:54.817+0800 7fc8afa60cc0  0 NOTICE: removed master_zone of zonegroup default
2026-08-18T15:54:54.821+0800 7fc8afa60cc0  0 Removed zone from zonegroup default
root@ceph-node1:~# radosgw-admin zonegroup delete --rgw-zonegroup=default
root@ceph-node1:~# radosgw-admin realm rm --rgw-realm=default
root@ceph-node1:~# ceph tell mon.* injectargs  --mon_allow_pool_delete=true
mon.ceph-node1: {
   }
mon.ceph-node2: {
   }
mon.ceph-node3: {
   }
root@ceph-node1:~# ceph osd pool delete default.rgw.log default.rgw.log --yes-i-really-really-mean-it
ceph osd pool delete default.rgw.control default.rgw.control --yes-i-really-really-mean-it
ceph osd pool delete default.rgw.meta default.rgw.meta --yes-i-really-really-mean-it
pool 'default.rgw.log' removed
pool 'default.rgw.control' removed
pool 'default.rgw.meta' removed
root@ceph-node1:~# ceph tell mon.* injectargs  --mon_allow_pool_delete=false

root@ceph-node1:~# ceph osd pool get cn-wuxi-1a.rgw.control crush_rule
crush_rule: replicated_rule
root@ceph-node1:~# ceph osd pool get cn-wuxi-1a.rgw.control size
size: 3
root@ceph-node1:~# ceph osd pool get cn-wuxi-1a.rgw.control pg_num
pg_num: 32
root@ceph-node1:~# ceph osd pool get cn-wuxi-1a.rgw.control pgp_num
pgp_num: 32

# 创建一个领域   global 
root@ceph-node1:~# radosgw-admin realm create --rgw-realm=cn --default
{
   
    "id": "667d70aa-8b6c-4652-9999-c2672aec5b34",
    "name": "cn",
    "current_period": "2cf5efca-7a8c-4521-a6ab-00cd2dcb6a72",
    "epoch": 1
}

# 创建一个区域组 region
root@ceph-node1:~# radosgw-admin zonegroup  create --rgw-realm=cn --rgw-zonegroup=cn-wuxi --master --default
{
   
    "id": "8bf485be-6d34-429b-ad7d-ab14ca24d5c2",
    "name": "cn-wuxi",
    "api_name": "cn-wuxi",
    "is_master": true,
    "endpoints": [],
    "hostnames": [],
    "hostnames_s3website": [],
    "master_zone": "",
    "zones": [],
    "placement_targets": [
        {
   
            "name": "default-placement",
            "tags": [],
            "storage_classes": []
        }
    ],
    "default_placement": "default-placement",
    "realm_id": "667d70aa-8b6c-4652-9999-c2672aec5b34",
    "sync_policy": {
   
        "groups": []
    },
    "enabled_features": [
        "notification_v2",
        "resharding"
    ]
}

# 创建一个区域     zone
root@ceph-node1:~# radosgw-admin zone create --rgw-realm=cn --rgw-zonegroup=cn-wuxi --rgw-zone=cn-wuxi-1a --master --default
NOTICE: set zone's realm_id=667d70aa-8b6c-4652-9999-c2672aec5b34
{
   
    "id": "77ced7cc-3b3d-4464-8bf2-fa7beaf70548",
    "name": "cn-wuxi-1a",
    "domain_root": "cn-wuxi-1a.rgw.meta:root",
    "control_pool": "cn-wuxi-1a.rgw.control",
    "dedup_pool": "cn-wuxi-1a.rgw.dedup",
    "gc_pool": "cn-wuxi-1a.rgw.log:gc",
    "lc_pool": "cn-wuxi-1a.rgw.log:lc",
    "log_pool": "cn-wuxi-1a.rgw.log",
    "intent_log_pool": "cn-wuxi-1a.rgw.log:intent",
    "usage_log_pool": "cn-wuxi-1a.rgw.log:usage",
    "roles_pool": "cn-wuxi-1a.rgw.meta:roles",
    "reshard_pool": "cn-wuxi-1a.rgw.log:reshard",
    "user_keys_pool": "cn-wuxi-1a.rgw.meta:users.keys",
    "user_email_pool": "cn-wuxi-1a.rgw.meta:users.email",
    "user_swift_pool": "cn-wuxi-1a.rgw.meta:users.swift",
    "user_uid_pool": "cn-wuxi-1a.rgw.meta:users.uid",
    "otp_pool": "cn-wuxi-1a.rgw.otp",
    "notif_pool": "cn-wuxi-1a.rgw.log:notif",
    "topics_pool": "cn-wuxi-1a.rgw.meta:topics",
    "account_pool": "cn-wuxi-1a.rgw.meta:accounts",
    "group_pool": "cn-wuxi-1a.rgw.meta:groups",
    "bucket_logging_pool": "cn-wuxi-1a.rgw.log:logging",
    "system_key": {
   
        "access_key": "",
        "secret_key": ""
    },
    "placement_pools": [
        {
   
            "key": "default-placement",
            "val": {
   
                "index_pool": "cn-wuxi-1a.rgw.buckets.index",
                "storage_classes": {
   
                    "STANDARD": {
   
                        "data_pool": "cn-wuxi-1a.rgw.buckets.data"
                    }
                },
                "data_extra_pool": "cn-wuxi-1a.rgw.buckets.non-ec",
                "index_type": 0,
                "inline_data": true
            }
        }
    ],
    "realm_id": "667d70aa-8b6c-4652-9999-c2672aec5b34",
    "restore_pool": "cn-wuxi-1a.rgw.log:restore"
}

为特性领域和区域部署一组radosgw守护程序

root@ceph-node1:~# ceph orch apply rgw prod-rgw --realm=cn --zone=cn-wuxi-1a --placement='2 ceph-node1 ceph-node2' --port=8000
Scheduled rgw.prod-rgw update...

2.5 客户端

创建用户

root@ceph-node1:~# radosgw-admin user create --uid='user' --display-name='user1'
2026-08-18T16:20:17.373+0800 7937e1d59cc0  0 ERROR: current period 2cf5efca-7a8c-4521-a6ab-00cd2dcb6a72 does not contain zone id 77ced7cc-3b3d-4464-8bf2-fa7beaf70548
{
   
    "user_id": "user",
    "display_name": "user1",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
   
            "user": "user",
            "access_key": "YSLEGQ0KGJQH7IXZOQRH",
            "secret_key": "k12Com6AEoEM1ghzSDrbdqSbhtsZnyuE481ezSx9",
            "active": true,
            "create_date": "2026-08-18T08:20:17.522846Z"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
   
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
   
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": [],
    "account_id": "",
    "path": "/",
    "create_date": "2026-08-18T08:20:17.522599Z",
    "tags": [],
    "group_ids": []
}

root@ceph-node1:~# radosgw-admin period get
{
   
    "id": "2cf5efca-7a8c-4521-a6ab-00cd2dcb6a72",
    "epoch": 1,
    "predecessor_uuid": "",
    "sync_status": [],
    "period_map": {
   
        "id": "2cf5efca-7a8c-4521-a6ab-00cd2dcb6a72",
        "zonegroups": [],
        "short_zone_ids": []
    },
    "master_zonegroup": "",
    "master_zone": "",
    "period_config": {
   
        "bucket_quota": {
   
            "enabled": false,
            "check_on_raw": false,
            "max_size": -1,
            "max_size_kb": 0,
            "max_objects": -1
        },
        "user_quota": {
   
            "enabled": false,
            "check_on_raw": false,
            "max_size": -1,
            "max_size_kb": 0,
            "max_objects": -1
        },
        "user_ratelimit": {
   
            "max_read_ops": 0,
            "max_write_ops": 0,
            "max_read_bytes": 0,
            "max_write_bytes": 0,
            "enabled": false
        },
        "bucket_ratelimit": {
   
            "max_read_ops": 0,
            "max_write_ops": 0,
            "max_read_bytes": 0,
            "max_write_bytes": 0,
            "enabled": false
        },
        "anonymous_ratelimit": {
   
            "max_read_ops": 0,
            "max_write_ops": 0,
            "max_read_bytes": 0,
            "max_write_bytes": 0,
            "enabled": false
        }
    },
    "realm_id": "667d70aa-8b6c-4652-9999-c2672aec5b34",
    "realm_epoch": 1
}

root@ceph-node1:~# radosgw-admin period update --rgw-realm=cn --rgw-zonegroup=cn-wuxi --commit
{
   
    "id": "db99f0d7-a7c5-4794-a60b-a7c77e50b691",
    "epoch": 1,
    "predecessor_uuid": "2cf5efca-7a8c-4521-a6ab-00cd2dcb6a72",
    "sync_status": [],
    "period_map": {
   
        "id": "db99f0d7-a7c5-4794-a60b-a7c77e50b691",
        "zonegroups": [
            {
   
                "id": "8bf485be-6d34-429b-ad7d-ab14ca24d5c2",
                "name": "cn-wuxi",
                "api_name": "cn-wuxi",
                "is_master": true,
                "endpoints": [],
                "hostnames": [],
                "hostnames_s3website": [],
                "master_zone": "77ced7cc-3b3d-4464-8bf2-fa7beaf70548",
                "zones": [
                    {
   
                        "id": "77ced7cc-3b3d-4464-8bf2-fa7beaf70548",
                        "name": "cn-wuxi-1a",
                        "endpoints": [],
                        "log_meta": false,
                        "log_data": false,
                        "bucket_index_max_shards": 11,
                        "read_only": false,
                        "tier_type": "",
                        "sync_from_all": true,
                        "sync_from": [],
                        "redirect_zone": "",
                        "supported_features": [
                            "compress-encrypted",
                            "notification_v2",
                            "resharding"
                        ]
                    }
                ],
                "placement_targets": [
                    {
   
                        "name": "default-placement",
                        "tags": [],
                        "storage_classes": [
                            "STANDARD"
                        ]
                    }
                ],
                "default_placement": "default-placement",
                "realm_id": "667d70aa-8b6c-4652-9999-c2672aec5b34",
                "sync_policy": {
   
                    "groups": []
                },
                "enabled_features": [
                    "notification_v2",
                    "resharding"
                ]
            }
        ],
        "short_zone_ids": [
            {
   
                "key": "77ced7cc-3b3d-4464-8bf2-fa7beaf70548",
                "val": 399264154
            }
        ]
    },
    "master_zonegroup": "8bf485be-6d34-429b-ad7d-ab14ca24d5c2",
    "master_zone": "77ced7cc-3b3d-4464-8bf2-fa7beaf70548",
    "period_config": {
   
        "bucket_quota": {
   
            "enabled": false,
            "check_on_raw": false,
            "max_size": -1,
            "max_size_kb": 0,
            "max_objects": -1
        },
        "user_quota": {
   
            "enabled": false,
            "check_on_raw": false,
            "max_size": -1,
            "max_size_kb": 0,
            "max_objects": -1
        },
        "user_ratelimit": {
   
            "max_read_ops": 0,
            "max_write_ops": 0,
            "max_read_bytes": 0,
            "max_write_bytes": 0,
            "enabled": false
        },
        "bucket_ratelimit": {
   
            "max_read_ops": 0,
            "max_write_ops": 0,
            "max_read_bytes": 0,
            "max_write_bytes": 0,
            "enabled": false
        },
        "anonymous_ratelimit": {
   
            "max_read_ops": 0,
            "max_write_ops": 0,
            "max_read_bytes": 0,
            "max_write_bytes": 0,
            "enabled": false
        }
    },
    "realm_id": "667d70aa-8b6c-4652-9999-c2672aec5b34",
    "realm_epoch": 2
}

2.5.1 S3cmd

apt update && apt install python3-venv python3-full -y
python3 -m venv /opt/s3cmd-env
source /opt/s3cmd-env/bin/activate
pip install s3cmd -i https://mirrors.aliyun.com/pypi/simple/

(s3cmd-env) root@client:~# s3cmd --configure

Enter new values or accept defaults in brackets with Enter.
Refer to user manual for detailed description of all options.

Access key and Secret key are your identifiers for Amazon S3. Leave them empty for using the env variables.
Access Key: YSLEGQ0KGJQH7IXZOQRH
Secret Key: k12Com6AEoEM1ghzSDrbdqSbhtsZnyuE481ezSx9
Default Region [US]:

Use "s3.amazonaws.com" for S3 Endpoint and not modify it to the target Amazon S3.
S3 Endpoint [s3.amazonaws.com]: 10.10.0.230:8000

Use "%(bucket)s.s3.amazonaws.com" to the target Amazon S3. "%(bucket)s" and "%(location)s" vars can be used
if the target S3 system supports dns based buckets.
DNS-style bucket+hostname:port template for accessing a bucket [%(bucket)s.s3.amazonaws.com]: %(bucket)s.10.10.0.230:8000

Encryption password is used to protect your files from reading
by unauthorized persons while in transfer to S3
Encryption password:
Path to GPG program [/usr/bin/gpg]:

When using secure HTTPS protocol all communication with Amazon S3
servers is protected from 3rd party eavesdropping. This method is
slower than plain HTTP, and can only be proxied with Python 2.7 or newer
Use HTTPS protocol [Yes]: no

On some networks all internet access must go through a HTTP proxy.
Try setting it here if you can't connect to S3 directly
HTTP Proxy server name:

New settings:
  Access Key: YSLEGQ0KGJQH7IXZOQRH
  Secret Key: k12Com6AEoEM1ghzSDrbdqSbhtsZnyuE481ezSx9
  Default Region: US
  S3 Endpoint: 10.10.0.230:8000
  DNS-style bucket+hostname:port template for accessing a bucket: [%(bucket)s.10.10.0.230:8000
  Encryption password:
  Path to GPG program: /usr/bin/gpg
  Use HTTPS protocol: False
  HTTP Proxy server name:
  HTTP Proxy server port: 0

Test access with supplied credentials? [Y/n] y
Please wait, attempting to list all buckets...
Success. Your access key and secret key worked fine :-)

Now verifying that encryption works...
Not configured. Never mind.

Save settings? [y/N] y
Configuration saved to '/root/.s3cfg'

s3cmd --help

(s3cmd-env) root@client:~# s3cmd rb s3://y99
Bucket 's3://y99/' removed
(s3cmd-env) root@client:~# s3cmd mb s3://hmk
Bucket 's3://hmk/' created
(s3cmd-env) root@client:~# s3cmd ls
2026-08-18 08:42  s3://hmk

(s3cmd-env) root@client:~# s3cmd rb s3://hmk
Bucket 's3://hmk/' removed
(s3cmd-env) root@client:~# s3cmd put hmk_flag s3://hmk
WARNING: Module python-magic is not available. Guessing MIME types based on file extensions.
upload: 'hmk_flag' -> 's3://hmk/hmk_flag'  [1 of 1]
 0 of 0     0% in    2s     0.00 B/s  done

(s3cmd-env) root@client:~# s3cmd get  s3://hmk/hmk_flag /root
download: 's3://hmk/hmk_flag' -> '/root/hmk_flag'  [1 of 1]
 0 of 0     0% in    0s     0.00 B/s  done
(s3cmd-env) root@client:~#

2.5.2 CS Browser （S3 Browser）

CS Browser 是一款功能强大的 Windows 客户端软件，能够与 Amazon S3、Amazon S3 Glacier 以及其他兼容的云存储服务进行协作。它让您可以通过熟悉的 Windows 界面来管理存储桶和文件：上传下载数据、管理权限、组织对象，以及处理各种存储任务，而无需切换到命令行工具进行操作！

2.6 授权

https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/userguide/example-bucket-policies.html

Resources-授权的目的Buckets、objects等资源，必须指定。
Actions: 要授予的动作,CreateBucket、 DeleteObject、 GetObject、 PutObject。必须指定
Effect: #要授予的操作效果是允许(allow)还是拒绝(deny)，默认为拒绝访问所有资源，必须指定。
Principal: 要授权的目的账号，必须指定，
Condition: 授权策略生效的条件，比如访问的TLS版本等，非必须、可不写。

"Condition": {

"NumericLessThan": {

"s3:TlsVersion": 1.2

}

}

权限集合：

https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/API/API_Operations.html

(s3cmd-env) root@client:~# s3cmd info s3://hmk
s3://hmk/ (bucket):
   Location:  cn-wuxi
   Payer:     BucketOwner
   Ownership: none
   Versioning:none
   Expiration rule: none
   Block Public Access: none
   Policy:    none
   CORS:      none
   ACL:       user1: FULL_CONTROL

(s3cmd-env) root@client:~# vi hmkbucket-single_policy
(s3cmd-env) root@client:~# s3cmd setpolicy  hmkbucket-single_policy s3://hmk
s3://hmk/: Policy updated

(s3cmd-env) root@client:~# s3cmd info s3://hmk
s3://hmk/ (bucket):
   Location:  cn-wuxi
   Payer:     BucketOwner
   Ownership: none
   Versioning:none
   Expiration rule: none
   Block Public Access: none
   Policy:    {
   
"Version": "2012-10-17",
"Statement": [{
   
"Effect": "Allow",
"Principal": "*",
"Action": "s3:GetObject",
"Resource": [
  "arn:aws:s3:::hmk/*"
 ]
 }]
}

   CORS:      none
   ACL:       user1: FULL_CONTROL

#  可以正常访问图片
http://10.10.0.230:8000/hmk/99999999999999.png

未授权存储桶

(s3cmd-env) root@client:~# s3cmd mb s3://videos
Bucket 's3://videos/' created
(s3cmd-env) root@client:~# s3cmd put 22.mp4 s3://videos
WARNING: Module python-magic is not available. Guessing MIME types based on file extensions.
upload: '22.mp4' -> 's3://videos/22.mp4'  [1 of 1]
 744888 of 744888   100% in    0s    23.09 MB/s  done

访问失败 匿名用户无权限
http://10.10.0.230:8000/videos/22.mp4
<Error>
<Code>AccessDenied</Code>
<Message>missing s3:GetObject permission</Message>
<BucketName>videos</BucketName>
<RequestId>tx000002fabd9b3b8147c94-006a8505c7-135477-cn-wuxi-1a</RequestId>
<HostId>135477-cn-wuxi-1a-cn-wuxi</HostId>
</Error>

授权
(s3cmd-env) root@client:~# s3cmd setpolicy hmk-videos_policy s3://videos
s3://videos/: Policy updated

Cephfs&&RGW

一、CephFS

1.1 创建cephfs

1.2 创建用户

1.3 内核空间挂载ceph-fs

二、对象存储网关RadosGW

2.1 RadosGW 对象存储网关简介

2.2 RadosGW 存储特点

2.2.1 bucket 特性

2.2.2 bucket 命名规范

2.3 对象存储访问对比

2.4 RGW 使用部署

2.5 客户端

2.5.1 S3cmd

2.5.2 CS Browser （S3 Browser）

2.6 授权

热门文章

最新文章

相关电子书

探索云世界

热门

云计算

大数据

云原生

人工智能

数据库

开发与运维

活动广场

任务中心

训练营

直播

乘风者计划

下载

镜像站

技术资料

Cephfs&&RGW

一、CephFS

1.1 创建cephfs

1.2 创建用户

1.3 内核空间挂载ceph-fs

二、 对象存储网关RadosGW

2.1 RadosGW 对象存储网关简介

2.2 RadosGW 存储特点

2.2.1 bucket 特性

2.2.2 bucket 命名规范

2.3 对象存储访问对比

2.4 RGW 使用 部署

2.5 客户端

2.5.1 S3cmd

2.5.2 CS Browser （S3 Browser）

2.6 授权

热门文章

最新文章

相关电子书

二、对象存储网关RadosGW

2.4 RGW 使用部署