mongoDB's GridFS used with replicaSet and sharding

版本: 
mongodb 1.6.5 x64 bin

环境图:
 
看到这个图是不是有点像RAID1/0的存储方式.

描述:
Server1 : 172.16.3.174
/app/mongodb1.6.5/bin/mongod --config /app/mongodb1.6.5/conf/mongod1953.conf --shardsvr --replSet rep/172.16.3.176:1954
/app/mongodb1.6.5/bin/mongod --config /app/mongodb1.6.5/conf/mongod1954.conf --shardsvr --replSet rep1/172.16.3.176:1953
Server2 : 172.16.3.176
/app/mongodb1.6.5/bin/mongod --config /app/mongodb1.6.5/conf/mongod1954.conf --shardsvr --replSet rep/172.16.3.174:1953
/app/mongodb1.6.5/bin/mongod --config /app/mongodb1.6.5/conf/mongod1953.conf --shardsvr --replSet rep1/172.16.3.174:1954
Server3 : 172.16.3.33
/app/mongodb1.6.5/bin/mongod --config /app/mongodb1.6.5/conf/mongod1953.conf --shardsvr --replSet rep2/172.16.3.39:1954
/app/mongodb1.6.5/bin/mongod --config /app/mongodb1.6.5/conf/mongod1954.conf --shardsvr --replSet rep3/172.16.3.39:1953
Server4 : 172.16.3.39
/app/mongodb1.6.5/bin/mongod --config /app/mongodb1.6.5/conf/mongod1954.conf --shardsvr --replSet rep2/172.16.3.33:1953
/app/mongodb1.6.5/bin/mongod --config /app/mongodb1.6.5/conf/mongod1953.conf --shardsvr --replSet rep3/172.16.3.33:1954
/app/mongodb1.6.5/bin/mongos --port 1956 --configdb 172.16.3.67:1955 --fork --logpath /var/log/mongo/mongodb_1956.log --logappend
Server5 : 172.16.3.67
/app/mongodb1.6.5/bin/mongod --config /app/mongodb1.6.5/conf/mongod1955.conf --configsvr

配置文件 : 
[root@db6 conf]# cat mongod1953.conf 
port=1953
fork=true
logpath=/var/log/mongo/mongodb_1953.log
logappend=true
dbpath=/mongodata/data1953
directoryperdb=true
noauth=true
nohttpinterface=true
nssize=128
oplogSize=512
maxConns=1000

[root@db6 conf]# cat mongod1954.conf 
port=1954
fork=true
logpath=/var/log/mongo/mongodb_1954.log
logappend=true
dbpath=/mongodata/data1954
directoryperdb=true
noauth=true
nohttpinterface=true
nssize=128
oplogSize=512
maxConns=1000

[root@kefu conf]# cat mongod1955.conf 
port=1955
fork=true
logpath=/var/log/mongo/mongodb_1955.log
logappend=true
dbpath=/mongodata/data1955
directoryperdb=true
noauth=true
nohttpinterface=true
nssize=128
oplogSize=512
maxConns=1000

# 初始化replica
> db.runCommand({"replSetInitiate" : {
... "_id" : "rep1",                     
... "members" : [
... {"_id" : 1,"host" : "172.16.3.174:1953"},
... {"_id" : 2,"host" : "172.16.3.176:1954"}]}})
{
        "info" : "Config now saved locally.  Should come online in about a minute.",
        "ok" : 1
}

> db.runCommand({"replSetInitiate" : {
... "_id" : "rep1",                     
... "members" : [
... {"_id" : 1,"host" : "172.16.3.176:1953"},
... {"_id" : 2,"host" : "172.16.3.174:1954"}]}})
{
        "info" : "Config now saved locally.  Should come online in about a minute.",
        "ok" : 1
}

> db.runCommand({"replSetInitiate" : {
... "_id" : "rep2",                     
... "members" : [
... {"_id" : 1,"host" : "172.16.3.33:1953"},
... {"_id" : 2,"host" : "172.16.3.39:1954"}]}})
{
        "info" : "Config now saved locally.  Should come online in about a minute.",
        "ok" : 1
}

> db.runCommand({"replSetInitiate" : {
... "_id" : "rep3",                     
... "members" : [
... {"_id" : 1,"host" : "172.16.3.39:1953"},
... {"_id" : 2,"host" : "172.16.3.33:1954"}]}})
{
        "info" : "Config now saved locally.  Should come online in about a minute.",
        "ok" : 1
}

# 上传文件
/app/mongodb1.6.5/bin/mongofiles -h 172.16.3.39:1956 -d test put ./SLES-11-SP1-DVD-x86_64-GM-DVD2.iso

# 允许数据库shard和collection shard
> 连接到mongos admin库,允许gridfs的files和chunks collection sharding
> use admin
> db.runCommand({"enablesharding" : "test"})
> db.runCommand({"shardcollection" : "test.fs.files", "key" : {"_id" : 1}})
> db.runCommand({"shardcollection" : "test.fs.chunks", "key" : {"_id" : 1}})

# 查看shard状态
> db.printShardingStatus()                             
--- Sharding Status --- 
  sharding version: { "_id" : 1, "version" : 3 }
  shards:
      { "_id" : "rep", "host" : "rep/172.16.3.174:1953" }
      { "_id" : "rep1", "host" : "rep1/172.16.3.176:1953" }
      { "_id" : "rep2", "host" : "rep2/172.16.3.33:1953" }
      { "_id" : "rep3", "host" : "rep3/172.16.3.39:1953" }
  databases:
        { "_id" : "admin", "partitioned" : false, "primary" : "config" }

# 查看gridfs上传文件后的状态
> db.fs.chunks.totalSize()
8905203712
> db.fs.chunks.stats()    
{
        "sharded" : true,
        "ns" : "test.fs.chunks",
        "count" : 23949,
        "size" : 6279589664,
        "avgObjSize" : 262206.7586955614,
        "storageSize" : 9065091584,
        "nindexes" : 2,
        "nchunks" : 27,
        "shards" : {
                "rep" : {
                        "ns" : "test.fs.chunks",
                        "count" : 10158,
                        "size" : 2663364768,
                        "avgObjSize" : 262193.81453041936,
                        "storageSize" : 4957868032,
                        "numExtents" : 31,
                        "nindexes" : 2,
                        "lastExtentSize" : 829803520,
                        "paddingFactor" : 1,
                        "flags" : 1,
                        "totalIndexSize" : 933888,
                        "indexSizes" : {
                                "_id_" : 434176,
                                "files_id_1_n_1" : 499712
                        },
                        "ok" : 1
                },
                "rep1" : {
                        "ns" : "test.fs.chunks",
                        "count" : 7865,
                        "size" : 2062311264,
                        "avgObjSize" : 262213.7652892562,
                        "storageSize" : 2380098048,
                        "numExtents" : 27,
                        "nindexes" : 2,
                        "lastExtentSize" : 400175616,
                        "paddingFactor" : 1,
                        "flags" : 1,
                        "totalIndexSize" : 827392,
                        "indexSizes" : {
                                "_id_" : 385024,
                                "files_id_1_n_1" : 442368
                        },
                        "ok" : 1
                },
                "rep2" : {
                        "ns" : "test.fs.chunks",
                        "count" : 3046,
                        "size" : 798730912,
                        "avgObjSize" : 262222.88640840445,
                        "storageSize" : 943973632,
                        "numExtents" : 22,
                        "nindexes" : 2,
                        "lastExtentSize" : 160821760,
                        "paddingFactor" : 1,
                        "flags" : 1,
                        "totalIndexSize" : 294912,
                        "indexSizes" : {
                                "_id_" : 139264,
                                "files_id_1_n_1" : 155648
                        },
                        "ok" : 1
                },
                "rep3" : {
                        "ns" : "test.fs.chunks",
                        "count" : 2880,
                        "size" : 755182720,
                        "avgObjSize" : 262216.22222222225,
                        "storageSize" : 783151872,
                        "numExtents" : 21,
                        "nindexes" : 2,
                        "lastExtentSize" : 134018304,
                        "paddingFactor" : 1,
                        "flags" : 1,
                        "totalIndexSize" : 278528,
                        "indexSizes" : {
                                "_id_" : 131072,
                                "files_id_1_n_1" : 147456
                        },
                        "ok" : 1
                }
        },
        "ok" : 1
}
> exit  


注意事项 : 
1. mongos和config db不能在同一台服务器，否则在addshard的时候会报如下错误
> db.runCommand({"addshard" : "rep/db5.product.sky-mobi.com.sh:1953"})
{
        "ok" : 0,
        "errmsg" : "can't use localhost as a shard since all shards need to communicate. either use all shards and configdbs in localhost or all in actual IPs "
}

2. shard的replicaSet的名字不能一样，否则会报如下错误 : 
> db.runCommand({"addshard" : "rep/172.16.3.174:1953"})
{ "shardAdded" : "rep", "ok" : 1 }
# 再次添加同一replica名字的shard,报错
> db.runCommand({"addshard" : "rep/172.16.3.176:1953"})
{
        "ok" : 0,
        "errmsg" : "E11000 duplicate key error index: config.shards.$_id_  dup key: { : \"rep\" }"
}
# 因此这里使用了rep , rep1 , rep2 , rep3

3. sharding 支持自动探测 replicaSet的存活状态，支持识别replicaSet的failover.