mongoDB's Capped Collections

在mongoDB中有一个非常好用的collection : Capped Collections。
这种Collection可以设置最大的extent size或max documents，除此之外,在auto-FIFO age-out特性方面具有非常好的性能。capped collection自动维护插入顺序,在某些特殊的使用场景中非常有效，如记录日志的场景,某监控软件保留一个月的监控数据可查,在RDBMS中一般可以使用分区表,定期清理历史数据。在mongodb中使用capped collection只要预先设置好capped collection的size与,或max值.后面就不用管了。
范例一 : 固定大小的collection,size指定的大小单位为byte,指定size中包含数据库头的大小.
1. # 创建一个大小为1byte的capped collection(size是预先指派的,normal collection默认情况下是现用现得，当然normal collection也可以预先指派空间):
> db.createCollection('mycappedcol4',{'capped' : true,'size' : 1})   
{ "ok" : 1 }
# 实际存储SIZE = 256 bytes
# db.m.storageSize() - includes free space allocated to this collection 包含剩余空间
> db.mycappedcol4.storageSize()                                   
256
# 实际extents = 1
# _id上没有创建索引
# 最大的允许记录数为2147483647
> db.mycappedcol4.stats()                                         
{
        "ns" : "test.mycappedcol4",
        "count" : 0,
        "size" : 0,
        "avgObjSize" : NaN,
        "storageSize" : 256,
        " numExtents " : 1,
        " nindexes " : 0,
        "lastExtentSize" : 256,
        "paddingFactor" : 1,
        "flags" : 0,
        " totalIndexSize " : 0,
        "indexSizes" : {

        },
        "capped" : 1,
        " max " : 2147483647,
        "ok" : 1
}
> db.mycappedcol4.getIndexes()
[ ]
# 插入测试,没有记录被插入,因为超过了限制.
> db.mycappedcol4.insert({'lastname':'digoal','firstname':'zhou'})
> db.mycappedcol4.find()

2. # 创建一个大小为8194byte的capped collection:
# 实际分配8448 bytes,256 bytes的倍数.
> db.createCollection('mycappedcol5',{'capped' : true,'size' : 8194})
{ "ok" : 1 }
> db.mycappedcol5.storageSize()
8448
> 8448/256
33
> db.mycappedcol5.stats()
{
        "ns" : "test.mycappedcol5",
        "count" : 0,
        "size" : 0,
        "avgObjSize" : NaN,
        "storageSize" : 8448,
        "numExtents" : 1,
        "nindexes" : 0,
        "lastExtentSize" : 8448,
        "paddingFactor" : 1,
        "flags" : 0,
        "totalIndexSize" : 0,
        "indexSizes" : {

        },
        "capped" : 1,
        "max" : 2147483647,
        "ok" : 1
}
# 插入记录测试
> for (i=1;i<100;i++) {
... db.mycappedcol5.insert({'lastname' : 'digoal','firstname' : 'zhou','insertid' : i})
... }
# 已经达到设定的SIZE上限，插入记录99条，已经发生了AGE-OUT，实际存放了81条。
> db.mycappedcol5.count()
81
> db.mycappedcol5.stats()
{
        "ns" : "test.mycappedcol5",
        "count" : 81,
        "size" : 6804,
        "avgObjSize" : 84,
        "storageSize" : 8448,
        "numExtents" : 1,
        "nindexes" : 0,
        "lastExtentSize" : 8448,
        "paddingFactor" : 1,
        "flags" : 0,
        "totalIndexSize" : 0,
        "indexSizes" : {

        },
        "capped" : 1,
        "max" : 2147483647,
        "ok" : 1
}
# 查询,由于capped collection需要追踪插入的顺序，因此find()返回结果的顺序等于插入的顺序,如下
> db.mycappedcol5.find().limit(5)
{ "_id" : ObjectId("4d23cc569ae655253f80e455"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 19 }
{ "_id" : ObjectId("4d23cc569ae655253f80e456"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 20 }
{ "_id" : ObjectId("4d23cc569ae655253f80e457"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 21 }
{ "_id" : ObjectId("4d23cc569ae655253f80e458"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 22 }
{ "_id" : ObjectId("4d23cc569ae655253f80e459"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 23 }
# 如果要按照最近插入的顺序来获得返回结果，可以使用sort({$natural:-1}),注意在normal collection中使用sort({$natural:-1})返回的只是接近的反向顺序结果，但是capped collection是精确的顺序。参考:
# For standard tables, natural order is not particularly useful because, although the order is often close to insertion order, it is not  guaranteed  to be. However, for Capped Collections, natural order is guaranteed to be the insertion order. This can be very useful.
> db.mycappedcol5.find().sort({$natural:-1}).limit(5)
{ "_id" : ObjectId("4d23cc569ae655253f80e4a5"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 99 }
{ "_id" : ObjectId("4d23cc569ae655253f80e4a4"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 98 }
{ "_id" : ObjectId("4d23cc569ae655253f80e4a3"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 97 }
{ "_id" : ObjectId("4d23cc569ae655253f80e4a2"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 96 }
{ "_id" : ObjectId("4d23cc569ae655253f80e4a1"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 95 }
# 达到设定的SIZE上限,再次插入记录，发送AGE-OUT，根据FIFO原理，insertid=19的记录被age-out。
> db.mycappedcol5.insert({'lastname' : 'digoal','firstname' : 'zhou','insertid' : 100})
> db.mycappedcol5.find().sort({$natural:-1}).limit(5)                                  
{ "_id" : ObjectId("4d23cd149ae655253f80e4a6"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 100 }
{ "_id" : ObjectId("4d23cc569ae655253f80e4a5"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 99 }
{ "_id" : ObjectId("4d23cc569ae655253f80e4a4"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 98 }
{ "_id" : ObjectId("4d23cc569ae655253f80e4a3"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 97 }
{ "_id" : ObjectId("4d23cc569ae655253f80e4a2"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 96 }
> db.mycappedcol5.find().limit(5)                                                      
{ "_id" : ObjectId("4d23cc569ae655253f80e456"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 20 }
{ "_id" : ObjectId("4d23cc569ae655253f80e457"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 21 }
{ "_id" : ObjectId("4d23cc569ae655253f80e458"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 22 }
{ "_id" : ObjectId("4d23cc569ae655253f80e459"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 23 }
{ "_id" : ObjectId("4d23cc569ae655253f80e45a"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 24 }
#　更新测试,当更新后的记录大于原有记录存放的空间，被阻止。
> db.mycappedcol5.update({"insertid" : 21},{$set : {"lastname" : "digoallllllllllllllllllllllllll"}})
failing update: objects in a capped ns cannot grow
# 当更新后的记录不大于原有记录存放的空间，被允许。或者说不会发生行迁移的情况下是允许的。
> db.mycappedcol5.update({"insertid" : 21},{$set : {"lastname" : "digoall"}})                        
> db.mycappedcol5.find().limit(4)                                            
{ "_id" : ObjectId("4d23cc569ae655253f80e457"), "firstname" : "zhou", "insertid" : 21, "lastname" : "digoall" }
{ "_id" : ObjectId("4d23cc569ae655253f80e458"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 22 }
{ "_id" : ObjectId("4d23cc569ae655253f80e459"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 23 }
{ "_id" : ObjectId("4d23cc569ae655253f80e45a"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 24 }
# 因此再次插入时，被age-out的记录还是保持和第一次插入的顺序一致，而不是以最后一次更新的顺序为准,下面可以看出insertid=21的记录被age-out
> db.mycappedcol5.insert({'lastname' : 'digoal','firstname' : 'zhou','insertid' : 101})
> db.mycappedcol5.find().limit(5)                                                      
{ "_id" : ObjectId("4d23cc569ae655253f80e458"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 22 }
{ "_id" : ObjectId("4d23cc569ae655253f80e459"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 23 }
{ "_id" : ObjectId("4d23cc569ae655253f80e45a"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 24 }
{ "_id" : ObjectId("4d23cc569ae655253f80e45b"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 25 }
{ "_id" : ObjectId("4d23cc569ae655253f80e45c"), "lastname" : "digoal", "firstname" : "zhou", "insertid" : 26 }
# 删除记录测试,不允许删除
> db.mycappedcol5.remove({"insertid" : 22})
can't remove from a capped collection
# 索引测试,虽然capped collection默认不在_id上创建索引，但是没说不可以创建索引.如下
> db.mycappedcol5.ensureIndex({"insertid" : 1})
> db.mycappedcol5.getIndexes()
[
        {
                "_id" : ObjectId("4d23d1d59ae655253f80e4a9"),
                "ns" : "test.mycappedcol5",
                "key" : {
                        "insertid" : 1
                },
                "name" : "insertid_1"
        }
]
# 执行计划输出测试,走索引
> db.mycappedcol5.find({"insertid" : 80}).explain()
{
        "cursor" : "BtreeCursor insertid_1",
        "nscanned" : 1,
        "nscannedObjects" : 1,
        "n" : 1,
        "millis" : 0,
        "indexBounds" : {
                "insertid" : [
                        [
                                80,
                                80
                        ]
                ]
        }
}
# 走索引
> db.mycappedcol5.find({"insertid" : {$gt : 1}}).explain() 
{
        "cursor" : "BtreeCursor insertid_1",
        "nscanned" : 81,
        "nscannedObjects" : 81,
        "n" : 81,
        "millis" : 0,
        "indexBounds" : {
                "insertid" : [
                        [
                                1,
                                1.7976931348623157e+308
                        ]
                ]
        }
}
# 显然不走索引
> db.mycappedcol5.find({"lastname" : "digoal"}).explain() 
{
        "cursor" : "ForwardCappedCursor",
        "nscanned" : 81,
        "nscannedObjects" : 81,
        "n" : 81,
        "millis" : 0,
        "indexBounds" : {

        }
}
> db.mycappedcol5.ensureIndex({"lastname" : 1})      
#  居然走索引              
> db.mycappedcol5.find({"lastname" : "digoal"}).explain()
{
        "cursor" : "BtreeCursor lastname_1",
        "nscanned" : 81,
        "nscannedObjects" : 81,
        "n" : 81,
        "millis" : 0,
        "indexBounds" : {
                "lastname" : [
                        [
                                "digoal",
                                "digoal"
                        ]
                ]
        }
}

范例二 : 指定最大记录数的capped collection.
1. # 创建一个最大允许记录1000条记录的capped collection
> db.createCollection('mycappedcol6',{'capped' : true,'max' : 1000})                                 
{ "ok" : 1 }
# 默认预先指派 SIZE = 8192 bytes
> db.mycappedcol6.stats()
{
        "ns" : "test.mycappedcol6",
        "count" : 0,
        "size" : 0,
        "avgObjSize" : NaN,
        "storageSize" : 8192,
        "numExtents" : 1,
        "nindexes" : 0,
        "lastExtentSize" : 8192,
        "paddingFactor" : 1,
        "flags" : 0,
        "totalIndexSize" : 0,
        "indexSizes" : {

        },
        "capped" : 1,
        "max" : 1000,
        "ok" : 1
}
# 插入测试
> for (i=1;i<100;i++) {
... db.mycappedcol6.insert({"insertid" : i,"firstname" : "zhou","lastname" : "digoal"})
... }
# 注意到,仅仅存放了78条记录,因此max和size应该结合使用
> db.mycappedcol6.stats()
{
        "ns" : "test.mycappedcol6",
        " count " :  78 ,
        "size" : 6552,
        "avgObjSize" : 84,
        "storageSize" : 8192,
        "numExtents" : 1,
        "nindexes" : 0,
        "lastExtentSize" : 8192,
        "paddingFactor" : 1,
        "flags" : 0,
        "totalIndexSize" : 0,
        "indexSizes" : {

        },
        "capped" : 1,
        "max" : 1000,
        "ok" : 1
}
# 从以上的输出看出平均一条记录占据 84 bytes ,需要存放1000条记录的话至少需要 84000 bytes,还不包括头的大小.
#　以下创建一个84000 bytes的capped collection，仅存放了839条记录.
> db.createCollection('mycappedcol7',{'capped' : true,'max' : 1000,'size' : 84000})
{ "ok" : 1 }
> for (i=1;i<1001;i++) {                                                           
... db.mycappedcol7.insert({"insertid" : i,"firstname" : "zhou","lastname" : "digoal"})
... }
> db.mycappedcol7.stats()
{
        "ns" : "test.mycappedcol7",
        "count" : 839,
        "size" : 70476,
        "avgObjSize" : 84,
        "storageSize" : 84224,
        "numExtents" : 1,
        "nindexes" : 0,
        "lastExtentSize" : 84224,
        "paddingFactor" : 1,
        "flags" : 0,
        "totalIndexSize" : 0,
        "indexSizes" : {

        },
        "capped" : 1,
        "max" : 1000,
        "ok" : 1
}

范例三 :  创建一个预先指派size的normal collection 

> db.createCollection('Iamdigoal',{'capped' : false,'size' : 8192000})

{ "ok" : 1 }

# _id的索引被自动创建

> db.Iamdigoal.getIndexes()                                                

[

{

"name" : "_id_",

"ns" : "test.Iamdigoal",

"key" : {

"_id" : 1

}

}

]

# 预先指派SIZE = 8192256

> db.Iamdigoal.stats()                                                                    

{

"ns" : "test.Iamdigoal",

"count" : 0,

"size" : 0,

"avgObjSize" : NaN,

"storageSize" : 8192256,

"numExtents" : 1,

"nindexes" : 1,

"lastExtentSize" : 8192256,

"paddingFactor" : 1,

"flags" : 1,

"totalIndexSize" : 8192,

"indexSizes" : {

"_id_" : 8192

},

"ok" : 1

}

限制:
1. Capped Collection默认不创建_id的索引.(normal collection默认情况下创建_id的索引)
2. Capped Collection are not shardable
3. 更新限制，超出原因记录大小的记录的更新操作被阻止.
4. Maximum size for a capped collection is currently 1e9 bytes on a thirty-two bit machine. The maximum size of a capped collection on a sixty-four bit machine is constrained only by system resources.
5. If you perform a find()  on the collection with no ordering specified, the objects will always be returned in insertion order.  Reverse order is always retrievable with find().sort({$natural:-1}).
6. 默认情况下指定size的max的默认值 = 2147483647 , 指定max的size的默认值 = 8192 .