Libevent中的timeout事件是使用最小堆来管理维护的.代码位于<minheap-internal.h>.
源码来源：https://github.com/libevent/libevent/blob/release-2.1.8-stable/minheap-internal.h
本篇实现Linux网络库epoll+时间堆实现高性能定时器，采用C语言实现
一、初衷
如何用epoll实现多个定时器的操作是本设计的关键，我们知道，epoll_wait的最后一个参数是阻塞等待的时候，单位是毫秒。可以这样设计：
1、当时间堆中没有定时器时，epoll_wait的超时时间T设为-1，表示一直阻塞等待新用户的到来；
2、当时间堆中有定时器时，epoll_wait的超时时间T设为最小堆堆顶的超时值，这样可以保证让最近触发的定时器能得以执行；
3、在epoll_wait阻塞等待期间，若有其它的用户到来，则epoll_wait返回n>0，进行常规的处理，随后应重新设置epoll_wait为小顶堆堆顶的超时时间。
二、源码
1、CMakeLists.txt
cmake_minimum_required(VERSION 2.8)
PROJECT(min_heap_libevent_epoll)
AUX_SOURCE_DIRECTORY(. SRC_LIST)
ADD_EXECUTABLE(${PROJECT_NAME} ${SRC_LIST})
2、minheap-event-firecat.h -- 与第1篇不同，回调函数int (*ev_callback)(void *arg);返回值我改成了int型
#ifndef MINHEAPEVENTFIRECAT_H #define MINHEAPEVENTFIRECAT_H #include <sys/time.h> #include <time.h> #include <stdio.h> #include <stdlib.h> //come from https://github.com/libevent/libevent/blob/release-2.1.8-stable/mm-internal.h #define mm_malloc(sz) malloc(sz) #define mm_calloc(n, sz) calloc((n), (sz)) #define mm_strdup(s) strdup(s) #define mm_realloc(p, sz) realloc((p), (sz)) #define mm_free(p) free(p) //come from https://github.com/libevent/libevent/blob/release-2.1.8-stable/include/event2/util.h #define evutil_timercmp(tvp, uvp, cmp) \ (((tvp)->tv_sec == (uvp)->tv_sec) ? \ ((tvp)->tv_usec cmp (uvp)->tv_usec) : \ ((tvp)->tv_sec cmp (uvp)->tv_sec)) #define evutil_timersub(tvp, uvp, vvp) \ do { \ (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \ (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec; \ if ((vvp)->tv_usec < 0) { \ (vvp)->tv_sec--; \ (vvp)->tv_usec += 1000000; \ } \ } while (0) #define evutil_timeradd(tvp, uvp, vvp) \ do { \ (vvp)->tv_sec = (tvp)->tv_sec + (uvp)->tv_sec; \ (vvp)->tv_usec = (tvp)->tv_usec + (uvp)->tv_usec; \ if ((vvp)->tv_usec >= 1000000) { \ (vvp)->tv_sec++; \ (vvp)->tv_usec -= 1000000; \ } \ } while (0) //come from https://github.com/libevent/libevent/blob/release-2.1.8-stable/include/event2/event_struct.h struct event { /* for managing timeouts */ union { //TAILQ_ENTRY(event) ev_next_with_common_timeout; int min_heap_idx; } ev_timeout_pos; unsigned int timer_id; struct timeval ev_interval; struct timeval ev_timeout; int ev_exe_num; int (*ev_callback)(void *arg); int ev_arg; int ev_res; /* result passed to event callback */ int ev_flags; }; static inline void gettime(struct timeval *tm); void gettime(struct timeval *tm) { gettimeofday(tm, NULL); } #endif // MINHEAPEVENTFIRECAT_H
3、minheap-internal.h
/* * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Copyright (c) 2006 Maxim Yegorushkin <maxim.yegorushkin@gmail.com> * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef MINHEAP_INTERNAL_H_INCLUDED_ #define MINHEAP_INTERNAL_H_INCLUDED_ //come from https://github.com/libevent/libevent/blob/release-2.1.8-stable/minheap-internal.h /*firecat del #include "event2/event-config.h" #include "evconfig-private.h" #include "event2/event.h" #include "event2/event_struct.h" #include "event2/util.h" #include "util-internal.h" #include "mm-internal.h" */ #include "minheap-event-firecat.h" //firecat add typedef struct min_heap { struct event** p; unsigned n, a; } min_heap_t; static inline void min_heap_ctor_(min_heap_t* s); static inline void min_heap_dtor_(min_heap_t* s); static inline void min_heap_elem_init_(struct event* e); static inline int min_heap_elt_is_top_(const struct event *e); static inline int min_heap_empty_(min_heap_t* s); static inline unsigned min_heap_size_(min_heap_t* s); static inline struct event* min_heap_top_(min_heap_t* s); static inline int min_heap_reserve_(min_heap_t* s, unsigned n); static inline int min_heap_push_(min_heap_t* s, struct event* e); static inline struct event* min_heap_pop_(min_heap_t* s); static inline int min_heap_adjust_(min_heap_t *s, struct event* e); static inline int min_heap_erase_(min_heap_t* s, struct event* e); static inline void min_heap_shift_up_(min_heap_t* s, unsigned hole_index, struct event* e); static inline void min_heap_shift_up_unconditional_(min_heap_t* s, unsigned hole_index, struct event* e); static inline void min_heap_shift_down_(min_heap_t* s, unsigned hole_index, struct event* e); #define min_heap_elem_greater(a, b) \ (evutil_timercmp(&(a)->ev_timeout, &(b)->ev_timeout, >)) void min_heap_ctor_(min_heap_t* s) { s->p = 0; s->n = 0; s->a = 0; } void min_heap_dtor_(min_heap_t* s) { if (s->p) mm_free(s->p); } void min_heap_elem_init_(struct event* e) { e->ev_timeout_pos.min_heap_idx = -1; } int min_heap_empty_(min_heap_t* s) { return 0u == s->n; } unsigned min_heap_size_(min_heap_t* s) { return s->n; } struct event* min_heap_top_(min_heap_t* s) { return s->n ? *s->p : 0; } int min_heap_push_(min_heap_t* s, struct event* e) { if (min_heap_reserve_(s, s->n + 1)) return -1; min_heap_shift_up_(s, s->n++, e); return 0; } struct event* min_heap_pop_(min_heap_t* s) { if (s->n) { struct event* e = *s->p; min_heap_shift_down_(s, 0u, s->p[--s->n]); e->ev_timeout_pos.min_heap_idx = -1; return e; } return 0; } int min_heap_elt_is_top_(const struct event *e) { return e->ev_timeout_pos.min_heap_idx == 0; } int min_heap_erase_(min_heap_t* s, struct event* e) { if (-1 != e->ev_timeout_pos.min_heap_idx) { struct event *last = s->p[--s->n]; unsigned parent = (e->ev_timeout_pos.min_heap_idx - 1) / 2; /* we replace e with the last element in the heap. We might need to shift it upward if it is less than its parent, or downward if it is greater than one or both its children. Since the children are known to be less than the parent, it can't need to shift both up and down. */ if (e->ev_timeout_pos.min_heap_idx > 0 && min_heap_elem_greater(s->p[parent], last)) min_heap_shift_up_unconditional_(s, e->ev_timeout_pos.min_heap_idx, last); else min_heap_shift_down_(s, e->ev_timeout_pos.min_heap_idx, last); e->ev_timeout_pos.min_heap_idx = -1; return 0; } return -1; } int min_heap_adjust_(min_heap_t *s, struct event *e) { if (-1 == e->ev_timeout_pos.min_heap_idx) { return min_heap_push_(s, e); } else { unsigned parent = (e->ev_timeout_pos.min_heap_idx - 1) / 2; /* The position of e has changed; we shift it up or down * as needed. We can't need to do both. */ if (e->ev_timeout_pos.min_heap_idx > 0 && min_heap_elem_greater(s->p[parent], e)) min_heap_shift_up_unconditional_(s, e->ev_timeout_pos.min_heap_idx, e); else min_heap_shift_down_(s, e->ev_timeout_pos.min_heap_idx, e); return 0; } } int min_heap_reserve_(min_heap_t* s, unsigned n) { if (s->a < n) { struct event** p; unsigned a = s->a ? s->a * 2 : 8; if (a < n) a = n; if (!(p = (struct event**)mm_realloc(s->p, a * sizeof *p))) return -1; s->p = p; s->a = a; } return 0; } void min_heap_shift_up_unconditional_(min_heap_t* s, unsigned hole_index, struct event* e) { unsigned parent = (hole_index - 1) / 2; do { (s->p[hole_index] = s->p[parent])->ev_timeout_pos.min_heap_idx = hole_index; hole_index = parent; parent = (hole_index - 1) / 2; } while (hole_index && min_heap_elem_greater(s->p[parent], e)); (s->p[hole_index] = e)->ev_timeout_pos.min_heap_idx = hole_index; } void min_heap_shift_up_(min_heap_t* s, unsigned hole_index, struct event* e) { unsigned parent = (hole_index - 1) / 2; while (hole_index && min_heap_elem_greater(s->p[parent], e)) { (s->p[hole_index] = s->p[parent])->ev_timeout_pos.min_heap_idx = hole_index; hole_index = parent; parent = (hole_index - 1) / 2; } (s->p[hole_index] = e)->ev_timeout_pos.min_heap_idx = hole_index; } void min_heap_shift_down_(min_heap_t* s, unsigned hole_index, struct event* e) { unsigned min_child = 2 * (hole_index + 1); while (min_child <= s->n) { min_child -= min_child == s->n || min_heap_elem_greater(s->p[min_child], s->p[min_child - 1]); if (!(min_heap_elem_greater(e, s->p[min_child]))) break; (s->p[hole_index] = s->p[min_child])->ev_timeout_pos.min_heap_idx = hole_index; hole_index = min_child; min_child = 2 * (hole_index + 1); } (s->p[hole_index] = e)->ev_timeout_pos.min_heap_idx = hole_index; } #endif /* MINHEAP_INTERNAL_H_INCLUDED_ */
5、mytimer.c-- 与第1篇不同，int timer_process()函数增加了kill timer的方法
#include "mytimer.h" struct min_heap _min_heap; void timer_init() { min_heap_ctor_(&_min_heap); } void timer_destroy() { int i = 0; for (i = 0; i < _min_heap.n; i++) { free(_min_heap.p[i]); } min_heap_dtor_(&_min_heap); } unsigned int timer_add(unsigned int timer_id, int interval, int(*fun)(void*), int arg, int flag /* = CYCLE_TIMER */, int exe_num /* = 0 */) { struct event * ev = (struct event*) malloc(sizeof(struct event)); min_heap_elem_init_(ev); if (NULL == ev) return NULL; struct timeval now; gettime(&now); ev->ev_interval.tv_sec = interval / 1000; ev->ev_interval.tv_usec = (interval % 1000) * 1000; evutil_timeradd(&now, &(ev->ev_interval), &(ev->ev_timeout)); ev->ev_flags = flag; ev->ev_callback = fun; ev->ev_arg = arg; ev->ev_exe_num = exe_num; ev->timer_id = timer_id; min_heap_push_(&_min_heap, ev); return ev->timer_id; } int timer_remove(unsigned int timer_id) { int i = 0; for (i = 0; i < _min_heap.n; i++) { if (timer_id == _min_heap.p[i]->timer_id) { struct event * e = _min_heap.p[i]; min_heap_erase_(&_min_heap, _min_heap.p[i]); free(e); return 1; } } return 0; } int timer_process() { struct event *event; struct timeval now; int ret = 0; while ((event = min_heap_top_(&_min_heap)) != NULL) { gettime(&now); if (evutil_timercmp(&now, &(event->ev_timeout), < )) break; min_heap_pop_(&_min_heap); ret = event->ev_callback(event->ev_arg); if (ret == 0)//kill timer { event->ev_flags = LIMIT_TIMER; event->ev_exe_num = 0; } if (event->ev_flags == CYCLE_TIMER || (event->ev_flags == LIMIT_TIMER && --event->ev_exe_num > 0)) { evutil_timeradd(&(event->ev_timeout), &(event->ev_interval), &(event->ev_timeout)); min_heap_push_(&_min_heap, event); } else { free(event); } } return 0; }
6、main.c -- 重点关注一下函数static void do_epoll(int listenfd)和static int timerfun_callback(int arg)
#include <stdio.h> #include <stdlib.h> #include <string.h> #include <errno.h> #include <sys/socket.h> #include <sys/epoll.h> #include <sys/types.h> #include <sys/resource.h> /*setrlimit */ #include <unistd.h> #include <string.h> #include <fcntl.h> #include <netinet/in.h> #include <arpa/inet.h> #include <assert.h> #include <signal.h> #include <pthread.h> #include "mytimer.h" #define IPADDRESS "127.0.0.1" #define PORT 8011 #define MAXSIZE 1024 #define LISTENQ 5 #define FDSIZE 50000 #define EPOLLEVENTS 100 #define CONFIG_MIN_RESERVED_FDS 32 //come from redis src #define CONFIG_FDSET_INCR (CONFIG_MIN_RESERVED_FDS+96) int stop_server = 0; //函数声明 //创建套接字并进行绑定 static int socket_bind(const char* ip,int port); //IO多路复用epoll static void do_epoll(int listenfd); //事件处理函数 static void handle_events(int epollfd,struct epoll_event *events,int num,int listenfd,char *buf); //处理接收到的连接 static void handle_accpet(int epollfd,int listenfd); //读处理 static void do_read(int epollfd,int fd,char *buf); //写处理 static void do_write(int epollfd,int fd,char *buf); //添加事件 static void add_event(int epollfd,int fd,int state); //修改事件 static void modify_event(int epollfd,int fd,int state); //删除事件 static void delete_event(int epollfd,int fd,int state); //other static int do_error(int fd, int *error); static int setnonblocking(int fd); static void daemonize(void); static int set_fdlimit(); static void signal_exit_handler(); static void signal_exit_func(int signo); static void handle_timer(); static int timerfun_callback(int arg); int main(int argc,char *argv[]) { //设置每个进程允许打开的最大文件数,socket if (set_fdlimit() < 0) { return -1; } int background = 0; if (background) { daemonize(); } //设置信号处理,SIG_IGN表示忽略信号,SIG_DFL表示使用信号的默认处理方式 //signal(SIGHUP, SIG_IGN); //开启的话，就捕获不到终端窗口关闭的信号了。即窗口关闭，进程仍然进行。 signal(SIGPIPE, SIG_IGN); /* if (argc != 2) { fprintf(stderr, "Usage: %s port\n", argv[0]); return 1; } int port = atoi(argv[1]);*/ int listenfd; listenfd = socket_bind(IPADDRESS,PORT); listen(listenfd,LISTENQ); printf("start listening...\n"); signal_exit_handler(); timer_init(); do_epoll(listenfd); timer_destroy(); return 0; } static int socket_bind(const char* ip,int port) { int listenfd; struct sockaddr_in servaddr; listenfd = socket(AF_INET,SOCK_STREAM,0); if (listenfd == -1) { perror("socket error:"); exit(1); } bzero(&servaddr,sizeof(servaddr)); servaddr.sin_family = AF_INET; //inet_pton(AF_INET,ip,&servaddr.sin_addr); servaddr.sin_port = htons(port); servaddr.sin_addr.s_addr = htonl(INADDR_ANY); int error; int reuse = 1; int ret = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); if (ret == -1) { return do_error(listenfd, &error); } if (bind(listenfd,(struct sockaddr*)&servaddr,sizeof(servaddr)) == -1) { perror("bind error: "); exit(1); } return listenfd; } static void do_epoll(int listenfd) { int epollfd; struct epoll_event events[EPOLLEVENTS]; int ret; //创建一个描述符 int error; epollfd = epoll_create(1024);//1024 is just a hint for the kernel if (epollfd == -1) { return do_error(epollfd, &error); } //添加监听描述符事件 add_event(epollfd,listenfd,EPOLLIN); struct event *event; struct timeval now; struct timeval tv; struct timeval *tvp = NULL; while ( stop_server == 0 ) { if ((event = min_heap_top_(&_min_heap)) != NULL) { long now_sec, now_ms;//come from redis src "ae.c", int aeProcessEvents(aeEventLoop *eventLoop, int flags) aeGetTime(&now_sec, &now_ms); tvp = &tv; /* How many milliseconds we need to wait for the next * time event to fire? */ long long ms = (event->ev_timeout.tv_sec - now_sec) * 1000 + (event->ev_timeout.tv_usec / 1000 - now_ms); if (ms > 0) { tvp->tv_sec = ms / 1000; tvp->tv_usec = (ms % 1000) * 1000; } else { tvp->tv_sec = 0; tvp->tv_usec = 0; } /* maybe error gettime(&now); tv.tv_sec = event->ev_timeout.tv_sec - now.tv_sec;; tv.tv_usec = event->ev_timeout.tv_usec - now.tv_usec; if ( tv.tv_usec < 0 ) { tv.tv_usec += 1000000; tv.tv_sec--; printf("tv.tv_usec < 0\n"); } tvp = &tv; */ } else { tvp = NULL; printf("tvp == NULL\n"); } //获取已经准备好的描述符事件 if (tvp == NULL) { ret = epoll_wait(epollfd, events, EPOLLEVENTS, -1); } else { printf("timer_wait:%d\n", tvp->tv_sec*1000 + tvp->tv_usec/1000);//ms ret = epoll_wait(epollfd, events, EPOLLEVENTS, tvp->tv_sec*1000 + tvp->tv_usec/1000);//ms } handle_events(epollfd,events,ret,listenfd); handle_timer(); } close(epollfd); } static void handle_events(int epollfd,struct epoll_event *events,int num,int listenfd,char *buf) { int i; int fd; //进行选好遍历 for (i = 0;i < num;i++) { fd = events[i].data.fd; //根据描述符的类型和事件类型进行处理 if ((fd == listenfd) &&(events[i].events & EPOLLIN)) handle_accpet(epollfd,listenfd); else if (events[i].events & EPOLLIN) do_read(epollfd,fd,buf); else if (events[i].events & EPOLLOUT) do_write(epollfd,fd,buf); } } static void handle_timer() { timer_process(); } static void handle_accpet(int epollfd,int listenfd) { int clifd; struct sockaddr_in cliaddr; socklen_t cliaddrlen = sizeof(cliaddr); clifd = accept(listenfd,(struct sockaddr*)&cliaddr,&cliaddrlen); if (clifd == -1) perror("accpet error:"); else { printf("accept a new client: %s:%d\n",inet_ntoa(cliaddr.sin_addr),cliaddr.sin_port); //添加一个客户描述符和事件 add_event(epollfd,clifd,EPOLLIN); //add timer timer_add(clifd, 1000, timerfun_callback, clifd, CYCLE_TIMER, 0);//ms } } static void do_read(int epollfd,int fd,char *buf) { int nread; nread = read(fd,buf,MAXSIZE); if (nread == -1) { perror("read error:"); close(fd); delete_event(epollfd,fd,EPOLLIN); } else if (nread == 0) { fprintf(stderr,"client close,fd=%d\n",fd); close(fd); delete_event(epollfd,fd,EPOLLIN); timer_remove(fd); } else { printf("read message is: %s,fd=%d\n",buf,fd); //修改描述符对应的事件，由读改为写 modify_event(epollfd,fd,EPOLLOUT); } } static void do_write(int epollfd,int fd,char *buf) { int nwrite; nwrite = write(fd,buf,strlen(buf)); if (nwrite == -1) { perror("write error:"); close(fd); delete_event(epollfd,fd,EPOLLOUT); } else modify_event(epollfd,fd,EPOLLIN); memset(buf,0,MAXSIZE); } static void add_event(int epollfd,int fd,int state) { struct epoll_event ev; ev.events = state; ev.data.fd = fd; epoll_ctl(epollfd,EPOLL_CTL_ADD,fd,&ev); setnonblocking(fd); } static void delete_event(int epollfd,int fd,int state) { struct epoll_event ev; ev.events = state; ev.data.fd = fd; epoll_ctl(epollfd,EPOLL_CTL_DEL,fd,&ev); } static void modify_event(int epollfd,int fd,int state) { struct epoll_event ev; ev.events = state; ev.data.fd = fd; epoll_ctl(epollfd,EPOLL_CTL_MOD,fd,&ev); } static int do_error(int fd, int *error) { fprintf(stderr, "error: %s\n", strerror(errno)); *error = errno; while ((close(fd) == -1) && (errno == EINTR)); errno = *error; return 1; } static int setnonblocking(int fd) { int old_option = fcntl(fd, F_GETFL); int new_option = old_option | O_NONBLOCK; fcntl(fd, F_SETFL, new_option); return old_option; } static void daemonize(void) { //come from /redis/server.c/daemonize() int fd; if (fork() != 0) exit(0); /* parent exits */ setsid(); /* create a new session */ /* Every output goes to /dev/null. If Redis is daemonized but * the 'logfile' is set to 'stdout' in the configuration file * it will not log at all. */ if ((fd = open("/dev/null", O_RDWR, 0)) != -1) { dup2(fd, STDIN_FILENO); dup2(fd, STDOUT_FILENO); dup2(fd, STDERR_FILENO); if (fd > STDERR_FILENO) close(fd); } } static int set_fdlimit() { //设置每个进程允许打开的最大文件数 //这项功能等价于linux终端命令 "ulimit -n 102400" struct rlimit rt; rt.rlim_max = rt.rlim_cur = FDSIZE + CONFIG_FDSET_INCR; if (setrlimit(RLIMIT_NOFILE, &rt) == -1) { perror("setrlimit error"); return -1; } return 0; } static void signal_exit_handler() { struct sigaction sa; memset(&sa, 0, sizeof(sa)); sa.sa_handler = signal_exit_func; sigaction(SIGINT, &sa, NULL);//当按下ctrl+c时，它的效果就是发送SIGINT信号 sigaction(SIGTERM, &sa, NULL);//kill pid sigaction(SIGQUIT, &sa, NULL);//ctrl+\代表退出SIGQUIT //SIGSTOP和SIGKILL信号是不可捕获的,所以下面两句话写了等于没有写 sigaction(SIGKILL, &sa, NULL);//kill -9 pid sigaction(SIGSTOP, &sa, NULL);//ctrl+z代表停止 //#define SIGTERM 15 //#define SIGKILL 9 //kill和kill -9，两个命令在linux中都有杀死进程的效果，然而两命令的执行过程却大有不同，在程序中如果用错了，可能会造成莫名其妙的现象。 //执行kill pid命令，系统会发送一个SIGTERM信号给对应的程序。 //执行kill -9 pid命令，系统给对应程序发送的信号是SIGKILL，即exit。exit信号不会被系统阻塞，所以kill -9能顺利杀掉进程。 } static void signal_exit_func(int signo) { printf("exit signo is %d\n", signo); stop_server = 1; } static int timerfun_callback(int arg) { printf("id is %d\n", arg); //return 0;//kill timer，返回值为0，目的是不让它再次加入堆 return 1; }
