爬虫Larbin解析(二)——sequencer()

//位置：larbin-2.6.3/src/fetch/sequencer.cc

void sequencer() 
{
    bool testPriority = true;
    if (space == 0)                //unit space = 0
    {
        space = global::inter->putAll();
    }
    int still = space;
    if (still > maxPerCall)        //#define maxPerCall 100
        still = maxPerCall;
    while (still) 
    {
        if (canGetUrl(&testPriority)) 
        {
            --space; 
            --still; 
        } 
        else 
        { 
            still = 0; 
        } 
    }
}

inter = new Interval(ramUrls);   //#define ramUrls 100000  (位置：larbin-2.6.3/src/types.h)

/* Get the next url
 * here is defined how priorities are handled
 按优先级从各个URL队列
 （比如URLsDisk,URLsDiskWait或URLsPriority,URLsPriorityWait）
 获取url保存到某个NameSite(通过url的hash值)

at "global.cc"
// FIFOs
URLsDisk         = new PersistentFifo(reload, fifoFile);
URLsDiskWait     = new PersistentFifo(reload, fifoFileWait);
URLsPriority     = new SyncFifo<url>;
URLsPriorityWait = new SyncFifo<url>;

 */
static bool canGetUrl (bool *testPriority) 
{
    url *u;
    if (global::readPriorityWait != 0)  // 在global.cc声明定义: uint global::readPriorityWait=0;
    {
        global::readPriorityWait--;
        u = global::URLsPriorityWait->get();
        global::namedSiteList[u->hostHashCode()].putPriorityUrlWait(u);
        return true;
    } 
    else if (*testPriority && (u=global::URLsPriority->tryGet()) != NULL) 
    {
        // We've got one url (priority)
        global::namedSiteList[u->hostHashCode()].putPriorityUrl(u);
        return true;
    } 
    else 
    {
        *testPriority = false;
        // Try to get an ordinary url
        if (global::readWait) 
        {
          global::readWait--;
          u = global::URLsDiskWait->get();
          global::namedSiteList[u->hostHashCode()].putUrlWait(u);
          return true;
        } 
        else 
        {
            u = global::URLsDisk->tryGet();
            if (u != NULL) 
            {
                global::namedSiteList[u->hostHashCode()].putUrl(u);
                return true;
            }
            else 
            {
                return false;
            }
        }
    }
}

// see if we should read again urls in fifowait
if ((global::now % 300) == 0) {
    global::readPriorityWait = global::URLsPriorityWait->getLength();
    global::readWait = global::URLsDiskWait->getLength();
}
if ((global::now % 300) == 150) {
    global::readPriorityWait = 0;
    global::readWait = 0;
}

/** Put an url in the fifo
 * If there are too much, put it back in UrlsInternal
 * Never fill totally the fifo => call at least with 1 */
void putGenericUrl(url *u, int limit, bool prio);
inline void putUrl(url *u) {
    putGenericUrl(u, 15, false);
}
inline void putUrlWait(url *u) {
    putGenericUrl(u, 10, false);
}
inline void putPriorityUrl(url *u) {
    putGenericUrl(u, 5, true);
}
inline void putPriorityUrlWait(url *u) {
    putGenericUrl(u, 1, true);
}

/* Put an url in the fifo if their are not too many */
void NamedSite::putGenericUrl(url *u, int limit, bool prio) 

{
    if (nburls > maxUrlsBySite - limit) 

{
       // Already enough Urls in memory for this Site
       // first check if it can already be forgotten
       if (!strcmp(name, u->getHost())) 

{
           if (dnsState == errorDns) 

{
              nburls++;
              forgetUrl(u, noDNS);
              return;
           }
           if (dnsState == noConnDns) 

{
              nburls++;
              forgetUrl(u, noConnection);
              return;
           }
           if (u->getPort() == port && dnsState == doneDns && !testRobots(u->getFile())) 

{
              nburls++;
              forgetUrl(u, forbiddenRobots);
              return;
           }
       }
       // else put it back in URLsDisk
       refUrl();
       global::inter->getOne();
       if (prio) 

{
           global::URLsPriorityWait->put(u);
       } 
       else 

{
           global::URLsDiskWait->put(u);
       }
   }

else {
    nburls++;
    if (dnsState == waitDns || strcmp(name, u->getHost()) || port
           != u->getPort() || global::now > dnsTimeout) {
       // dns not done or other site
       putInFifo(u);
       addNamedUrl();
       // Put Site in fifo if not yet in
       if (!isInFifo) {
           isInFifo = true;
           global::dnsSites->put(this);
       }
    } else
       switch (dnsState) {
       case doneDns:
           transfer(u);
           break;
       case errorDns:
           forgetUrl(u, noDNS);
           break;
       default: // noConnDns
           forgetUrl(u, noConnection);
       }
}

void NamedSite::transfer(url *u) {
    if (testRobots(u->getFile())) {
       if (global::proxyAddr == NULL) {
           memcpy(&u->addr, &addr, sizeof(struct in_addr));
       }
       global::IPSiteList[ipHash].putUrl(u);
    } else {
       forgetUrl(u, forbiddenRobots);
    }
}

NamedSite *global::namedSiteList;
namedSiteList = new NamedSite[namedSiteListSize];