phpBB : настройка и модификация

Код: Выделить всё

<?php

// It may take a whils to crawl a site ...
set_time_limit(36000);

// Inculde the phpcrawl-mainclass
include("libs/PHPCrawler.class.php");

// Extend the class and override the handleDocumentInfo()-method 
class MyCrawler extends PHPCrawler 
{
  public $print_all = true;

  public $lb = "\n";

  public $b_pre = '';
  public $b_end = '';

  public $link_pre = '';
  public $link_succ = '';
  public $link_end = '';

  function handleDocumentInfo($DocInfo) 
  {
    if (!$this->print_all && ((empty($DocInfo->http_status_code)) || ($DocInfo->http_status_code == 200))) {
      echo ' ';
    } else {
      $b__pre = '';
      $b__end = '';
      if ($DocInfo->http_status_code == 404) {
        $b__pre = $this->b_pre;
        $b__end = $this->b_end;
      }
      echo $b__pre;
      // Print the refering URL, URL and the HTTP-status-Code
      $url = $DocInfo->referer_url;
      if (empty($url) ) {
        echo '(NO Referer)';
      } else {
        // Remove sid
        if (preg_match("#[\?\&]sid\=(.*)\&#i",$url.'&',$matches) !== false) {
           $url = str_replace('?sid='.$matches[1].'&','?',$url);
           $url = str_replace('?sid='.$matches[1],'',$url);
           $url = str_replace('&sid='.$matches[1],'',$url);
        }
        echo $this->link_pre.$url.$this->link_succ.(($this->link_pre!='') ? $url : '').$this->link_end;
      }
      echo ' → ';
      echo $DocInfo->url;
      echo ' ('.$DocInfo->http_status_code.')';
      echo $b__end;
      echo $this->lb;
    
      // Now you should do something with the content of the actual
      // received page or file ($DocInfo->source), we skip it in this example 
    
    }
    @ob_end_flush(); @flush(); @ob_start();
  } 
}

// Now, create a instance of your class, define the behaviour
// of the crawler (see class-reference for more options and details)
// and start the crawling-process.

@ob_start();

$crawler = new MyCrawler();

if (PHP_SAPI != 'cli') {
  $crawler->lb = '<br />';

  $crawler->b_pre = '<b>';
  $crawler->b_end = '</b>';
  
  $crawler->link_pre = '<a href="';
  $crawler->link_succ = '" target="_blank" style="text-decoration: none; color:navy;">';
  $crawler->link_end = '</a>';

  @header( 'Content-type: text/html; charset=utf-8' );
}

// URL to crawl
$url = 'c61.no-ip.org';

echo 'Start PHPCrawl: search bad server return codes at '.$crawler->b_pre.$url.$crawler->b_end.$crawler->lb.$crawler->lb;

// URL to crawl
$crawler->setURL($url);

// Delay before every request
$crawler->setRequestDelay(0.1);

// Obey robots.txt-files
//$crawler->obeyRobotsTxt(true);

// Timeouts
$crawler->setConnectionTimeout(20);
$crawler->setStreamTimeout(20);

// Follow mode
$crawler->setFollowMode(2);

// Only receive content of files with content-type "text/html"
$crawler->addContentTypeReceiveRule("#text/html#");

// Ignore links to pictures, dont even request pictures
//$crawler->addURLFilterRule("#\.(jpg|jpeg|gif|png|ico|tif)$# i");

// Ignore links to usb ups chart
$crawler->addURLFilterRule("#(usb_ups_chart\.php)# i");

// Ignore links to posting
$crawler->addURLFilterRule("#/posting(.*)\.php# i");

// Ignore links to members
$crawler->addURLFilterRule("#/member(.*)\.php# i");

// Store and send cookie-data like a browser does
$crawler->enableCookieHandling(true);

// Set the traffic-limit to 1 MB (in bytes,
// for testing we dont want to "suck" the whole site)
//$crawler->setTrafficLimit(1024 * 1024);

// Thats enough, now here we go
$crawler->go();

// At the end, after the process is finished, we print a short
// report (see method getProcessReport() for more information)
$report = $crawler->getProcessReport();

echo $crawler->lb.'Summary:'.$crawler->lb;
echo 'Links followed: '.$report->links_followed.$crawler->lb;
echo 'Documents received: '.$report->files_received.$crawler->lb;
echo 'Bytes received: '.$report->bytes_received.' bytes'.$crawler->lb;
echo 'Process runtime: '.$report->process_runtime.' sec'.$crawler->lb; 
echo 'Done.';

?>

Код: Выделить всё

      $non_follow_path_complpete = $normalized_base_url.substr($disallow_pathes[$x], 1); // "http://www.foo.com/bla/"

Код: Выделить всё

      $non_follow_path_complpete = $normalized_base_url.$disallow_pathes[$x]; // "http://www.foo.com/bla/"