程式師世界 >> 編程語言 >> 網頁編程 >> PHP編程 >> 關於PHP編程 >> 抓取“維庫電子市場”供應商程序

抓取“維庫電子市場”供應商程序

編輯：關於PHP編程

<?php
/**
* 抓取“維庫電子市場”供應商主程序
* author Lee.
* Last modify $Date: 2012-2-3 9:30:21 $
* 注：本程序按照編碼 GB2312 執行，因為“維庫電子市場”網站是GB2312編碼，數據庫也得保持一致
*/
class weiku {
 private $key; // 型號
 private $pageNum; // 頁碼

 /**
 * 入口程序
 */
 public function go($key) {
 $this->key = $key;
 $this->pageNum = $this->getPageNum();
 $this->getInfo();
 }

 /**
 * 獲取供應商 url 鏈接數組
 * @return ArrayObject
 */
 private function getInfo() {
 if ($this->pageNum==1) { # 處理只有一頁的情況
 $arr = $this->shopAddContact($this->shopUrlMatchReArr($this->getContent()));
 $this->isAddSuccess($arr);
 } elseif ($this->pageNum>1) { # 多頁
 for ($i=1; $i<=$this->pageNum; $i++) {
 $arr = $this->shopAddContact($this->shopUrlMatchReArr($this->getContent($i)));
 $this->isAddSuccess($arr);
 }
 }
 }

 /**
 * 打印是否添加成功
 * @param ArrayObject $arr
 * @return string
 */
 private function isAddSuccess($arr) {
 foreach ($arr as $k=>$v) {
 if ($this->execAdd($this->getInfoByShopUrl($v))) {
 echo 'Add Success!!';
 } else {
 echo 'Add Faild!!';
 }
 }
 }

 /**
 * 執行添加到數據庫
 * @param ArrayObject $infoArr
 * @return Number 受影響的行數
 */
 private function execAdd($infoArr) {
 $mysqli = $this->getDb();
 if (!emptyempty($infoArr['company'])) {
 if (!$this->isExists($mysqli, $infoArr)) {
 $num = $mysqli->query("INSERT INTO weiku(company,person,phone,fax,mobile,qq,msn,email,address,shopUrl) VALUES ('{$infoArr['company']}','{$infoArr['person']}','{$infoArr['phone']}','{$infoArr['fax']}','{$infoArr['mobile']}','{$infoArr['qq']}','{$infoArr['msn']}','{$infoArr['email']}','{$infoArr['address']}','{$infoArr['shopUrl']}')");
 return $num;
 } else {
 return false; # 表示數據已經存在
 }
 } else {
 return false;
 }
 }

 /**
 * 連接數據庫
 */
 private function getDb() {
 $mysqli = new mysqli('localhost', 'root', '1715544', 'weiku');
 $mysqli->query('SET NAMES GB2312');
 return $mysqli;
 }

 /**
 * 檢查公司是否已經存在
 * @param Resource $mysqli
 * @param ArrayObject $infoArr
 * @return bool
 */
 private function isExists($mysqli, $infoArr) {
 $mysqli->query("SELECT company FROM weiku WHERE company = '{$infoArr['company']}'");
 if ($mysqli->affected_rows) {
 return true;
 } else {
 return false;
 }
 }

 /**
 * 抓取信息
 * @param $url
 * @return ArrayObject
 */
 private function getInfoByShopUrl($url) {
 $re = $this->getUrlInfo($url);
 preg_match_all('/公司名稱：<\/b>(.*)<\/span>/Usi', $re, $companyArr);
 preg_match_all('/聯系人：<\/b>(.*)<\/span>/Usi', $re, $personArr);
 preg_match_all('/電話：<\/b>(.*)<\/span>/Usi', $re, $phoneArr);
 preg_match_all('/傳真：<\/b>(.*)<\/span>/Usi', $re, $faxArr);
 preg_match_all('/手機：<\/b>(.*)<\/span>/Usi', $re, $mobileArr);
 preg_match_all('/QQ：<\/b>(.*)<\/span>/Usi', $re, $qqArr);
 preg_match_all('/MSN：<\/b>(.*)<\/span>/Usi', $re, $msnArr);
 preg_match_all('/E-Mail：<\/b>(.*)<\/span>/Usi', $re, $emailArr);
 preg_match_all('/公司地址：<\/b>(.*)<\/span>/Usi', $re, $addressArr);
 $infoArr = array(
 'company'=>$this->stripATags($companyArr[1][0]),
 'person'=>trim($personArr[1][0]),
 'phone'=>trim($phoneArr[1][0]),
 'fax'=>trim($faxArr[1][0]),
 'mobile'=>trim($mobileArr[1][0]),
 'qq'=>$this->formatQqMsn($qqArr[1][0]),
 'msn'=>$this->formatQqMsn($msnArr[1][0], 'MSN'),
 'email'=>$this->stripATags($emailArr[1][0]),
 'address'=>trim($addressArr[1][0]),
 'shopUrl'=>$url
 );
 return $infoArr;
 }

 /**
 * 根據頁面獲取供應商 url 數組
 * @param string $re
 * @return ArrayObject
 */
 private function shopUrlMatchReArr($re) {
 $re = preg_replace('/<img.* [\/]>/', '', $re);
 $re = preg_replace('/<img.*>/', '', $re);
 $re = preg_replace('/<a href=\".+\" target=\"\_blank\">[A-Z]<\/a>/', '', $re);
 $re = preg_replace('/<a href=\".+\" target=\"\_blank\">[0-9]<\/a>/', '', $re);
 $re = preg_replace('/<a href=\".+\" target=\"\_blank\">.*<\/a>/', '', $re);
 $re = preg_replace('/<a href="javascript.+">.*<\/a>/', '', $re);
 $re = preg_replace('/<a href.+>營業執照<\/a>/', '', $re);
 $re = preg_replace('/<a href.+>該企業更多資質>><\/a>/', '', $re);
 $re = preg_replace('/<a href.+>點此反饋<\/a>/', '', $re);
 $re = preg_replace('/<a href.+>首頁<\/a>/', '', $re);
 $re = preg_replace('/<a href.+>IC<\/a>/', '', $re);
 $re = preg_replace('/<a href.+>簡潔<\/a>/', '', $re);
 $re = preg_replace('/<a href.+>信用<\/a>/', '', $re);
 $re = preg_replace('/<a href.+>.*更多報價信息>><\/a>/', '', $re);
 $re = preg_replace('/<a href=\".*\" target=\"\_blank\" rel=\"nofollow\">.*<\/a>/', '', $re);
 $re = preg_replace('/<div class="kingbanan mb8">.*/', '', $re);
 preg_match_all('/<a href=\"(.+)\".*>.+<\/a>/Usi', $re, $arr);
 $arr = $this->formatUrlArr(array_unique($arr[1]));
 return $arr;
 }

 /**
 * 格式化數組
 * @param Array $arr
 * @return ArrayObject
 */
 private function formatUrlArr($arr) {
 $newArr = array();
 foreach ($arr as $key=>$value) {
 if ($this->isExistsHttp($value)) {
 $newArr[$key] = $value;
 }
 }
 return $newArr;
 }

 /**
 * 格式化 QQ
 * @param string $str
 * @return string
 */
 private function formatQqMsn($str, $e='QQ') {
 if (emptyempty($str)) return '';
 preg_match_all('/alt="'.$e.'\:(.+)"/Usi', $str, $arr);
 if (count($arr[1])==1) return $arr[1][0];
 $newStr = null;
 foreach ($arr[1] as $value) {
 $newStr .= $value . ' ';
 }
 return rtrim($newStr, ' ');
 }

 /**
 * 供應商店鋪鏈接添加 contact.html
 * @param array $arr
 * @return string
 */
 private function shopAddContact($arr) {
 foreach ($arr as $k=>$v) {
 $arr[$k] = $v . '/contact.html';
 }
 return $arr;
 }

 /**
 * 去掉網址的 A 標簽
 * @param string $site
 * @return string
 */
 private function stripATags($site) {
 $site = preg_replace('/<a.+>(.+)<\/a>/', '\1', $site);
 return $site;
 }

 /**
 * 檢查 url 是否有 http
 * @param string $url
 * @return bool
 */
 private function isExistsHttp($url) {
 if (stristr($url, 'http://')) {
 return true;
 } else {
 return false;
 }
 }

 /**
 * 獲取頁面內容
 * @param Number $page
 * @return string
 */
 private function getContent($page=1) {
 $re = file_get_contents($this->getUrl($this->key, $page));
 return $re;
 }

 /**
 * 獲取頁碼
 * @return Number
 */
 private function getPageNum() {
 $i = 1;
 while (true) {
 $re = $this->getContent($i);
 # 處理單頁避免處理死循環
 if (!strstr($re, '下一頁')) {
 break;
 } else {
 # 多頁，計算出頁碼
 if (stristr($re, '下一頁</li>')) break;
 $i++;
 }
 }
 return $i;
 }

 /**
 * 獲取 URL 鏈接
 * @param string $str
 * @param int $page 頁碼
 * @return string
 */
 private function getUrl($str, $page=1) {
 return "http://www.dzsc.com/ic/sell_search.html?keyword={$str}&ic_sel=supplygoods&Submit=%26%23160%3B&page={$page}"; }

 /**
 * 獲取頁面內容
 * @param string $url
 * @return string
 */
 private function getUrlInfo($url) {
 $re = file_get_contents($url);
 return $re;
 }
}

/*
程序運行思路：根據“華強電子網”的IC搜索功能，輸入型號進行搜索，然後抓取供應商信息

數據庫結構
CREATE TABLE `weiku` (
 `id` mediumint(8) unsigned NOT NULL auto_increment COMMENT 'ID',
 `company` varchar(300) default NULL COMMENT '公司名稱',
 `person` varchar(200) default NULL COMMENT '聯系人',
 `phone` varchar(300) default NULL COMMENT '電話',
 `fax` varchar(300) default NULL COMMENT '傳真',
 `mobile` varchar(300) default NULL COMMENT '手機',
 `qq` varchar(200) default NULL COMMENT 'QQ',
 `msn` varchar(200) default NULL COMMENT 'MSN',
 `email` varchar(300) default NULL COMMENT '郵箱',
 `address` varchar(500) default NULL COMMENT '公司地址',
 `shopUrl` varchar(200) default NULL COMMENT '維庫網店鋪地址',
 PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=gb2312
*/

$k = new weiku();
$arr = array_unique(array('MAX3232', 'AML8613', 'MT6225A', 'OM8373PS/N3/A', 'PT7313', 'MAX8212ESA', 'TL431', 'S3C2440', 'TMS320F2812PGFA', 'PCM1704', 'AN6717', 'CA3162E', 'CA3161E', 'LM393N', 'DS18B20', 'SHT10', 'AML8613', 'AN6717', 'LM393N', 'CA3161E', 'CA3162E', 'PCM1704', 'STK392-040', 'K1667', 'MAX232', 'STM32F103', 'LM358'));
foreach ($arr as $v) {
 $k->go($v);
}
?>
<?php
/**
* 抓取“維庫電子市場”供應商主程序
* author Lee.
* Last modify $Date: 2012-2-3 9:30:21 $
* 注：本程序按照編碼 GB2312 執行，因為“維庫電子市場”網站是GB2312編碼，數據庫也得保持一致
*/
class weiku {
private $key; // 型號
private $pageNum; // 頁碼

/**
* 入口程序
*/
public function go($key) {
  $this->key = $key;
  $this->pageNum = $this->getPageNum();
  $this->getInfo();
}

/**
* 獲取供應商 url 鏈接數組
* @return ArrayObject
*/
private function getInfo() {
 if ($this->pageNum==1) { # 處理只有一頁的情況
 $arr = $this->shopAddContact($this->shopUrlMatchReArr($this->getContent()));
 $this->isAddSuccess($arr);
 } elseif ($this->pageNum>1) { # 多頁
 for ($i=1; $i<=$this->pageNum; $i++) {
 $arr = $this->shopAddContact($this->shopUrlMatchReArr($this->getContent($i)));
 $this->isAddSuccess($arr);
 }
 }
}

/**
* 打印是否添加成功
* @param ArrayObject $arr
* @return string
*/
private function isAddSuccess($arr) {
 foreach ($arr as $k=>$v) {
 if ($this->execAdd($this->getInfoByShopUrl($v))) {
 echo 'Add Success!!';
 } else {
 echo 'Add Faild!!';
 }
 }
}

/**
* 執行添加到數據庫
* @param ArrayObject $infoArr
* @return Number 受影響的行數
*/
private function execAdd($infoArr) {
  $mysqli = $this->getDb();
  if (!empty($infoArr['company'])) {
   if (!$this->isExists($mysqli, $infoArr)) {
    $num = $mysqli->query("INSERT INTO weiku(company,person,phone,fax,mobile,qq,msn,email,address,shopUrl) VALUES ('{$infoArr['company']}','{$infoArr['person']}','{$infoArr['phone']}','{$infoArr['fax']}','{$infoArr['mobile']}','{$infoArr['qq']}','{$infoArr['msn']}','{$infoArr['email']}','{$infoArr['address']}','{$infoArr['shopUrl']}')");
    return $num;
   } else {
    return false; # 表示數據已經存在
   }
  } else {
   return false;
  }
}

/**
* 連接數據庫
*/
private function getDb() {
  $mysqli = new mysqli('localhost', 'root', '1715544', 'weiku');
  $mysqli->query('SET NAMES GB2312');
  return $mysqli;
}

/**
* 檢查公司是否已經存在
* @param Resource $mysqli
* @param ArrayObject $infoArr
* @return bool
*/
private function isExists($mysqli, $infoArr) {
  $mysqli->query("SELECT company FROM weiku WHERE company = '{$infoArr['company']}'");
  if ($mysqli->affected_rows) {
   return true;
  } else {
   return false;
  }
}

/**
* 抓取信息
* @param $url
* @return ArrayObject
*/
private function getInfoByShopUrl($url) {
 $re = $this->getUrlInfo($url);
 preg_match_all('/公司名稱：<\/b>(.*)<\/span>/Usi', $re, $companyArr);
 preg_match_all('/聯系人：<\/b>(.*)<\/span>/Usi', $re, $personArr);
 preg_match_all('/電話：<\/b>(.*)<\/span>/Usi', $re, $phoneArr);
 preg_match_all('/傳真：<\/b>(.*)<\/span>/Usi', $re, $faxArr);
 preg_match_all('/手機：<\/b>(.*)<\/span>/Usi', $re, $mobileArr);
 preg_match_all('/QQ：<\/b>(.*)<\/span>/Usi', $re, $qqArr);
 preg_match_all('/MSN：<\/b>(.*)<\/span>/Usi', $re, $msnArr);
 preg_match_all('/E-Mail：<\/b>(.*)<\/span>/Usi', $re, $emailArr);
 preg_match_all('/公司地址：<\/b>(.*)<\/span>/Usi', $re, $addressArr);
 $infoArr = array(
 'company'=>$this->stripATags($companyArr[1][0]),
 'person'=>trim($personArr[1][0]),
 'phone'=>trim($phoneArr[1][0]),
 'fax'=>trim($faxArr[1][0]),
 'mobile'=>trim($mobileArr[1][0]),
 'qq'=>$this->formatQqMsn($qqArr[1][0]),
 'msn'=>$this->formatQqMsn($msnArr[1][0], 'MSN'),
 'email'=>$this->stripATags($emailArr[1][0]),
 'address'=>trim($addressArr[1][0]),
 'shopUrl'=>$url
 );
 return $infoArr;
}

/**
* 根據頁面獲取供應商 url 數組
* @param string $re
* @return ArrayObject
*/
private function shopUrlMatchReArr($re) {
 $re = preg_replace('/<img.* [\/]>/', '', $re);
 $re = preg_replace('/<img.*>/', '', $re);
 $re = preg_replace('/<a href=\".+\" target=\"\_blank\">[A-Z]<\/a>/', '', $re);
 $re = preg_replace('/<a href=\".+\" target=\"\_blank\">[0-9]<\/a>/', '', $re);
 $re = preg_replace('/<a href=\".+\" target=\"\_blank\">.*<\/a>/', '', $re);
 $re = preg_replace('/<a href="javascript.+">.*<\/a>/', '', $re);
 $re = preg_replace('/<a href.+>營業執照<\/a>/', '', $re);
 $re = preg_replace('/<a href.+>該企業更多資質>><\/a>/', '', $re);
 $re = preg_replace('/<a href.+>點此反饋<\/a>/', '', $re);
 $re = preg_replace('/<a href.+>首頁<\/a>/', '', $re);
 $re = preg_replace('/<a href.+>IC<\/a>/', '', $re);
 $re = preg_replace('/<a href.+>簡潔<\/a>/', '', $re);
 $re = preg_replace('/<a href.+>信用<\/a>/', '', $re);
 $re = preg_replace('/<a href.+>.*更多報價信息>><\/a>/', '', $re);
 $re = preg_replace('/<a href=\".*\" target=\"\_blank\" rel=\"nofollow\">.*<\/a>/', '', $re);
 $re = preg_replace('/<div class="kingbanan mb8">.*/', '', $re);
 preg_match_all('/<a href=\"(.+)\".*>.+<\/a>/Usi', $re, $arr);
 $arr = $this->formatUrlArr(array_unique($arr[1]));
 return $arr;
}

/**
* 格式化數組
* @param Array $arr
* @return ArrayObject
*/
private function formatUrlArr($arr) {
 $newArr = array();
 foreach ($arr as $key=>$value) {
 if ($this->isExistsHttp($value)) {
 $newArr[$key] = $value;
 }
 }
 return $newArr;
}

/**
* 格式化 QQ
* @param string $str
* @return string
*/
private function formatQqMsn($str, $e='QQ') {
 if (empty($str)) return '';
 preg_match_all('/alt="'.$e.'\:(.+)"/Usi', $str, $arr);
 if (count($arr[1])==1) return $arr[1][0];
 $newStr = null;
 foreach ($arr[1] as $value) {
 $newStr .= $value . ' ';
 }
 return rtrim($newStr, ' ');
}

/**
* 供應商店鋪鏈接添加 contact.html
* @param array $arr
* @return string
*/
private function shopAddContact($arr) {
  foreach ($arr as $k=>$v) {
   $arr[$k] = $v . '/contact.html';
  }
  return $arr;
}

/**
* 去掉網址的 A 標簽
* @param string $site
* @return string
*/
private function stripATags($site) {
$site = preg_replace('/<a.+>(.+)<\/a>/', '\1', $site);
return $site;
}

/**
* 檢查 url 是否有 http
* @param string $url
* @return bool
*/
private function isExistsHttp($url) {
 if (stristr($url, 'http://')) {
 return true;
 } else {
 return false;
 }
}

/**
* 獲取頁面內容
* @param Number $page
* @return string
*/
private function getContent($page=1) {
 $re = file_get_contents($this->getUrl($this->key, $page));
 return $re;
}

/**
* 獲取頁碼
* @return Number
*/
private function getPageNum() {
 $i = 1;
 while (true) {
 $re = $this->getContent($i);
 # 處理單頁避免處理死循環
 if (!strstr($re, '下一頁')) {
 break;
 } else {
 # 多頁，計算出頁碼
 if (stristr($re, '下一頁</li>')) break;
 $i++;
 }
 }
 return $i;
}

/**
* 獲取 URL 鏈接
* @param string $str
* @param int $page 頁碼
* @return string
*/
private function getUrl($str, $page=1) {
retrn "http://www.dzsc.com/ic/sell_search.html?keyword={$str}&ic_sel=supplygoods&Submit=%26%23160%3B&page={$page}"u;

}

/**
* 獲取頁面內容
* @param string $url
* @return string
*/
private function getUrlInfo($url) {
$re = file_get_contents($url);
return $re;
}
}

/*
程序運行思路：根據“華強電子網”的IC搜索功能，輸入型號進行搜索，然後抓取供應商信息

數據庫結構
CREATE TABLE `weiku` (
`id` mediumint(8) unsigned NOT NULL auto_increment COMMENT 'ID',
`company` varchar(300) default NULL COMMENT '公司名稱',
`person` varchar(200) default NULL COMMENT '聯系人',
`phone` varchar(300) default NULL COMMENT '電話',
`fax` varchar(300) default NULL COMMENT '傳真',
`mobile` varchar(300) default NULL COMMENT '手機',
`qq` varchar(200) default NULL COMMENT 'QQ',
`msn` varchar(200) default NULL COMMENT 'MSN',
`email` varchar(300) default NULL COMMENT '郵箱',
`address` varchar(500) default NULL COMMENT '公司地址',
`shopUrl` varchar(200) default NULL COMMENT '維庫網店鋪地址',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=gb2312
*/

$k = new weiku();
$arr = array_unique(array('MAX3232', 'AML8613', 'MT6225A', 'OM8373PS/N3/A', 'PT7313', 'MAX8212ESA', 'TL431', 'S3C2440', 'TMS320F2812PGFA', 'PCM1704', 'AN6717', 'CA3162E', 'CA3161E', 'LM393N', 'DS18B20', 'SHT10', 'AML8613', 'AN6717', 'LM393N', 'CA3161E', 'CA3162E', 'PCM1704', 'STK392-040', 'K1667', 'MAX232', 'STM32F103', 'LM358'));
foreach ($arr as $v) {
$k->go($v);
}
?>

摘自 Lee.的專欄