超级易懂爬虫系列之爬虫简单的架构

# -*- coding: UTF-8 -*-
import os
import urllib2
from urllib2 import URLError
import url_manager, html_downloader, html_parser
 
 
class SpiderMain(object):
 
    def __init__(self):
        self.urls = url_manager.UrlManager()
        self.downloader = html_downloader.HtmlDownloader()
        self.parser = html_parser.HtmlParser()
 
    def craw(self, root_url):
        self.urls.add_new_url(root_url)
        count = 1
        girl_dir = 'C:\Users\htq\Desktop\girl'
        if not os.path.exists(girl_dir):
            os.mkdir(girl_dir)
        os.chdir(girl_dir)
        while self.urls.has_new_url():
            try :
                new_url = self.urls.get_new_url()
                print new_url
                html_cont = self.downloader.download(new_url)
                new_urls, img_srcs = self.parser.parse(new_url, html_cont)
                self.urls.add_new_urls(new_urls) #将爬取的新的url添加到待爬取的url集合中
 
                #保存获取到的妹子图
                for img in img_srcs:
                    src = img['src']
                    # print src
                    name = 'iyangzi' + str(count)
                    with open(name + '.jpg', 'ab') as img_object:
                        img_content = urllib2.urlopen(src).read()
                        img_object.write(img_content)
                        img_object.flush()
                        
                    count += 1
            except URLError,e:
                print 'spider craw failed ',e
 
if __name__ == "__main__":
    root_url ="http://iyangzi.com/?p=21"
    obj_spider = SpiderMain()
    obj_spider.craw(root_url)

html_downloader.py模块负责从传入的url中将网页以html的形式抓取下来，功能很简单，因此代码很简单，如下：

# -*- coding: UTF-8 -*-
import urllib2
 
class HtmlDownloader(object):
 
    def download(self, url):
        if url is None:
            return None
        response = urllib2.urlopen(url)
        if response.getcode() != 200:
            return None
        return response.read()

经过前面两篇文章理论与动手操作相信大家很容易看懂这段代码了，和前面是一样的。

html_parser.py模块主要负责从传入的url参数中解析出我们感兴趣的数据和新增加的url，代码如下:

# -*- coding: UTF-8 -*-
import re
from bs4 import BeautifulSoup
 
 
class HtmlParser(object):
 
    def parse(self, page_url, html_cont):
        if page_url is None or html_cont is None:
            return
        #print html_cont
        soup = BeautifulSoup(html_cont, 'html.parser', from_encoding='utf-8')
        new_urls = self.get_new_urls(soup)
        img_srcs = self.get_img_data(soup)
        return new_urls, img_srcs
 
    def get_new_urls(self,soup):
        new_urls = set()
        links = soup.find_all('a', href=re.compile(r"http://iyangzi.com/\?p=\d+$"))
        for link in links:
            new_urls.add(link['href'])
        return new_urls
 
    def get_img_data(self,soup):
        img_srcs = []
        all_img = soup.find('div', class_='post-content').find_all('img')
        for img in all_img:
            img_srcs.append(img)
        return img_srcs

一般在解析出新的url时需要用到正则表达式，因为需要抽象出我们感兴趣的url的共同点，从而过滤掉不相关网页，只爬取我们需要的网页。这在前面文章中已经为大家分析过了。

url管理器主要负责判断哪些url已经访问过，哪些url虽然未访问但是已经在待爬取的url集合中。

# -*- coding: UTF-8 -*-
 
class UrlManager(object):
    def __init__(self):
        self.new_urls = set()    #新增加的待爬取的url集合
        self.crawed_urls = set() #已爬取的url集合
 
    def add_new_url(self, url):
        if url is None:
            return
        if url not in self.new_urls and url not in self.crawed_urls:
            self.new_urls.add(url)
 
    def add_new_urls(self, urls):
        if urls is None or len(urls) == 0:
            return
        for url in urls:
            self.add_new_url(url)
 
    def has_new_url(self):
        return len(self.new_urls) != 0
 
    def get_new_url(self):
        # 爬取一个url时将该url添加到已访问的url集合中表示该url已经访问过
        new_url = self.new_urls.pop()
        self.crawed_urls.add(new_url)
        return new_url

代码很简单，注释也很详细。