以豌豆荚为例，用 Scrapy 爬取分类多级页面

import scrapy

class WandoujiaItem(scrapy.Item):
    cate_name = scrapy.Field() #分类名
    child_cate_name = scrapy.Field() #分类编号
    app_name = scrapy.Field()   # 子分类名
    install = scrapy.Field()    # 子分类编号
    volume = scrapy.Field()     # 体积
    comment = scrapy.Field()    # 评论
    icon_url = scrapy.Field()   # 图标url

import base64
proxyServer = "http://http-dyn.abuyun.com:9020"
proxyUser = "你的信息"
proxyPass = "你的信息"

proxyAuth = "Basic " + base64.urlsafe_b64encode(bytes((proxyUser + ":" + proxyPass), "ascii")).decode("utf8")
class AbuyunProxyMiddleware(object):
    def process_request(self, request, spider):
        request.meta["proxy"] = proxyServer
        request.headers["Proxy-Authorization"] = proxyAuth
        logging.debug('Using Proxy:%s'%proxyServer)

MongoDB 存储
class MongoPipeline(object):
    def __init__(self,mongo_url,mongo_db):
        self.mongo_url = mongo_url
        self.mongo_db = mongo_db
    
    @classmethod
    def from_crawler(cls,crawler):
        return cls(
            mongo_url = crawler.settings.get('MONGO_URL'),
            mongo_db = crawler.settings.get('MONGO_DB')
        )
    
    def open_spider(self,spider):
        self.client = pymongo.MongoClient(self.mongo_url)
        self.db = self.client[self.mongo_db]

    def process_item(self,item,spider):
        name = item.__class__.__name__
        # self.db[name].insert(dict(item))
        self.db[name].update_one(item, {'$set': item}, upsert=True)
        return item

    def close_spider(self,spider):
        self.client.close()

# 分文件夹下载
class ImagedownloadPipeline(ImagesPipeline):
    def get_media_requests(self,item,info):
        if item['icon_url']:
            yield scrapy.Request(item['icon_url'],meta={'item':item})

    def file_path(self, request, response=None, info=None):
        name = request.meta['item']['app_name']
        cate_name = request.meta['item']['cate_name']
        child_cate_name = request.meta['item']['child_cate_name']
      
        path1 = r'/wandoujia/%s/%s' %(cate_name,child_cate_name)
        path = r'{}\{}.{}'.format(path1, name, 'jpg')
        return path

    def item_completed(self,results,item,info):
        image_path = [x['path'] for ok,x in results if ok]
        if not image_path:
            raise DropItem('Item contains no images')
        return item

BOT_NAME = 'wandoujia'
SPIDER_MODULES = ['wandoujia.spiders']
NEWSPIDER_MODULE = 'wandoujia.spiders'

MONGO_URL = 'localhost'
MONGO_DB = 'wandoujia'

# 是否遵循机器人规则
ROBOTSTXT_OBEY = False
# 下载设置延迟 由于买的阿布云一秒只能请求5次，所以每个请求设置了 0.2s延迟
DOWNLOAD_DELAY = 0.2

DOWNLOADER_MIDDLEWARES = {
    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
    'scrapy_fake_useragent.middleware.RandomUserAgentMiddleware': 100, # 随机UA
    'wandoujia.middlewares.AbuyunProxyMiddleware': 200 # 阿布云代理
    ）
    
ITEM_PIPELINES = {
   'wandoujia.pipelines.MongoPipeline': 300,
   'wandoujia.pipelines.ImagedownloadPipeline': 400,
}
    
# URL不去重
DUPEFILTER_CLASS = 'scrapy.dupefilters.BaseDupeFilter'

def __init__(self):
        self.cate_url = 'https://www.wandoujia.com/category/app'
        # 子分类首页url
        self.url = 'https://www.wandoujia.com/category/'
        # 子分类 ajax请求页url
        self.ajax_url = 'https://www.wandoujia.com/wdjweb/api/category/more?'
        # 实例化分类标签
        self.wandou_category = Get_category()
def start_requests(self):
        yield scrapy.Request(self.cate_url,callback=self.get_category)
        
def get_category(self,response):    
        cate_content = self.wandou_category.parse_category(response)
        # ...

class Get_category():
    def parse_category(self, response):
        category = response.css('.parent-cate')
        data = [{
            'cate_name': item.css('.cate-link::text').extract_first(),
            'cate_code': self.get_category_code(item),
            'child_cate_codes': self.get_child_category(item),
        } for item in category]
        return data
    
    # 获取所有主分类标签数值代码
    def get_category_code(self, item):
        cate_url = item.css('.cate-link::attr("href")').extract_first()
        pattern = re.compile(r'.*/(\d+)')  # 提取主类标签代码
        cate_code = re.search(pattern, cate_url)
        return cate_code.group(1)

    # 获取所有子分类名称和编码
    def get_child_category(self, item):
        child_cate = item.css('.child-cate a')
        child_cate_url = [{
            'child_cate_name': child.css('::text').extract_first(),
            'child_cate_code': self.get_child_category_code(child)
        } for child in child_cate]
        return child_cate_url

    # 正则提取子分类编码
    def get_child_category_code(self, child):
        child_cate_url = child.css('::attr("href")').extract_first()
        pattern = re.compile(r'.*_(\d+)')  # 提取小类标签编号
        child_cate_code = re.search(pattern, child_cate_url)
        return child_cate_code.group(1)

{'cate_name': '影音播放', 'cate_code': '5029', 'child_cate_codes': [
    {'child_cate_name': '视频', 'child_cate_code': '716'}, 
    {'child_cate_name': '直播', 'child_cate_code': '1006'}, 
    ...
	]}, 
{'cate_name': '系统工具', 'cate_code': '5018', 'child_cate_codes': [
    {'child_cate_name': 'WiFi', 'child_cate_code': '895'}, 
    {'child_cate_name': '浏览器', 'child_cate_code': '599'}, 
    ...
	]}, 
...

def get_category(self,response):    
        cate_content = self.wandou_category.parse_category(response)
        # ...
        for item in cate_content:
            child_cate = item['child_cate_codes']
            for cate in child_cate:
                cate_code = item['cate_code']
                cate_name = item['cate_name']
                child_cate_code = cate['child_cate_code']
                child_cate_name = cate['child_cate_name']
                
                page = 1 # 设置爬取起始页数
                if page == 1:
                    # 构造首页url
                    category_url = '{}{}_{}' .format(self.url, cate_code, child_cate_code)
                else:
                    params = {
                        'catId': cate_code,  # 类别
                        'subCatId': child_cate_code,  # 子类别
                        'page': page,
                        }
                    category_url = self.ajax_url + urlencode(params)
                dict = {'page':page,'cate_name':cate_name,'cate_code':cate_code,'child_cate_name':child_cate_name,'child_cate_code':child_cate_code}
                yield scrapy.Request(category_url,callback=self.parse,meta=dict)

def parse(self, response):
        if len(response.body) >= 100:  # 判断该页是否爬完，数值定为100是因为无内容时长度是87
            page = response.meta['page']
            cate_name = response.meta['cate_name']
            cate_code = response.meta['cate_code']
            child_cate_name = response.meta['child_cate_name']
            child_cate_code = response.meta['child_cate_code']

            if page == 1:
                contents = response
            else:
                jsonresponse = json.loads(response.body_as_unicode())
                contents = jsonresponse['data']['content']
                # response 是json,json内容是html，html 为文本不能直接使用.css 提取，要先转换
                contents = scrapy.Selector(text=contents, type="html")

            contents = contents.css('.card')
            for content in contents:
                # num += 1
                item = WandoujiaItem()
                item['cate_name'] = cate_name
                item['child_cate_name'] = child_cate_name
                item['app_name'] = self.clean_name(content.css('.name::text').extract_first())  
                item['install'] = content.css('.install-count::text').extract_first()
                item['volume'] = content.css('.meta span:last-child::text').extract_first()
                item['comment'] = content.css('.comment::text').extract_first().strip()
                item['icon_url'] = self.get_icon_url(content.css('.icon-wrap a img'),page)
                yield item
            
            # 递归爬下一页
            page += 1
            params = {
                    'catId': cate_code,  # 大类别
                    'subCatId': child_cate_code,  # 小类别
                    'page': page,
                    }
            ajax_url = self.ajax_url + urlencode(params)
            dict = {'page':page,'cate_name':cate_name,'cate_code':cate_code,'child_cate_name':child_cate_name,'child_cate_code':child_cate_code}
            yield scrapy.Request(ajax_url,callback=self.parse,meta=dict)

plt.style.use('ggplot')
colors = '#6D6D6D' #字体颜色
colorline = '#63AB47'  #红色CC2824  #豌豆荚绿
fontsize_title = 20
fontsize_text = 10

# 下载量总排名
def analysis_maxmin(data):
    data_max = (data[:10]).sort_values(by='install_count')
    data_max['install_count'] = (data_max['install_count'] / 100000000).round(1)
    data_max.plot.barh(x='app_name',y='install_count',color=colorline)
    for y, x in enumerate(list((data_max['install_count']))):
        plt.text(x + 0.1, y - 0.08, '%s' %
                 round(x, 1), ha='center', color=colors)

    plt.title('安装量最多的 10 款 App ?',color=colors)
    plt.xlabel('下载量(亿次)')
    plt.ylabel('App')
    plt.tight_layout()
    # plt.savefig('安装量最多的App.png',dpi=200)
    plt.show()

def analysis_distribution(data):
    data = data.loc[10:,:]
    data['install_count'] = data['install_count'].apply(lambda x:x/10000)
    bins = [0,1,10,100,1000,10000]
    group_names = ['1万以下','1-10万','10-100万','100-1000万','1000万-1亿']
    cats = pd.cut(data['install_count'],bins,labels=group_names)
    cats = pd.value_counts(cats)
    bar = Bar('App 下载数量分布','高达 94% 的 App 下载量低于1万')
    bar.use_theme('macarons')
    bar.add(
        'App 数量',
        list(cats.index),
        list(cats.values),
        is_label_show = True,
        xaxis_interval = 0,
        is_splitline_show = 0,
        )
    bar.render(path='App下载数量分布.png',pixel_ration=1)

include = data3.shape[0]
notinclude = data2.shape[0] - data3.shape[0]
sizes= [include,notinclude]
labels = [u'包含',u'不包含']
explode = [0,0.05]
plt.pie(
    sizes,
    autopct = '%.1f%%',
    labels = labels,
    colors = [colorline,'#7FC161'], # 豌豆荚绿
    shadow = False,
    startangle = 90,
    explode = explode,
    textprops = {'fontsize':14,'color':colors}
)
plt.title('豌豆荚仅包括酷安上一半的 App 数量',color=colorline,fontsize=16)
plt.axis('equal')
plt.axis('off')
plt.tight_layout()
plt.savefig('包含不保包含对比.png',dpi=200)
plt.show()