Python For 和 While 循环爬取不确定页数的网页

class Get_page():
    def __init__(self):
        # ajax 请求url
        self.ajax_url = 'https://www.wandoujia.com/wdjweb/api/category/more'

    def get_page(self,page,cate_code,child_cate_code):
        params = {
            'catId': cate_code,
            'subCatId': child_cate_code,
            'page': page,
        }
        response = requests.get(self.ajax_url, headers=headers, params=params)
        content = response.json()['data']['content'] #提取json中的html页面数据
        return content

    def parse_page(self, content):
        # 解析网页内容
        contents = pq(content)('.card').items()
        data = []
        for content in contents:
            data1 = {
                'app_name': content('.name').text(),
                'install': content('.install-count').text(),
                'volume': content('.meta span:last-child').text(),
                'comment': content('.comment').text(),
            }
            data.append(data1)
        if data:
            # 写入MongoDB
            self.write_to_mongodb(data)
            
if __name__ == '__main__':
    # 实例化数据提取类
    wandou_page = Get_page()
    cate_code = 5029 # 影音播放大类别编号
    child_cate_code = 716 # 视频小类别编号
     for page in range(2, 100):
        print('*' * 50)
        print('正在爬取：第 %s 页' % page)
        content = wandou_page.get_page(page,cate_code,child_cate_code)
        # 添加循环判断，如果content 为空表示此页已经下载完成了,break 跳出循环
        if not content == '':
            wandou_page.parse_page(content)
            sleep = np.random.randint(3,6)
            time.sleep(sleep)
        else:
            print('该类别已下载完最后一页')
            break

page = 2 # 设置爬取起始页数
while True:
    print('*' * 50)
    print('正在爬取：第 %s 页' %page)
    content = wandou_page.get_page(page,cate_code,child_cate_code)
    if not content == '':
        wandou_page.parse_page(content)
        page += 1
        sleep = np.random.randint(3,6)
        time.sleep(sleep)
    else:
        print('该类别已下载完最后一页')
        break

page = 2 # 设置爬取起始页数
page_last = False # while 循环初始条件
while not page_last:
   #...
    else:
        # break
        page_last = True # 更改page_last 为 True 跳出循环

def start_requests(self):
    pages=[]
    for i in range(1,10):
        url='http://www.example.com/?page=%s'%i
        page = scrapy.Request(url,callback==self.pare)
        pages.append(page)
    return pages

def start_requests(self):
    cate_code = 5019 # 新闻阅读
    child_cate_code = 940 # 电子书
    print('*' * 50)
    pages = []
    for page in range(2,50):
        print('正在爬取：第 %s 页 ' %page)
        params = {
        'catId': cate_code,
        'subCatId': child_cate_code,
        'page': page,
        }
        category_url = self.ajax_url + urlencode(params)
        pa = yield scrapy.Request(category_url,callback=self.parse)
        pages.append(pa)
    return pages

def parse(self, response):
    if len(response.body) >= 100:  # 判断该页是否爬完，数值定为100是因为response无内容时的长度是87
        jsonresponse = json.loads(response.body_as_unicode())
        contents = jsonresponse['data']['content']
        # response 是json,json内容是html，html 为文本不能直接使用.css 提取，要先转换
        contents = scrapy.Selector(text=contents, type="html")
        contents = contents.css('.card')
        for content in contents:
            item = WandoujiaItem()
            item['app_name'] = content.css('.name::text').extract_first()
            item['install'] = content.css('.install-count::text').extract_first()
            item['volume'] = content.css('.meta span:last-child::text').extract_first()
            item['comment'] = content.css('.comment::text').extract_first().strip()
            yield item

{"state":{"code":2000000,"msg":"Ok","tips":""},"data":{"currPage":-1,"content":""}}

contents = scrapy.Selector(text=contents, type="html")

def start_requests(self):
        page = 2 # 设置爬取起始页数
        dict = {'page':page,'cate_code':cate_code,'child_cate_code':child_cate_code} # meta传递参数
        yield scrapy.Request(category_url,callback=self.parse,meta=dict)

def parse(self, response):
    if len(response.body) >= 100:  # 判断该页是否爬完，数值定为100是因为无内容时长度是87
        page = response.meta['page']
        cate_code = response.meta['cate_code']
        child_cate_code = response.meta['child_cate_code']
	   #...
       for content in contents:
        	yield item
        
        # while循环构造url递归爬下一页
        page += 1
        params = {
                'catId': cate_code,
                'subCatId': child_cate_code,
                'page': page,
                }
        ajax_url = self.ajax_url + urlencode(params)
        dict = {'page':page,'cate_code':cate_code,'child_cate_code':child_cate_code}
        yield scrapy.Request(ajax_url,callback=self.parse,meta=dict)