二十一、数据采集实战第2关网站爬取策略任务描述本关任务编写一个爬虫实现深度优先爬虫。相关知识主要介绍两种爬虫爬取策略1. 深度优先爬虫 2. 广度优先爬虫。深度优先爬虫一路到底在一个网页中当一个超链被选择后被链接的网页将执行深度优先搜索即在搜索其余的超链结果之前必须先完整地搜索单独的一条链。深度优先搜索沿着网页上的超链走到不能再深入为止然后返回到某一个网页再继续选择该网页中的其他超链。当不再有其他超链可选择时说明搜索已经结束。示例爬取顺序为1-2-4-8-5-3-6-7广度优先爬虫逐层爬取广度优先爬虫过程就是从一系列的种子节点开始把这些网页中的“子节点” 提取出来放入队列中依次进行抓取被处理过的链接需要放入一张表中。每次新处理一个链接之前需要查看这个链接是否已经存在于表中。如果存在证明链接已经处理过 跳过不做处理否则进行下一步处理。示例爬取顺序为1-2-3-4-5-6-7-8编程要求请仔细阅读右侧代码结合相关知识在 Begin-End 区域内进行代码补充编写一个爬虫实现深度优先爬虫爬取的网站为 www.baidu.com。测试说明平台会对你编写的代码进行测试下方预期输出随着网页的更新中间的数据会有一定的变化。预期输出Add the seeds url [http://www.baidu.com] to the unvisited url listPop out one url http://www.baidu.com from unvisited url listGet 10 new linksVisited url count: 1Visited deepth: 110 unvisited links:Pop out one url http://news.baidu.com from unvisited url listGet 52 new linksVisited url count: 2Visited deepth: 2Pop out one url http://www.hao123.com from unvisited url listGet 311 new linksVisited url count: 3Visited deepth: 2Pop out one url http://map.baidu.com from unvisited url listGet 0 new linksVisited url count: 4Visited deepth: 2Pop out one url http://v.baidu.com from unvisited url listGet 566 new linksVisited url count: 5Visited deepth: 2Pop out one url http://tieba.baidu.com from unvisited url listGet 21 new linksVisited url count: 6Visited deepth: 2Pop out one url http://www.baidu.com/bdorz/login.gif?logintplmnuhttp%3A%2F%2Fwww.baidu.com%2f%3fbdorz_come%3d1 from unvisited url listGet 1 new linksVisited url count: 7Visited deepth: 2Pop out one url http://home.baidu.com from unvisited url listGet 27 new linksVisited url count: 8Visited deepth: 2Pop out one url http://ir.baidu.com from unvisited url listGet 22 new linksVisited url count: 9Visited deepth: 2Pop out one url http://www.baidu.com/duty/ from unvisited url listGet 1 new linksVisited url count: 10Visited deepth: 2Pop out one url http://jianyi.baidu.com/ from unvisited url listGet 4 new linksVisited url count: 11Visited deepth: 22 unvisited links:Pop out one url http://baozhang.baidu.com/guarantee/ from unvisited url listGet 0 new linksVisited url count: 12Visited deepth: 3Pop out one url http://ir.baidu.com/phoenix.zhtml?c188488pirol-irhome from unvisited url listGet 22 new linksVisited url count: 13Visited deepth: 322 unvisited links:注意右侧预期输出为部分输出。开始你的任务吧祝你成功from bs4 import BeautifulSoupimport requestsimport re#自定义队列类class linkQuence:def __init__(self):# 已访问的url集合self.visted []# 待访问的url集合self.unVisited []# 获取访问过的url队列def getVisitedUrl(self):return self.visted# 获取未访问的url队列def getUnvisitedUrl(self):return self.unVisited# 添加到访问过得url队列中def addVisitedUrl(self, url):self.visted.append(url)# 移除访问过得urldef removeVisitedUrl(self, url):self.visted.remove(url)# 未访问过得url出队列 深度优先 pop() 后进先出栈def unVisitedUrlDeQuence(self):try:return self.unVisited.pop()except:return None# 保证每个url只被访问一次def addUnvisitedUrl(self, url):if url ! and url not in self.visted and url not in self.unVisited:self.unVisited.insert(0, url)# 获得已访问的url数目def getVisitedUrlCount(self):return len(self.visted)# 获得未访问的url数目def getUnvistedUrlCount(self):return len(self.unVisited)# 判断未访问的url队列是否为空def unVisitedUrlsEnmpy(self):return len(self.unVisited) 0class MyCrawler:def __init__(self, seeds):# 初始化当前抓取的深度self.current_deepth 1# 使用种子初始化url队列self.linkQuence linkQuence()if isinstance(seeds, str):self.linkQuence.addUnvisitedUrl(seeds)if isinstance(seeds, list):for i in seeds:self.linkQuence.addUnvisitedUrl(i)print(Add the seeds url %s to the unvisited url list % str(self.linkQuence.unVisited))# 抓取过程主函数def crawling(self, seeds, crawl_deepth):# ********** Begin **********## 循环条件抓取深度不超过crawl_deepthwhile self.current_deepth crawl_deepth:# 循环条件待抓取的链接不空while not self.linkQuence.unVisitedUrlsEnmpy():# 队头url出队列 深度优先 popurl self.linkQuence.unVisitedUrlDeQuence()print(Pop out one url %s from unvisited url list % url)# 获取超链接new_links self.getHyperLinks(url)print(fGet {len(new_links)} new links)# 将url放入已访问的url中self.linkQuence.addVisitedUrl(url)print(fVisited url count: {self.linkQuence.getVisitedUrlCount()})print(fVisited deepth: {self.current_deepth})if self.linkQuence.getUnvistedUrlCount() 0:print(f{self.linkQuence.getUnvistedUrlCount()} unvisited links:)# 未访问的url入列for link in new_links:self.linkQuence.addUnvisitedUrl(link)self.current_deepth 1# ********** End **********## 获取源码中得超链接def getHyperLinks(self, url):# ********** Begin **********#links []source self.getPageSource(url)if source is None:return linkssoup BeautifulSoup(source, html.parser)a_list soup.find_all(a, hrefTrue)for a in a_list:href a[href]# 简单处理http开头链接if href.startswith(http):links.append(href)return links# ********** End **********## 获取网页源码def getPageSource(self, url):# ********** Begin **********#headers {User‑Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/100.0.0.0 Safari/537.36}try:resp requests.get(url, headersheaders, timeout3)resp.encoding resp.apparent_encodingreturn resp.textexcept Exception:return None# ********** End **********#def main(seeds, crawl_deepth):craw MyCrawler(seeds)craw.crawling(seeds, crawl_deepth)if __name__ __main__:main(http://www.baidu.com, 3)有任何问题都可以随时关注私信 头歌实践教学平台：数据科学与大数据技术导论（二十一2）
二十一、数据采集实战第2关网站爬取策略任务描述本关任务编写一个爬虫实现深度优先爬虫。相关知识主要介绍两种爬虫爬取策略1. 深度优先爬虫 2. 广度优先爬虫。深度优先爬虫一路到底在一个网页中当一个超链被选择后被链接的网页将执行深度优先搜索即在搜索其余的超链结果之前必须先完整地搜索单独的一条链。深度优先搜索沿着网页上的超链走到不能再深入为止然后返回到某一个网页再继续选择该网页中的其他超链。当不再有其他超链可选择时说明搜索已经结束。示例爬取顺序为1-2-4-8-5-3-6-7广度优先爬虫逐层爬取广度优先爬虫过程就是从一系列的种子节点开始把这些网页中的“子节点” 提取出来放入队列中依次进行抓取被处理过的链接需要放入一张表中。每次新处理一个链接之前需要查看这个链接是否已经存在于表中。如果存在证明链接已经处理过 跳过不做处理否则进行下一步处理。示例爬取顺序为1-2-3-4-5-6-7-8编程要求请仔细阅读右侧代码结合相关知识在 Begin-End 区域内进行代码补充编写一个爬虫实现深度优先爬虫爬取的网站为 www.baidu.com。测试说明平台会对你编写的代码进行测试下方预期输出随着网页的更新中间的数据会有一定的变化。预期输出Add the seeds url [http://www.baidu.com] to the unvisited url listPop out one url http://www.baidu.com from unvisited url listGet 10 new linksVisited url count: 1Visited deepth: 110 unvisited links:Pop out one url http://news.baidu.com from unvisited url listGet 52 new linksVisited url count: 2Visited deepth: 2Pop out one url http://www.hao123.com from unvisited url listGet 311 new linksVisited url count: 3Visited deepth: 2Pop out one url http://map.baidu.com from unvisited url listGet 0 new linksVisited url count: 4Visited deepth: 2Pop out one url http://v.baidu.com from unvisited url listGet 566 new linksVisited url count: 5Visited deepth: 2Pop out one url http://tieba.baidu.com from unvisited url listGet 21 new linksVisited url count: 6Visited deepth: 2Pop out one url http://www.baidu.com/bdorz/login.gif?logintplmnuhttp%3A%2F%2Fwww.baidu.com%2f%3fbdorz_come%3d1 from unvisited url listGet 1 new linksVisited url count: 7Visited deepth: 2Pop out one url http://home.baidu.com from unvisited url listGet 27 new linksVisited url count: 8Visited deepth: 2Pop out one url http://ir.baidu.com from unvisited url listGet 22 new linksVisited url count: 9Visited deepth: 2Pop out one url http://www.baidu.com/duty/ from unvisited url listGet 1 new linksVisited url count: 10Visited deepth: 2Pop out one url http://jianyi.baidu.com/ from unvisited url listGet 4 new linksVisited url count: 11Visited deepth: 22 unvisited links:Pop out one url http://baozhang.baidu.com/guarantee/ from unvisited url listGet 0 new linksVisited url count: 12Visited deepth: 3Pop out one url http://ir.baidu.com/phoenix.zhtml?c188488pirol-irhome from unvisited url listGet 22 new linksVisited url count: 13Visited deepth: 322 unvisited links:注意右侧预期输出为部分输出。开始你的任务吧祝你成功from bs4 import BeautifulSoupimport requestsimport re#自定义队列类class linkQuence:def __init__(self):# 已访问的url集合self.visted []# 待访问的url集合self.unVisited []# 获取访问过的url队列def getVisitedUrl(self):return self.visted# 获取未访问的url队列def getUnvisitedUrl(self):return self.unVisited# 添加到访问过得url队列中def addVisitedUrl(self, url):self.visted.append(url)# 移除访问过得urldef removeVisitedUrl(self, url):self.visted.remove(url)# 未访问过得url出队列 深度优先 pop() 后进先出栈def unVisitedUrlDeQuence(self):try:return self.unVisited.pop()except:return None# 保证每个url只被访问一次def addUnvisitedUrl(self, url):if url ! and url not in self.visted and url not in self.unVisited:self.unVisited.insert(0, url)# 获得已访问的url数目def getVisitedUrlCount(self):return len(self.visted)# 获得未访问的url数目def getUnvistedUrlCount(self):return len(self.unVisited)# 判断未访问的url队列是否为空def unVisitedUrlsEnmpy(self):return len(self.unVisited) 0class MyCrawler:def __init__(self, seeds):# 初始化当前抓取的深度self.current_deepth 1# 使用种子初始化url队列self.linkQuence linkQuence()if isinstance(seeds, str):self.linkQuence.addUnvisitedUrl(seeds)if isinstance(seeds, list):for i in seeds:self.linkQuence.addUnvisitedUrl(i)print(Add the seeds url %s to the unvisited url list % str(self.linkQuence.unVisited))# 抓取过程主函数def crawling(self, seeds, crawl_deepth):# ********** Begin **********## 循环条件抓取深度不超过crawl_deepthwhile self.current_deepth crawl_deepth:# 循环条件待抓取的链接不空while not self.linkQuence.unVisitedUrlsEnmpy():# 队头url出队列 深度优先 popurl self.linkQuence.unVisitedUrlDeQuence()print(Pop out one url %s from unvisited url list % url)# 获取超链接new_links self.getHyperLinks(url)print(fGet {len(new_links)} new links)# 将url放入已访问的url中self.linkQuence.addVisitedUrl(url)print(fVisited url count: {self.linkQuence.getVisitedUrlCount()})print(fVisited deepth: {self.current_deepth})if self.linkQuence.getUnvistedUrlCount() 0:print(f{self.linkQuence.getUnvistedUrlCount()} unvisited links:)# 未访问的url入列for link in new_links:self.linkQuence.addUnvisitedUrl(link)self.current_deepth 1# ********** End **********## 获取源码中得超链接def getHyperLinks(self, url):# ********** Begin **********#links []source self.getPageSource(url)if source is None:return linkssoup BeautifulSoup(source, html.parser)a_list soup.find_all(a, hrefTrue)for a in a_list:href a[href]# 简单处理http开头链接if href.startswith(http):links.append(href)return links# ********** End **********## 获取网页源码def getPageSource(self, url):# ********** Begin **********#headers {User‑Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/100.0.0.0 Safari/537.36}try:resp requests.get(url, headersheaders, timeout3)resp.encoding resp.apparent_encodingreturn resp.textexcept Exception:return None# ********** End **********#def main(seeds, crawl_deepth):craw MyCrawler(seeds)craw.crawling(seeds, crawl_deepth)if __name__ __main__:main(http://www.baidu.com, 3)有任何问题都可以随时关注私信