python爬虫 设置了去重但是还是爬几条后就无限爬同一个URL，下面是去重的代
class UrlManager(object):
def __init__(self):
self.new_urls = set()
self.old_urls = set()
def add_new_url(self, url):
if url is None:
return
if url not in self.new_urls and url not in self.old_urls:
self.new_urls.add(url)
def add_new_urls(self, urls):
if urls is None or len(urls) == 0:
return
for url in urls:
self.new_urls.add(url)
def has_new_url(self):
return len(self.new_urls) != 0
def get_new_url(self):
new_url = self.new_urls.pop()
self.old_urls.add(new_url)
return new_url |
免责声明：本内容仅代表回答会员见解不代表天盟观点，请谨慎对待。
版权声明：作者保留权利，不代表天盟立场。
|
|
|
|
|
|
|
|
天盟APP下载
微信扫一扫访问
