Scrapy安装、爬虫入门教程、爬虫实例（豆瓣电影爬虫）

  1 #coding=utf-8   2 import sys   3 reload(sys)   4 #python默认环境编码时ascii   5 sys.setdefaultencoding("utf-8")   6 from scrapy.spider import BaseSpider   7 from scrapy.http import Request   8 from scrapy.selector import HtmlXPathSelector   9 from tutorial.items import TutorialItem  10 import re  11   12 class DoubanSpider(BaseSpider):  13     name = "douban"  14     allowed_domains = ["movie.douban.com"]  15     start_urls = []  16   17     def start_requests(self):  18         file_object = open('movie_name.txt','r')  19   20         try:  21             url_head = "http://movie.douban.com/subject_search?search_text="  22             for line in file_object:  23                 self.start_urls.append(url_head + line)  24               25             for url in self.start_urls:  26                 yield self.make_requests_from_url(url)  27         finally:  28             file_object.close()  29             #years_object.close()  30   31     def parse(self, response):  32         #open("test.html",'wb').write(response.body)  33         hxs = HtmlXPathSelector(response)  34         #movie_name = hxs.select('//*[@id="content"]/div/div[1]/div[2]/table[1]/tr/td[1]/a/@title').extract()  35         movie_link = hxs.select('//*[@id="content"]/div/div[1]/div[2]/table[1]/tr/td[1]/a/@href').extract()  36         #movie_desc = hxs.select('//*[@id="content"]/div/div[1]/div[2]/table[1]/tr/td[2]/div/p/text()').extract()  37   38         if movie_link:  39             yield Request(movie_link[0],callback=self.parse_item)  40           41           42     def parse_item(self,response):  43         hxs = HtmlXPathSelector(response)  44         movie_name = hxs.select('//*[@id="content"]/h1/span[1]/text()').extract()  45         movie_director = hxs.select('//*[@id="info"]/span[1]/span[2]/a/text()').extract()  46         movie_writer = hxs.select('//*[@id="info"]/span[2]/span[2]/a/text()').extract()  47         #爬取电影详情需要在已有对象中继续爬取  48         movie_description_paths = hxs.select('//*[@id="link-report"]')  49         movie_description = []  50         for movie_description_path in movie_description_paths:  51             movie_description = movie_description_path.select('.//*[@property="v:summary"]/text()').extract()  52   53         #提取演员需要从已有的xPath对象中继续爬我要的内容  54         movie_roles_paths = hxs.select('//*[@id="info"]/span[3]/span[2]')  55         movie_roles = []  56         for movie_roles_path in movie_roles_paths:  57             movie_roles = movie_roles_path.select('.//*[@rel="v:starring"]/text()').extract()  58   59         #获取电影详细信息序列  60         movie_detail = hxs.select('//*[@id="info"]').extract()  61   62         item = TutorialItem()  63         item['movie_name'] = ''.join(movie_name).strip().replace(',',';').replace('/'','///'').replace('/"','///"').replace(':',';')  64         #item['movie_link'] = movie_link[0]  65         item['movie_director'] = movie_director[0].strip().replace(',',';').replace('/'','///'').replace('/"','///"').replace(':',';') if len(movie_director) > 0 else ''  66         #由于逗号是拿来分割电影所有信息的，所以需要处理逗号;引号也要处理，否则插入数据库会有问题  67         item['movie_description'] = movie_description[0].strip().replace(',',';').replace('/'','///'').replace('/"','///"').replace(':',';') if len(movie_description) > 0 else ''  68         item['movie_writer'] = ';'.join(movie_writer).strip().replace(',',';').replace('/'','///'').replace('/"','///"').replace(':',';')  69         item['movie_roles'] = ';'.join(movie_roles).strip().replace(',',';').replace('/'','///'').replace('/"','///"').replace(':',';')  70         #item['movie_language'] = movie_language[0].strip() if len(movie_language) > 0 else ''  71         #item['movie_date'] = ''.join(movie_date).strip()  72         #item['movie_long'] = ''.join(movie_long).strip()  73           74         #电影详情信息字符串  75         movie_detail_str = ''.join(movie_detail).strip()  76         #print movie_detail_str  77   78         movie_language_str = ".*语言:</span> (.+?)<br><span.*".decode("utf8")  79         movie_date_str = ".*上映日期:</span> <span property=/"v:initialReleaseDate/" content=/"(/S+?)/">(/S+?)</span>.*".decode("utf8")  80         movie_long_str = ".*片长:</span> <span property=/"v:runtime/" content=/"(/d+).*".decode("utf8")  81           82         pattern_language =re.compile(movie_language_str,re.S)  83         pattern_date = re.compile(movie_date_str,re.S)  84         pattern_long = re.compile(movie_long_str,re.S)  85           86           87         movie_language = re.search(pattern_language,movie_detail_str)  88         movie_date = re.search(pattern_date,movie_detail_str)  89         movie_long = re.search(pattern_long,movie_detail_str)  90   91         item['movie_language'] = ""  92         if movie_language:  93             item['movie_language'] = movie_language.group(1).strip().replace(',',';').replace('/'','///'').replace('/"','///"').replace(':',';')  94         #item['movie_detail'] = ''.join(movie_detail).strip()  95   96         item['movie_date'] = ""  97         if movie_date:  98             item['movie_date'] = movie_date.group(1).strip().replace(',',';').replace('/'','///'').replace('/"','///"').replace(':',';')  99  100         item['movie_long'] = "" 101         if movie_long: 102             item['movie_long'] = movie_long.group(1) 103  104         yield item