新增安居客网站小区爬取类

1 changed files with 216 additions and 0 deletions
--- a/anjuke.py
+++ b/anjuke.py
@ -0,0 +1,216 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+# 文件名：anjuke.py
+# 日期：2019-08-08
+
+"""
+AnJuke类的作用是模拟浏览器，逐页爬取安居客网站上单个城市的小区数据。
+通过创建类实例，并调用相应的方法将爬取的数据保存到.txt文件，或读取已有小区数据的.txt文件并插入到指定数据库表。
+一、创建AnJuke类的实例：
+ajk = AnJuKe(province, city, save_func)
+province 提供该字段作为保存文件名的参数之一和写进.txt文件的数据源之一
+city 提供该字段指定爬取单个城市的所有小区
+save_func 该字段仅支持两个预设值 ：1、file  2、mysql
+二、实例调用 ajk.start()方法
+start()方法通过判断 self.save_func的值，决定调用 save_file() 或 save_mysql()方法
+save_file()方法会先调用 get_page()方法获取并解析安居客的小区，再调用 write_file()方法写进.txt文件
+save_mysql()方法会先调用 read_file()方法读取指定的.txt文件，读取数据后调用 insert_mysql(data_list)方法将数据插入到指定数据库表
+另：
+1、save_file()方法中的cur_page默认为1，即默认从第一页开始爬取，每爬完一页就追加到.txt文件末尾。
+2、如果爬取时程序出错停止，则可以依据print打印的当前页数及小区，修改cur_page继续爬取。
+"""
+
+from bs4 import BeautifulSoup
+from urllib import parse
+from xpinyin import Pinyin
+from pymysql import connect
+import re
+import requests
+import time
+
+
+class AnJuKe:
+    # 安居客爬取类
+    def __init__(self, province, city, save_func):
+        # 初始化请求header、城市列表、文件路径
+        self.headers = {
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Language': 'zh-CN,zh;q=0.8',
+            'Referer': 'https: // wuhan.anjuke.com / sale /?from=navigation',
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'}
+        self.save_func = save_func
+        self.province = province
+        self.city = city
+        self.time = time.strftime("%Y-%m-%d", time.localtime())
+        self.file_path = 'C:\\Users\\Administrator\\Desktop\\anjuke_{}_{}_{}.txt'.format(self.province, self.city,
+                                                                                     self.time)
+
+    def get_page(self, city_py, cur_page):
+        # 获取并返回某一页的小区页面对象
+        url = 'https://{}.anjuke.com/community/p{}/'.format(city_py, cur_page)
+        response = requests.get(url, headers=self.headers, timeout=30)
+        return response
+
+    def get_location(self, city_py, estate_id):
+        # 获取并返回小区经纬度
+        url = 'https://{}.anjuke.com/community/view/{}'.format(city_py, estate_id)
+        response = requests.get(url, headers=self.headers, timeout=30)
+        bs = BeautifulSoup(response.text, 'html.parser')
+        comm_title = bs.find_all('div', class_='comm-title')
+        for title in comm_title:
+            a_tag = title.find_all('a', class_='map-link')
+            for link in a_tag:
+                get_link = link.get('href')
+                # 解析url后得到参数字典query_dict，解析字典后得到经纬度
+                url_data = parse.urlparse(get_link)
+                query_dict = parse.parse_qs(url_data.fragment)
+                latitude = query_dict['l1'][0]
+                longitude = query_dict['l2'][0]
+        return latitude, longitude
+
+    def has_next(self, city_py, cur_page):
+        # 判断当前页面是否有下一页并返回结果
+        response = self.get_page(city_py, cur_page)
+        bs = BeautifulSoup(response.text, 'html.parser')
+        multi_page = bs.find_all('div', class_='multi-page')
+        for page in multi_page:
+            next_page = page.find_all('a', class_='aNxt')
+            for anxt in next_page:
+                next_flag = anxt.attrs['href']
+                return next_flag
+
+    def write_file(self, estate_id, estate_name, lat, lng, estate_district, estate_town, estate_street, fw):
+        # 将爬取的'城市-小区-街道-纬度-经度'保存到文件
+        province = self.province.strip()
+        city = self.city.strip()
+        id = estate_id.strip()
+        estate = estate_name.strip()
+        lat = lat.strip()
+        lng = lng.strip()
+        district = estate_district.strip()
+        town = estate_town.strip()
+        street = estate_street.strip()
+        fw.write(
+                province + '^' + city + '^' + id + '^' + estate + '^' + lat + '^' + lng + '^' +
+                district + '^' + town + '^' + street + '\n')
+
+    def read_file(self):
+        open_file = open(self.file_path, "r", encoding='UTF-8-sig')
+        data_list = []
+        for line in open_file:
+            data = line.rstrip('\n').split('^')
+            data_list.append(data)
+        print(data_list)
+        open_file.close()
+        return data_list
+
+    def insert_mysql(self, data_list):
+        # 保存到数据库 '47.106.79.88', 3306, '1hjz', 'root', 'u2018@U2018'
+        print("连接数据库")
+        db = connect(host='47.106.79.88', port=3306, db='1hjz', user='root', passwd='u2018@U2018')
+        cursor = db.cursor()
+        insert_sql = "INSERT INTO import_estate(province_name,city_name,estate_id,estate_name,lat,lng,estate_district,estate_town,estate_street) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)"
+        try:
+            cursor.execute('''CREATE TABLE if not exists `import_estate` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
+  `province_name` varchar(255) DEFAULT '' COMMENT '省份名称',
+  `city_name` varchar(255) DEFAULT '' COMMENT '城市名称',
+  `estate_id` varchar(255) DEFAULT '' COMMENT '安居客的小区id',
+  `estate_name` varchar(255) DEFAULT '' COMMENT '小区名称',
+  `lat` decimal(10,6) DEFAULT '0.000000' COMMENT '纬度',
+  `lng` decimal(10,6) DEFAULT '0.000000' COMMENT '经度',
+  `estate_district` varchar(255) DEFAULT '' COMMENT '行政区',
+  `estate_town` varchar(255) DEFAULT '' COMMENT '区镇/县',
+  `estate_street` varchar(255) DEFAULT '' COMMENT '街道',
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT COMMENT='小区信息导入表';''')
+            cursor.executemany(insert_sql, data_list)
+            db.commit()
+        except Exception as e:
+            print(e)
+            db.rollback()
+        cursor.close()
+        db.close()
+
+    def save_mysql(self):
+        print("等待写入数据库...")
+        data_list = self.read_file()
+        if data_list:
+            self.insert_mysql(data_list)
+            print("写入数据库完成!")
+        else:
+            print("文件不存在或没有数据")
+
+    def save_file(self):
+        fw = open(self.file_path, "a", encoding='utf-8')
+        print("开始爬取{}小区数据，请稍等...".format(self.city))
+        cur_page = 1
+        # 获取城市全拼作为参数
+        pinyin = Pinyin()
+        city_py = str(pinyin.get_pinyin(self.city, ''))
+        while True:
+            self.parse_page(city_py, cur_page, fw)
+            if self.has_next(city_py, cur_page):
+                cur_page += 1
+            else:
+                break
+        fw.close()
+        print("小区数据已写入文件...")
+
+    def parse_page(self, city_py, cur_page, fw):
+        # 调用self.get_estate()方法得到小区页面对象，并结合其他类方法抓取页面信息
+        response = self.get_page(city_py, cur_page)
+        bs = BeautifulSoup(response.text, 'html.parser')
+        estate_list = bs.find_all('div', class_='li-itemmod')
+        for li in estate_list:
+            infos = li.find_all('div', class_='li-info')
+            for info in infos:
+                # 通过h3标签中的标题找到小区名:estate_name
+                titles = info.find_all('h3')
+                for title in titles:
+                    estate_name = title.get_text().strip()
+                    # 通过a标签中的链接获取小区id:estate_id  href = 'https://guangzhou/anjuke.com/community/view/607976'
+                    a_tag = title.find_all('a')
+                    for link in a_tag:
+                        get_link = link.attrs['href']
+                        pattern = re.compile('[0-9]{1,}')
+                        estate_id = pattern.findall(get_link)[0]
+                        # 调用self.get_location()方法获取小区经纬度
+                        estate_location = self.get_location(city_py, estate_id)
+                        lat = estate_location[0]
+                        lng = estate_location[1]
+                # 获取小区的地址
+                address_list = info.find_all('address')
+                for address in address_list:
+                    ajk_address = address.get_text().strip().split('］')
+                    estate_district = ajk_address[0].split('［')[1].split('-')[0]
+                    estate_town = ajk_address[0].split('［')[1].split('-')[1]
+                    estate_street = ajk_address[1]
+            self.write_file(estate_id, estate_name, lat, lng, estate_district, estate_town, estate_street, fw)
+            # 关键打印信息：爬取时出错能判断已经抓取到哪一页的哪个小区
+            print(
+                    '省份：' + self.province + '、城市：' + self.city + '、id：' + estate_id + '、小区：' + estate_name + '、纬度：' + lat +
+                    '、经度：' + lng + '、行政区:' + estate_district + '、区镇/县：' + estate_town + '、街道:' + estate_street +
+                    '、当前页：' + str(cur_page))
+
+    def start(self):
+        if self.save_func == 'file':
+            self.save_file()
+        elif self.save_func == 'mysql':
+            self.save_mysql()
+        else:
+            print('不支持的保存方式')
+
+
+if __name__ == '__main__':
+    # 目标城市 广东省：广州,深圳,清远,湛江；湖南省 ：长沙,衡阳
+    start_time = time.time()
+    # province = '广东省'
+    province = '湖南省'
+    city = '衡阳'
+    # save_func = 'file'
+    save_func = 'mysql'
+    spider = AnJuKe(province, city, save_func)
+    spider.start()
+    end_time = time.time()
+    print("已完成,用时%.2f秒" % (end_time - start_time))