#!/usr/bin/python # -*- coding: UTF-8 -*- # 文件名：anjuke.py # 日期：2019-08-08 """ AnJuke类的作用是模拟浏览器，逐页爬取安居客网站上单个城市的小区数据。 通过创建类实例，并调用相应的方法将爬取的数据保存到.txt文件，或读取已有小区数据的.txt文件并插入到指定数据库表。 一、创建AnJuke类的实例： ajk = AnJuKe(province, city, save_func) province 提供该字段作为保存文件名的参数之一和写进.txt文件的数据源之一 city 提供该字段指定爬取单个城市的所有小区 save_func 该字段仅支持两个预设值 ：1、file 2、mysql 二、实例调用 ajk.start()方法 start()方法通过判断 self.save_func的值，决定调用 save_file() 或 save_mysql()方法 save_file()方法会先调用 get_page()方法获取并解析安居客的小区，再调用 write_file()方法写进.txt文件 save_mysql()方法会先调用 read_file()方法读取指定的.txt文件，读取数据后调用 insert_mysql(data_list)方法将数据插入到指定数据库表 另： 1、save_file()方法中的cur_page默认为1，即默认从第一页开始爬取，每爬完一页就追加到.txt文件末尾。 2、如果爬取时程序出错停止，则可以依据print打印的当前页数及小区，修改cur_page继续爬取。 """ from bs4 import BeautifulSoup from urllib import parse from xpinyin import Pinyin from pymysql import connect import re import requests import time class AnJuKe: # 安居客爬取类 def __init__(self, province, city, save_func): # 初始化请求header、城市列表、文件路径 self.headers = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Language': 'zh-CN,zh;q=0.8', 'Referer': 'https: // wuhan.anjuke.com / sale /?from=navigation', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'} self.save_func = save_func self.province = province self.city = city self.time = time.strftime("%Y-%m-%d", time.localtime()) self.file_path = 'C:\\Users\\Administrator\\Desktop\\anjuke_{}_{}_{}.txt'.format(self.province, self.city, self.time) def get_page(self, city_py, cur_page): # 获取并返回某一页的小区页面对象 url = 'https://{}.anjuke.com/community/p{}/'.format(city_py, cur_page) response = requests.get(url, headers=self.headers, timeout=30) return response def get_location(self, city_py, estate_id): # 获取并返回小区经纬度 url = 'https://{}.anjuke.com/community/view/{}'.format(city_py, estate_id) response = requests.get(url, headers=self.headers, timeout=30) bs = BeautifulSoup(response.text, 'html.parser') comm_title = bs.find_all('div', class_='comm-title') for title in comm_title: a_tag = title.find_all('a', class_='map-link') for link in a_tag: get_link = link.get('href') # 解析url后得到参数字典query_dict，解析字典后得到经纬度 url_data = parse.urlparse(get_link) query_dict = parse.parse_qs(url_data.fragment) latitude = query_dict['l1'][0] longitude = query_dict['l2'][0] return latitude, longitude def has_next(self, city_py, cur_page): # 判断当前页面是否有下一页并返回结果 response = self.get_page(city_py, cur_page) bs = BeautifulSoup(response.text, 'html.parser') multi_page = bs.find_all('div', class_='multi-page') for page in multi_page: next_page = page.find_all('a', class_='aNxt') for anxt in next_page: next_flag = anxt.attrs['href'] return next_flag def write_file(self, estate_id, estate_name, lat, lng, estate_district, estate_town, estate_street, fw): # 将爬取的'城市-小区-街道-纬度-经度'保存到文件 province = self.province.strip() city = self.city.strip() id = estate_id.strip() estate = estate_name.strip() lat = lat.strip() lng = lng.strip() district = estate_district.strip() town = estate_town.strip() street = estate_street.strip() fw.write( province + '^' + city + '^' + id + '^' + estate + '^' + lat + '^' + lng + '^' + district + '^' + town + '^' + street + '\n') def read_file(self): open_file = open(self.file_path, "r", encoding='UTF-8-sig') data_list = [] for line in open_file: data = line.rstrip('\n').split('^') data_list.append(data) print(data_list) open_file.close() return data_list def insert_mysql(self, data_list): # 保存到数据库 '47.106.79.88', 3306, '1hjz', 'root', 'u2018@U2018' print("连接数据库") db = connect(host='47.106.79.88', port=3306, db='1hjz', user='root', passwd='u2018@U2018') cursor = db.cursor() insert_sql = "INSERT INTO import_estate(province_name,city_name,estate_id,estate_name,lat,lng,estate_district,estate_town,estate_street) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)" try: cursor.execute('''CREATE TABLE if not exists `import_estate` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, `province_name` varchar(255) DEFAULT '' COMMENT '省份名称', `city_name` varchar(255) DEFAULT '' COMMENT '城市名称', `estate_id` varchar(255) DEFAULT '' COMMENT '安居客的小区id', `estate_name` varchar(255) DEFAULT '' COMMENT '小区名称', `lat` decimal(10,6) DEFAULT '0.000000' COMMENT '纬度', `lng` decimal(10,6) DEFAULT '0.000000' COMMENT '经度', `estate_district` varchar(255) DEFAULT '' COMMENT '行政区', `estate_town` varchar(255) DEFAULT '' COMMENT '区镇/县', `estate_street` varchar(255) DEFAULT '' COMMENT '街道', PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT COMMENT='小区信息导入表';''') cursor.executemany(insert_sql, data_list) db.commit() except Exception as e: print(e) db.rollback() cursor.close() db.close() def save_mysql(self): print("等待写入数据库...") data_list = self.read_file() if data_list: self.insert_mysql(data_list) print("写入数据库完成!") else: print("文件不存在或没有数据") def save_file(self): fw = open(self.file_path, "a", encoding='utf-8') print("开始爬取{}小区数据，请稍等...".format(self.city)) cur_page = 1 # 获取城市全拼作为参数 pinyin = Pinyin() city_py = str(pinyin.get_pinyin(self.city, '')) while True: self.parse_page(city_py, cur_page, fw) if self.has_next(city_py, cur_page): cur_page += 1 else: break fw.close() print("小区数据已写入文件...") def parse_page(self, city_py, cur_page, fw): # 调用self.get_estate()方法得到小区页面对象，并结合其他类方法抓取页面信息 response = self.get_page(city_py, cur_page) bs = BeautifulSoup(response.text, 'html.parser') estate_list = bs.find_all('div', class_='li-itemmod') for li in estate_list: infos = li.find_all('div', class_='li-info') for info in infos: # 通过h3标签中的标题找到小区名:estate_name titles = info.find_all('h3') for title in titles: estate_name = title.get_text().strip() # 通过a标签中的链接获取小区id:estate_id href = 'https://guangzhou/anjuke.com/community/view/607976' a_tag = title.find_all('a') for link in a_tag: get_link = link.attrs['href'] pattern = re.compile('[0-9]{1,}') estate_id = pattern.findall(get_link)[0] # 调用self.get_location()方法获取小区经纬度 estate_location = self.get_location(city_py, estate_id) lat = estate_location[0] lng = estate_location[1] # 获取小区的地址 address_list = info.find_all('address') for address in address_list: ajk_address = address.get_text().strip().split('］') estate_district = ajk_address[0].split('［')[1].split('-')[0] estate_town = ajk_address[0].split('［')[1].split('-')[1] estate_street = ajk_address[1] self.write_file(estate_id, estate_name, lat, lng, estate_district, estate_town, estate_street, fw) # 关键打印信息：爬取时出错能判断已经抓取到哪一页的哪个小区 print( '省份：' + self.province + '、城市：' + self.city + '、id：' + estate_id + '、小区：' + estate_name + '、纬度：' + lat + '、经度：' + lng + '、行政区:' + estate_district + '、区镇/县：' + estate_town + '、街道:' + estate_street + '、当前页：' + str(cur_page)) def start(self): if self.save_func == 'file': self.save_file() elif self.save_func == 'mysql': self.save_mysql() else: print('不支持的保存方式') if __name__ == '__main__': # 目标城市 广东省：广州,深圳,清远,湛江；湖南省 ：长沙,衡阳 start_time = time.time() # province = '广东省' province = '湖南省' city = '衡阳' # save_func = 'file' save_func = 'mysql' spider = AnJuKe(province, city, save_func) spider.start() end_time = time.time() print("已完成,用时%.2f秒" % (end_time - start_time))