selenium爬取全国房价排行榜数据可视化

爬取聚合数据的全国房价排行榜

from selenium import webdriver
from bs4 import BeautifulSoup
import csv
from selenium import webdriver
from fake_useragent import UserAgent
import random
import subprocess
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import os

ips = []
with open('ip.txt', 'r') as f:
    for line in f:
        ip = line.strip()
        ips.append(ip.strip())

# 启动Chrome浏览器调试服务
subprocess.Popen('cmd', shell=True)
subprocess.Popen('"chrome-win64\chrome.exe" --remote-debugging-port=9222', shell=True)

chrome_options = webdriver.ChromeOptions()
chrome_options.add_experimental_option("debuggerAddress", "localhost:9222")
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable‐gpu')
chrome_options.add_argument("--disable-blink-features=AutomationControlled")
chrome_options.add_argument('--proxy-server=http://' + random.choice(ips))
chrome_options.add_argument(f"user-agent={UserAgent().random}")
driver = webdriver.Chrome(options=chrome_options)


# 打开网页
url = 'https://fangjia.gotohui.com/'
driver.get(url)

# 等待表格加载完成
table_locator = (By.CSS_SELECTOR, 'body > div.container.recommend > div > div.listcontent.top15 > div.toplist.w900 > table')
table = WebDriverWait(driver, 10).until(EC.presence_of_element_located(table_locator))

# 获取表格的HTML内容
table_html = table.get_attribute('outerHTML')

# 使用 BeautifulSoup 解析表格
soup = BeautifulSoup(table_html, 'html.parser')

folder_path = os.getcwd()+"/data/房价/"
if not os.path.exists(folder_path):
    os.makedirs(folder_path)
    
# 打开CSV文件进行写入
with open(folder_path+'房价排行榜.csv', 'w', newline='', encoding='utf-8') as csvfile:
    writer = csv.writer(csvfile)
    writer.writerow(['排名', '城市', '二手房(元/㎡)', '同比(去年)', '环比(上月)','新房(元/㎡)'])
    rows = soup.find('tbody').find_all('tr')

    # 遍历每一行并提取数据
    for row in rows:
        cells = row.find_all('td')
        row_data = [cell.text.strip() for cell in cells]
        writer.writerow(row_data)

# 关闭 WebDriver
driver.quit()

可视化代码

import pandas as pd
import matplotlib.pyplot as plt
import os
# 设置全局字体
plt.rcParams['font.sans-serif'] = ['SimHei']  # 使用微软雅黑字体，可以显示中文
plt.rcParams['axes.unicode_minus'] = False  # 解决负号显示问题

folder_path = os.getcwd()+"/data/房价/"
if not os.path.exists(folder_path):
    os.makedirs(folder_path)
# 读取 CSV 文件
df = pd.read_csv(folder_path+'房价排行榜.csv')

# 将城市分成9组
num_groups = 9
group_size = len(df) // num_groups

# 创建9个小图
fig, axs = plt.subplots(3, 3, figsize=(15, 15))

for i in range(num_groups):
    # 计算当前组的开始索引和结束索引
    start_index = i * group_size
    end_index = (i + 1) * group_size
    
    # 获取当前组的城市和二手房价数据
    cities = df['城市'][start_index:end_index]
    prices = df['新房(元/㎡)'][start_index:end_index]
    
    # 计算当前小图的行索引和列索引
    row_index = i // 3
    col_index = i % 3
    
    # 在当前小图中绘制当前组的数据
    axs[row_index, col_index].bar(cities, prices, color='darkred')
    axs[row_index, col_index].set_title(f'第{i+1}组城市新房价排行榜')
    axs[row_index, col_index].set_xlabel('城市')
    axs[row_index, col_index].set_ylabel('新房(元/㎡)')
    axs[row_index, col_index].tick_params(axis='x', rotation=90)  # 旋转x轴标签，以便更好地显示城市名

# 调整布局，防止标签重叠
plt.tight_layout()
plt.show()

二手房

import pandas as pd
import matplotlib.pyplot as plt
import os
# 设置全局字体
plt.rcParams['font.sans-serif'] = ['SimHei']  # 使用微软雅黑字体，可以显示中文
plt.rcParams['axes.unicode_minus'] = False  # 解决负号显示问题

folder_path = os.getcwd()+"/data/房价/"
if not os.path.exists(folder_path):
    os.makedirs(folder_path)
# 读取 CSV 文件
df = pd.read_csv(folder_path+'房价排行榜.csv')

# 将城市分成9组
num_groups = 9
group_size = len(df) // num_groups

# 创建9个小图
fig, axs = plt.subplots(3, 3, figsize=(15, 15))

for i in range(num_groups):
    # 计算当前组的开始索引和结束索引
    start_index = i * group_size
    end_index = (i + 1) * group_size
    
    # 获取当前组的城市和二手房价数据
    cities = df['城市'][start_index:end_index]
    prices = df['二手房(元/㎡)'][start_index:end_index]
    
    # 计算当前小图的行索引和列索引
    row_index = i // 3
    col_index = i % 3
    
    # 在当前小图中绘制当前组的数据
    axs[row_index, col_index].bar(cities, prices, color='lightblue')
    axs[row_index, col_index].set_title(f'第{i+1}组城市二手房价排行榜')
    axs[row_index, col_index].set_xlabel('城市')
    axs[row_index, col_index].set_ylabel('二手房价(元/㎡)')
    axs[row_index, col_index].tick_params(axis='x', rotation=90)  # 旋转x轴标签，以便更好地显示城市名

# 调整布局，防止标签重叠
plt.tight_layout()
plt.show()