抓取指定网站上的所有图片的Python脚本

精选原创

James6688 2024-10-09 09:47:43 博主文章分类：实用程序 ©著作权

文章标签 html 网页内容 Python 文章分类 Python 后端开发

©著作权归作者所有：来自51CTO博客作者James6688的原创作品，请联系作者获取转载授权，否则将追究法律责任

简介

网络爬虫是一种用于自动化访问和提取网页内容的技术。通过网络爬虫，我们可以自动化地从一个或多个网站上抓取信息，例如文本、图像、视频等。在这篇博文中，我们将重点讨论如何使用Python抓取指定网站上的所有图片。

所需工具与库

在开始编写代码之前，我们需要安装一些必要的Python库。这些库包括：

requests: 用于发送HTTP请求
beautifulsoup4: 用于解析HTML内容
os: 用于创建文件夹和处理文件
urllib: 用于处理URL

我们将使用Python的包管理器pip来安装这些库。

准备工作

安装依赖库

首先，我们需要安装所需的库。打开命令行或终端，输入以下命令：

pip install requests beautifulsoup4

这些命令将安装我们在编写爬虫时所需的库。

目标网站选择

在选择目标网站时，请确保你有权抓取该网站的内容，并且你遵守网站的使用条款。本文将以一个公开的图片展示网站为例来介绍抓取过程。

代码实现

在这一部分，我们将详细介绍如何编写Python代码来抓取指定网站上的所有图片。

基本结构

首先，我们将创建一个基本的Python脚本结构。脚本名称可以是image_scraper.py，其基本结构如下：

import os
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse

def fetch_page(url):
    response = requests.get(url)
    response.raise_for_status()  # 确保请求成功
    return response.text

def parse_images(html, base_url):
    soup = BeautifulSoup(html, 'html.parser')
    img_tags = soup.find_all('img')
    img_urls = [urljoin(base_url, img['src']) for img in img_tags if 'src' in img.attrs]
    return img_urls

def download_image(url, folder):
    if not os.path.exists(folder):
        os.makedirs(folder)
    response = requests.get(url)
    response.raise_for_status()  # 确保请求成功
    filename = os.path.join(folder, os.path.basename(urlparse(url).path))
    with open(filename, 'wb') as f:
        f.write(response.content)

def main(url, folder='images'):
    html = fetch_page(url)
    img_urls = parse_images(html, url)
    for img_url in img_urls:
        download_image(img_url, folder)

if __name__ == '__main__':
    target_url = 'https://example.com'  # 替换为你的目标网站
    main(target_url)

获取网页内容

在函数fetch_page中，我们使用requests库发送HTTP请求并获取网页内容。通过response.raise_for_status()确保请求成功，否则抛出异常。

解析网页并提取图片URL

在函数parse_images中，我们使用BeautifulSoup解析网页内容并提取所有图片的URL。使用urljoin函数将相对URL转换为绝对URL。

下载图片

在函数download_image中，我们使用requests库下载图片并将其保存到指定文件夹中。文件名基于图片URL自动生成。

存储图片

在main函数中，我们首先获取网页内容，然后解析出所有图片的URL，最后逐个下载并保存到指定文件夹。

代码优化与改进

在编写完基本的爬虫脚本后，我们可以进一步优化和改进代码以提高其性能和可维护性。

多线程下载

为了提高下载速度，我们可以使用多线程来并发下载图片。以下是修改后的代码：

import os
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
from concurrent.futures import ThreadPoolExecutor

def fetch_page(url):
    response = requests.get(url)
    response.raise_for_status()
    return response.text

def parse_images(html, base_url):
    soup = BeautifulSoup(html, 'html.parser')
    img_tags = soup.find_all('img')
    img_urls = [urljoin(base_url, img['src']) for img in img_tags if 'src' in img.attrs]
    return img_urls

def download_image(url, folder):
    if not os.path.exists(folder):
        os.makedirs(folder)
    response = requests.get(url)
    response.raise_for_status()
    filename = os.path.join(folder, os.path.basename(urlparse(url).path))
    with open(filename, 'wb') as f:
        f.write(response.content)

def main(url, folder='images', max_workers=5):
    html = fetch_page(url)
    img_urls = parse_images(html, url)
    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        for img_url in img_urls:
            executor.submit(download_image, img_url, folder)

if __name__ == '__main__':
    target_url = 'https://example.com'  # 替换为你的目标网站
    main(target_url)

在上面的代码中，我们使用ThreadPoolExecutor来并发下载图片。

增加日志功能

为了更好地监控爬虫的运行状态，我们可以增加日志功能。以下是修改后的代码：

import os
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
from concurrent.futures import ThreadPoolExecutor
import logging

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

def fetch_page(url):
    logging.info(f'Fetching page: {url}')
    response = requests.get(url)
    response.raise_for_status()
    return response.text

def parse_images(html, base_url):
    logging.info('Parsing images...')
    soup = BeautifulSoup(html, 'html.parser')
    img_tags = soup.find_all('img')
    img_urls = [urljoin(base_url, img['src']) for img in img_tags if 'src' in img.attrs]
    logging.info(f'Found {len(img_urls)} images')
    return img_urls

def download_image(url, folder):
    if not os.path.exists(folder):
        os.makedirs(folder)
    logging.info(f'Downloading image: {url}')
    response = requests.get(url)
    response.raise_for_status()
    filename = os.path.join(folder, os.path.basename(urlparse(url).path))
    with open(filename, 'wb') as f:
        f.write(response.content)
    logging.info(f'Saved image: {filename}')

def main(url, folder='images', max_workers=5):
    html = fetch_page(url)
    img_urls = parse_images(html, url)
    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        for img_url in img_urls:
            executor.submit(download_image, img_url, folder)

if __name__ == '__main__':
    target_url = 'https://example.com'  # 替换为你的目标网站
    main(target_url)

保存图片名称避免冲突

我们可以通过为图片生成唯一名称来避免文件名冲突。以下是修改后的代码：

import os
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
from concurrent.futures import ThreadPoolExecutor
import logging
import hashlib

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

def fetch_page(url):
    logging.info(f'Fetching page: {url}')
    response = requests.get(url)
    response.raise_for_status()
    return response.text

def parse_images(html, base_url):
    logging.info('Parsing images...')
    soup = BeautifulSoup(html, 'html.parser')
    img_tags = soup.find_all('img')
    img_urls = [urljoin(base_url, img['src']) for img in img_tags if 'src' in img.attrs]
    logging.info(f'Found {len(img_urls)} images')
    return img_urls

def download_image(url, folder):
    if not os.path.exists(folder):
        os.makedirs(folder)
    logging.info(f'Downloading image: {url}')
    response = requests.get(url)
    response.raise_for_status()
    img_hash = hashlib.md5(response.content).hexdigest()
    filename = os.path.join(folder, f'{img_hash}.jpg')
    with open(filename, 'wb') as f:
        f.write(response.content)
    logging.info(f'Saved image: {filename}')

def main(url, folder='images', max_workers=5):
    html = fetch_page(url)
    img_urls = parse_images(html, url)
    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        for img_url in img_urls:
            executor.submit(download_image, img_url, folder)

if __name__ == '__main__':
    target_url = 'https://example.com'  # 替换为你的目标网站
    main(target_url)

错误处理与调试

在实际运行中，网络爬虫可能会遇到各种错误，例如网络错误、解析错误等。我们可以增加错误处理和调试信息来提高代码的可靠性。

import os
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
from concurrent.futures import ThreadPoolExecutor
import logging
import hashlib

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

def fetch_page(url):
    logging.info(f'Fetching page: {url}')
    try:
        response = requests.get(url)
        response.raise_for_status()
        return response.text
    except requests.RequestException as e:
        logging.error(f'Error fetching page: {e}')
        return None

def parse_images(html, base_url):
    if html is None:
        return []
    logging.info('Parsing images...')
    try:
        soup = BeautifulSoup(html, 'html.parser')
        img_tags = soup.find_all('img')
        img_urls = [urljoin(base_url, img['src']) for img in img_tags if 'src' in img.attrs]
        logging.info(f'Found {len(img_urls)} images')
        return img_urls
    except Exception as e:
        logging.error(f'Error parsing images: {e}')
        return []

def download_image(url, folder):
    if not os.path.exists(folder):
        os.makedirs(folder)
    logging.info(f'Downloading image: {url}')
    try:
        response = requests.get(url)
        response.raise_for_status()
        img_hash = hashlib.md5(response.content).hexdigest()
        filename = os.path.join(folder, f'{img_hash}.jpg')
        with open(filename, 'wb') as f:
            f.write(response.content)
        logging.info(f'Saved image: {filename}')
    except requests.RequestException as e:
        logging.error(f'Error downloading image: {e}')

def main(url, folder='images', max_workers=5):
    html = fetch_page(url)
    img_urls = parse_images(html, url)
    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        for img_url in img_urls:
            executor.submit(download_image, img_url, folder)

if __name__ == '__main__':
    target_url = 'https://example.com'  # 替换为你的目标网站
    main(target_url)

在上面的代码中，我们为每个步骤增加了错误处理和日志记录，以便在出现问题时能够快速定位和解决。