Python爬取表情包

转载

mob604756e78484 2021-09-24 19:24:00

今天又爬取了一些表情包，但是这次的处理方式有些许的不同，就是在查找到网页中所有的img标签后，处理图片url和图片名字的时候有些许不同。前面是通过处理字符串的方式来处理，效率上会慢一些，今天用bs4处理标签的方式来处理。代码如下：

"""
    爬点表情包，目标网址https://fxxx.com/
"""
import os
import requests
from bs4 import BeautifulSoup

# 首先定义请求网页
def get_html(url):
    # 设置headers
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36'
    }
    # 请求网页
    request = requests.get(url, headers=headers)
    # 解码
    request.raise_for_status()
    request.encoding = request.apparent_encoding
    request.encoding = 'utf-8'

    # 获取内容
    content = request.text

    # 返回内容
    return content

# 获取单页的image_list
def get_image_lists(html_content):

    # 解析网页
    soup = BeautifulSoup(html_content, 'lxml')
    # 找到我们需要的信息
    image_lists = soup.findAll(name='img', attrs={"class":"ui image lazy" }) # 找到网页中所有的图片

    return image_lists

# 获取图片的地址
def get_image_urls(image_lists):
    # 开始循环遍历
    image_urls = []
    for i in image_lists:
        image_url = i['data-original'] # 获取img标签下属性为data-original的内容
        image_urls.append(image_url)
    return image_urls

# 获取图片名字
def get_image_names(image_lists):
    # 开始循环
    image_names = []
    for i in image_lists:
        image_name = i['title'] # 获取img标签下属性为title的内容
        image_names.append(image_name)

    # 因为图片名称中可能出现windows不支持的字符如? /等，我们全部替换掉
    str_image_names = [str(image_name).replace('?','').replace('/','') for image_name in image_names] #表达式获取一个新的名字列表
    # 上面我们处理了特殊字符，接下来处理长度，可能会存在长度超出限制的情况
    new_image_names = [str(image_name)[0:20] for image_name in str_image_names] # 如果文件名超出20，我们就截取前20个字符作为标题
    # 当然可能还有其他情况需要处理
    return new_image_names

# 开始处理
if __name__ == "__main__":
    # 读取20页
    for page in range(1, 5):
        url = f'https://xxx.com/biaoqing/lists/page/{page}.html'
        html_content = get_html(url) # 获取网页内容
        image_lists = get_image_lists(html_content) # 解析网页获取图片信息列表
        image_urls = get_image_urls(image_lists) # 获取网址列表
        image_names = get_image_names(image_lists) # 获取标题列表

        all_data = zip(image_urls, image_names) # 将其封装好处理
        # print(list(all_data))
        filePath = './images' # 当前路径下的一个文件夹

        for image_url, image_name in all_data:
            if os.path.exists(filePath):
                rimage = requests.get(image_url)
                rimage.raise_for_status()
                with open(os.path.join(filePath, image_name + str(image_url).split('.')[-1]), 'wb') as fileObject:
                    fileObject.write(rimage.content)
                    fileObject.close()
                    print("正在保存：" + image_name)
            else:
                os.mkdir(filePath)
                rimage = requests.get(image_url)
                rimage.raise_for_status()
                with open(os.path.join(filePath, image_name + str(image_url).split('.')[-1]), 'wb') as fileObject:
                    fileObject.write(rimage.content)
                    fileObject.close()
                    print("正在保存：" + image_name)