使用Selenium与CSS选择器：动态网页数据提取实战指南

from selenium.webdriver.common.by import By  # 定位ID为'product-search-results'的元素 element = driver.find_element(By.CSS_SELECTOR, '#product-search-results')

# 定位所有class为'product-grid-item'的产品卡片 product_cards = driver.find_elements(By.CSS_SELECTOR, '.product-grid-item')

Cutout.Pro抠图

AI批量抠图去背景

13

查看详情

pip install selenium

from selenium import webdriver from selenium.webdriver.chrome.service import Service from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.common.exceptions import TimeoutException, NoSuchElementException import time import csv  # 配置ChromeDriver路径 (根据实际情况修改) # service = Service(executable_path='/path/to/chromedriver') # driver = webdriver.Chrome(service=service)  # 推荐使用webdriver_manager自动管理ChromeDriver from webdriver_manager.chrome import ChromeDriverManager driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))  # 可选：配置为无头模式，不显示浏览器界面，提高效率 # options = webdriver.ChromeOptions() # options.add_argument('--headless') # options.add_argument('--disable-gpu') # 某些linux系统需要 # driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)  driver.maximize_window() # 最大化窗口，确保元素可见

target_url = "https://www.patagonia.com/shop/womens-jackets" # 示例URL，请替换为实际目标 driver.get(target_url)

def load_all_products(driver):     """     尝试点击“加载更多”按钮，直到所有产品加载完毕。     """     while True:         try:             # 等待“加载更多”按钮出现并可点击             # 这里的CSS选择器需要根据实际网站结构调整             load_more_button_selector = '#product-search-results > div.row.product-grid.load-more-present > div.col-12.grid-footer > div > div > button'             load_more_button = WebDriverWait(driver, 10).until(                 EC.element_to_be_clickable((By.CSS_SELECTOR, load_more_button_selector))             )              # 滚动到按钮位置，确保其在视图内             driver.execute_script("arguments[0].scrollIntoView({ behavior: 'auto', block: 'center' });", load_more_button)             time.sleep(0.5) # 短暂等待，模拟用户操作              # 点击按钮             load_more_button.click()             time.sleep(2) # 等待新内容加载          except TimeoutException:             print("未找到或无法点击“加载更多”按钮，或所有内容已加载。")             break # 按钮不再出现或不可点击，说明已加载完毕         except NoSuchElementException:             print("“加载更多”按钮的CSS选择器不正确或元素不存在。")             break         except Exception as e:             print(f"加载更多时发生未知错误: {e}")             break  print("开始加载所有产品...") load_all_products(driver) print("所有产品加载完成。")

products_data = []  # 定位所有产品容器，这里的CSS选择器需要根据实际网站结构调整 # 示例：每个产品项可能是一个带有特定类名的div product_items_selector = '#product-search-results > div.row.product-grid.load-more-present > div.col-6.col-sm-4.col-md-3.product-grid-item' product_elements = driver.find_elements(By.CSS_SELECTOR, product_items_selector)  print(f"找到 {len(product_elements)} 个产品。")  for i, product_element in enumerate(product_elements):     product_info = {         'product_title': '',         'product_url': '',         'product_image_url': '',         'product_price': '',         'rating': '',         'review_count': ''     }     try:         # 提取产品标题和URL         # 示例：标题通常在h2或h3标签内的a链接中         title_link = product_element.find_element(By.CSS_SELECTOR, 'h2.product-name > a')         product_info['product_title'] = title_link.text.strip()         product_info['product_url'] = title_link.get_attribute('href')          # 提取产品图片URL         # 示例：图片通常在img标签的src属性中         image_element = product_element.find_element(By.CSS_SELECTOR, 'img.product-image')         product_info['product_image_url'] = image_element.get_attribute('src')          # 提取产品价格         # 示例：价格可能在一个带有特定类名的span或div中         price_element = product_element.find_element(By.CSS_SELECTOR, '.price-sales')         product_info['product_price'] = price_element.text.strip()          # 提取评分和评论数量（如果可用）         # 这些元素可能不存在，需要用try-except处理         try:             rating_element = product_element.find_element(By.CSS_SELECTOR, '.rating-stars span.sr-only')             product_info['rating'] = rating_element.get_attribute('innerhtml').strip() # 假设评分在sr-only的innerHTML中         except NoSuchElementException:             product_info['rating'] = 'N/A'          try:             review_count_element = product_element.find_element(By.CSS_SELECTOR, '.review-count')             product_info['review_count'] = review_count_element.text.strip().replace('(', '').replace(')', '')         except NoSuchElementException:             product_info['review_count'] = 'N/A'          products_data.append(product_info)      except NoSuchElementException as e:         print(f"第 {i+1} 个产品元素缺少部分信息: {e}")         # 可以选择跳过此产品或记录部分信息     except Exception as e:         print(f"处理第 {i+1} 个产品时发生未知错误: {e}")  # 打印部分结果 for product in products_data[:5]:     print(product)

# 定义CSV文件头 csv_headers = ['product_title', 'product_url', 'product_image_url', 'product_price', 'rating', 'review_count'] csv_file_name = 'patagonia_womens_jackets.csv'  with open(csv_file_name, 'w', newline='', encoding='utf-8') as csvfile:     writer = csv.DictWriter(csvfile, fieldnames=csv_headers)     writer.writeheader()     writer.writerows(products_data)  print(f"数据已成功保存到 {csv_file_name}")  # 关闭浏览器 driver.quit()