1. CSSセレクターの基本
BeautifulSoupを用いてHTMLドキュメントから要素を選択することができます。
from bs4 import BeautifulSoup
html_doc = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p id="my_p" class="title">asdfasdf<b id="bbb" class="boldest">The Dormouse's story</b>
</p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
"""
soup = BeautifulSoup(html_doc, 'lxml')
# res = soup.select('a')
# res = soup.select('#link1') # ID属性
# res = soup.select('.sister') # クラス属性
# res = soup.select('body > p > a') # 子要素のみ
# res = soup.select('body p a') # 子孫要素まで
# 特定属性を指定
res = soup.select('a[href="http://example.com/tillie"]')
print(res)
2. Seleniumの基本的な使い方
from selenium import webdriver
import time
# Chromeドライバーを初期化
driver = webdriver.Chrome()
# ページを読み込む
driver.get('http://www.baidu.com')
time.sleep(3)
# タブを閉じる
driver.close()
# ブラウザを完全に閉じる
driver.quit()
3. 無界面ブラウザの設定
from selenium import webdriver
import time
from selenium.webdriver.chrome.options import Options
# 無界面モードを有効にする
chrome_options = Options()
chrome_options.add_argument('--headless')
driver = webdriver.Chrome(options=chrome_options)
# サイトにアクセス
driver.get('https://www.jd.com/')
print(driver.page_source)
time.sleep(3)
driver.close()
4. Seleniumの高度な機能
4.1 ブラウザ制御の例（百度ログイン）
from selenium import webdriver
from selenium.webdriver.common.by import By
import time
driver = webdriver.Chrome()
driver.get('http://www.baidu.com')
driver.implicitly_wait(10)
# ログインリンクをクリック
login_link = driver.find_element(By.LINK_TEXT, '登录')
login_link.click()
# ユーザー名を入力
username = driver.find_element(By.ID, 'TANGRAM__PSP_11__userName')
username.send_keys('33333@qq.com')
# パスワードを入力
password = driver.find_element(By.ID, 'TANGRAM__PSP_11__password')
password.send_keys('lqz12345')
# ログインボタンをクリック
login_button = driver.find_element(By.ID, 'TANGRAM__PSP_11__submit')
login_button.click()
time.sleep(5)
driver.close()
4.2 要素操作とJavaScriptの実行
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
import time
driver = webdriver.Chrome()
driver.get('https://www.jd.com/')
# JavaScriptを用いてページをスクロール
driver.execute_script('window.scrollTo(0, document.body.scrollHeight);')
time.sleep(3)
driver.close()
5. XPathの利用
from lxml import etree
doc = '''
<html>
<head>
<base href='http://example.com/' />
<title>Example website</title>
</head>
<body>
<div id='images'>
<a href='image1.html' id='id_a'>Name: My image 1 <br/><img src='image1_thumb.jpg' /></a>
<a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' /></a>
<a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' /></a>
<a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' /></a>
<a href='image5.html' class='li li-item' name='items'>Name: My image 5 <br /><img src='image5_thumb.jpg' /></a>
<a href='image6.html' name='items'><span><h5>test</h5></span>Name: My image 6 <br /><img src='image6_thumb.jpg' /></a>
</div>
</body>
</html>
'''
html = etree.HTML(doc)
# XPathで要素を選択
result = html.xpath('//a[2]/@href')
print(result)
6. オートスクレイピングの実践例（京东商品情報収集）
from selenium import webdriver
from selenium.webdriver.common.by import By
import time
def collect_products(driver):
try:
products = driver.find_elements(By.CLASS_NAME, 'gl-item')
for product in products:
name = product.find_element(By.CSS_SELECTOR, '.p-name em').text
price = product.find_element(By.CSS_SELECTOR, '.p-price i').text
reviews = product.find_element(By.CSS_SELECTOR, '.p-commit a').text
link = product.find_element(By.CSS_SELECTOR, '.p-name a').get_attribute('href')
image = product.find_element(By.CSS_SELECTOR, '.p-img img').get_attribute('src')
if not image:
image = 'https://' + product.find_element(By.CSS_SELECTOR, '.p-img img').get_attribute('data-lazy-img')
print(f"""
商品名：{name}
価格：{price}
商品リンク：{link}
画像URL：{image}
購買レビュー：{reviews}
""")
next_button = driver.find_element(By.PARTIAL_LINK_TEXT, '下一页')
next_button.click()
time.sleep(1)
collect_products(driver)
except Exception as e:
print(e)
def main(url, keyword):
driver = webdriver.Chrome()
driver.get(url)
driver.implicitly_wait(10)
try:
search_box = driver.find_element(By.ID, 'key')
search_box.send_keys(keyword)
search_box.send_keys(Keys.ENTER)
collect_products(driver)
finally:
driver.close()
if __name__ == '__main__':
main('https://www.jd.com/', keyword='华为手機')
7月26日 22:43 投稿