依頼者からの要望
Request from the client

スクレイピング開始。
Scraping is started.

---

---

<td class="classCode"><a href=".php?RowNo=1&Code1=00&Code2=000111" title="詳細" id="Detail">00170-000615</a></td>

---

---

---

---

---

---

directory
    data
      TAIS_codes.csv # result output data
    py37 # virtual environment
    selenium_spider.py
    requirements.txt

mkdir directory

cd directory

mkdir data

python -m venv py37

pip install selenium

pip install chrome-driver

selenium_spider.py

"""
### Usage ###
python -m venv py37

# activate for mac
source py37/bin/activate
# windows
py37\Scripts\activate.bat

pip install -r requirements.txt

# If you want to install the ChromeDriver directly from pip.
pip install chromedriver-binary

# down load chrome driver
# ダウンロードを選んだ場合は作業用ダイレクトリにChromeDriverを直接置いてください。
# If you choose to download, place the ChromeDriver directly in the working directory.
https://chromedriver.chromium.org/downloads

######################
your working directory
      data
        csv file
      py37 # virtual env
      chromedriver binary file
      selenium_spider.py
      requirements.txt
######################

# run script
python selenium_spider.py

# deactivate venv
# mac
deactivate
# windows
py37\Scripts\deactivate.bat


もしドライバーのバージョンが違うと警告されたら。
If you are warned that the driver version is different.

指定されたバージョンのChromeDriverをダウンロードし直してください。
Please re-download the specified version of ChromeDriver.

This version of ChromeDriver only supports Chrome version 75

https://chromedriver.chromium.org/downloads
"""

from selenium.webdriver import Chrome,ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

import csv

# if you want to use choromedriver_binary
# import chromedriver_binary
# chromedriver_binary.add_chromedriver_to_path()

def set_web_driver():
    options = ChromeOptions()

    driver_path = "chromedriver.exe"

    # 下をコメントアウトすると、ヘッドレスモードで検証できます。
    # You can verify in headless mode by commenting out below.
    # options.add_argument('--headless')
    options.add_argument('--incognito')


    # If you are using chromedriver _ binary
    # driver = Chrome(options=options)
    driver = Chrome(driver_path, options=options)

    base_url = 'http://www.spamhamspamspameggbaconspamspam.php'
    driver.get(base_url)
    return driver

def search_product(driver):
    driver.find_element_by_xpath('//input[@id="spam"]').send_keys(1)
    driver.find_element_by_xpath('//div[@class="spam search"]/input').click()

def pagenation(driver):
    try:
        driver.find_element_by_xpath('//input[@onclick="btnNext_OnClick()"]').click()
        return True
    except:
        print('TAIS codes for all pages were scraped.')
        return False

def get_code(driver):

    TAIS_codes = []

    page = 1

    while True:
        print(f'page: {page}')
        try:
            WebDriverWait(driver,10).until(EC.presence_of_element_located((By.XPATH,'//table[@class="spam table"]')))
        except:
            time.sleep(1)

        table = driver.find_element_by_xpath('//table[@class="spam table"]')

        for a in table.find_elements_by_xpath('//tr/td[@class="eggbacon"]/a'):
            try:
                TAIS_codes.append(a.text)
            except:
                pass

        if pagenation(driver):
            page += 1
        else:
            break
    
    with open('data/tais_codes.csv', 'w', newline='') as f:
        csv_writer = csv.writer(f, delimiter=',')
        csv_writer.writerow(["TAIS"])
        for code in TAIS_codes:
            csv_writer.writerow([code])



if __name__=='__main__':
    driver = set_web_driver()
    search_product(driver)
    get_code(driver)
    driver.quit()

python selenium_spider.py

---

---

---

---

---

---

pip freeze > requirements.txt

case001
  data
    TAIS_codes.csv # result output data
  selenium_spider.py
  requirements.txt

git push -u orgin master

---

以上です。

That's all.

See You Next Page!