네이버부동산크롤링에 selenium을 활용

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
import time

# 드라이버 설정
options = webdriver.ChromeOptions()
options.add_argument("--start-maximized")  # 시작 시 최대화
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)

# 네이버 부동산 페이지 접속
driver.get("https://land.naver.com/")

# 매매 버튼 클릭
driver.find_element(By.XPATH, '/html/body/div[4]/div[3]/div[2]/div/div[1]/div[1]/ul/div[2]/li[2]').click()

# 리스트 버튼 클릭
driver.find_element(By.XPATH, '/html/body/div[4]/div[3]/div[2]/div/div[1]/div[1]/p').click()

# 서울시 선택
driver.find_element(By.XPATH, '/html/body/div[4]/div[3]/div[2]/div/div[1]/div[3]/div[3]/div[1]/ul/li[1]').click()

# 서초구 선택
driver.find_element(By.XPATH, '/html/body/div[4]/div[3]/div[2]/div/div[1]/div[3]/div[3]/div[2]/ul/li[15]').click()

# 반포동 선택 (명시적 대기 사용)
wait = WebDriverWait(driver, 10)
element = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[4]/div[3]/div[2]/div/div[1]/div[3]/div[3]/div[3]/div/ul/li[2]')))
element.click()

# 확인매물 버튼 클릭
driver.find_element(By.XPATH, '/html/body/div[4]/div[3]/div[2]/div/div[1]/div[3]/div[3]/div[4]/p[2]/a').click()

# 전체 화면 모드로 전환
driver.fullscreen_window()

# 단지 정보 탭으로 전환
driver.find_element(By.XPATH, '/html/body/div[2]/div/section/div[2]/div/div[1]/div/div/a/span[4]').click()

# 단지명을 저장할 리스트
complex_names = []

# 반복문 내에서 각 단지명을 추출하는 부분 수정
for i in range(1, 111):
    xpath = f'/html/body/div[2]/div/section/div[2]/div/div[1]/div/div/div/div[3]/ul/li[{i}]'

    # 요소가 존재하고 가시적일 때까지 대기
    complex_name_element = WebDriverWait(driver, 10).until(
        EC.visibility_of_element_located((By.XPATH, xpath))
    )

    complex_name = complex_name_element.text
    complex_names.append(complex_name)
    print(f"단지 {i}: {complex_name}")

# 드라이버 종료
driver.quit()

# 추출된 단지명 출력
for name in complex_names:
    print(name)

import requests
import json
from datetime import datetime

DISTRICT_CODES = {
    "강남구": "1168000000", "강동구": "1174000000", "강북구": "1130500000",
    "강서구": "1150000000", "관악구": "1162000000", "광진구": "1121500000",
    "구로구": "1153000000", "금천구": "1154500000", "노원구": "1135000000",
    "도봉구": "1132000000", "동대문구": "1123000000", "동작구": "1159000000",
    "마포구": "1144000000", "서대문구": "1141000000", "서초구": "1165000000",
    "성동구": "1120000000", "성북구": "1129000000", "송파구": "1171000000",
    "양천구": "1147000000", "영등포구": "1156000000", "용산구": "1117000000",
    "은평구": "1138000000", "종로구": "1111000000", "중구": "1114000000",
    "중랑구": "1126000000"
}

SORT_OPTIONS = {
    "1": "rank",  # 랭킹순
    "2": "dates",  # 최신순
    "3": "lowPrc",  # 가격순
    "4": "highSpc"  # 면적순
}


def fetch_naver_land_data(district, sort_option, page):
    url = "https://m.land.naver.com/cluster/ajax/articleList"

    cortar_no = DISTRICT_CODES.get(district)
    if not cortar_no:
        print(f"Error: '{district}' is not a valid district name.")
        return None

    params = {
        "rletTpCd": "APT",
        "tradTpCd": "A1:B1:B2:B3",
        "cortarNo": cortar_no,
        "page": str(page),
        "sort": sort_option
    }

    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
    }

    try:
        response = requests.get(url, params=params, headers=headers)
        response.raise_for_status()
        data = response.json()
        return data
    except requests.RequestException as e:
        print(f"Error fetching data: {e}")
        return None


def get_sort_option():
    print("\n정렬 옵션:")
    for key, value in SORT_OPTIONS.items():
        print(f"{key}. {value}")

    while True:
        choice = input("정렬 옵션을 선택하세요 (1-4): ")
        if choice in SORT_OPTIONS:
            return SORT_OPTIONS[choice]
        print("잘못된 선택입니다. 다시 시도해주세요.")


def get_page_count():
    while True:
        try:
            count = int(input("출력할 페이지 수를 입력하세요 (0 입력 시 모든 데이터 추출): "))
            if count >= 0:
                return count
            print("0 이상의 숫자를 입력해주세요.")
        except ValueError:
            print("올바른 숫자를 입력해주세요.")


def save_to_html(data, filename):
    if not data:
        print("저장할 데이터가 없습니다.")
        return

    # 포함할 키를 정의
    include_keys = ['cortarNo', 'rletTpNm', 'hanPrc', 'spc1', 'spc2', 'direction', 'atclFetrDesc', 'tagList']

    # HTML 테이블 생성
    html = "<html><head><meta charset='UTF-8'><title>부동산 데이터</title></head><body>"
    html += "<table border='1'><thead><tr>"
    for key in include_keys:
        html += f"<th>{key}</th>"
    html += "</tr></thead><tbody>"

    for row in data:
        html += "<tr>"
        for key in include_keys:
            html += f"<td>{row.get(key, '')}</td>"
        html += "</tr>"

    html += "</tbody></table></body></html>"

    with open(filename, 'w', encoding='utf-8') as output_file:
        output_file.write(html)

    print(f"데이터가 {filename}에 저장되었습니다.")


def main():
    print("서울시 구 목록:")
    for district in DISTRICT_CODES.keys():
        print(district)

    district = input("\n데이터를 조회할 구 이름을 입력하세요: ")
    if district not in DISTRICT_CODES:
        print(f"Error: '{district}'는 유효한 구 이름이 아닙니다.")
        return

    sort_option = get_sort_option()
    page_count = get_page_count()

    all_data = []
    page = 1
    total_items = 0

    while page_count == 0 or page <= page_count:
        data = fetch_naver_land_data(district, sort_option, page)
        if not data or 'body' not in data or not data['body']:
            break

        items = data['body']
        if not items:
            break

        all_data.extend(items)
        total_items += len(items)
        print(f"페이지 {page} 데이터 추출 완료 (누적 {total_items}개)")

        if len(items) < 20:  # 마지막 페이지
            break

        page += 1

    if all_data:
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        filename = f"{district}_{sort_option}_{total_items}items_{timestamp}.html"
        save_to_html(all_data, filename)
    else:
        print("추출된 데이터가 없습니다.")


if __name__ == "__main__":
    main()