인프런 커뮤니티 질문&답변

https://kwonkyo.tistory.com/529

해당 블로그의 방법처럼 joson 형식을 직접 코드에 작성하여 네이버 쇼핑 크롤링을 완성 하였습니다. 대신 이 방법대로 하면 매번 코드에 cookies,headers,params 를 작성해 줘야하는데요.

headers와 params는 매번 똑같아도 괜찮지만 cookies는 매일 달라지는 것 같습니다?

이 경우에는 cookies를 자동으로 업데이트 시켜줘야 할 것 같은데

어떤 방법이 있을까요?

구글링 하여 session 방법도 나름 시도해 보았지만 잘 해결되지 않아 질문 드립니다.

아래는 코드 입니다.

import requests
from bs4 import BeautifulSoup
import json

page = 1
keyword = "과자"

print(f"------------------{page}페이지--------------------")

cookies = {
    'NNB': 'WUXWWRMOOVZWG',
    'nx_ssl': '2',
    'NaverSuggestUse': 'use%26unuse',
    'NFS': '2',
    '_ga': 'GA1.2.1826370085.1669177707',
    '_ga_8P4PY65YZ2': 'GS1.1.1670556699.3.1.1670556781.0.0.0',
    'autocomplete': 'use',
    'AD_SHP_BID': '14',
    'ASID': '0e221f9100000185044256e400000053',
    'SHP_BID': '4',
    'demo': 'm-20',
    'nid_inf': '856411158',
    'NID_AUT': 'HZYH2iEU+pydap3C2ZeK6SYL11+ZQZzCIxAEVW3QF+iQi6lSXeyTWKIk5Q2u+b9S',
    'NID_JKL': 'SVy2JH/PZVBFeP9cfyNUo3IgRz5c4Q/O3D1m7qX2s00=',
    'page_uid': 'hHD3nsp0YihssMEgEXGssssssBd-506797',
    'spage_uid': 'hHD3nsp0YihssMEgEXGssssssBd-506797',
    'NID_SES': 'AAABnE+3Z8P/zCp3/+XE9IwB85rhcdGolR2WTfQ+4pgrPTDVp5OXKvHRGfHLRDEkgIPkWbhnTGodcztE7cv9hlVOxWbV/AzVGwmPzQZaD1fLKkFAp6C6687HIZnNTsT4eV03q6QZxMfBOw8XTTfsuYRwsYwbkJDEuKeXXmfFFdC2cy3sGgI6ZPG3ktjZIyg+rkfuxcf1ctpeNOHDdt8KmCKlPx5dstYGtlbWHjZo3A4nW6uaRMt+2C9GB/GLrR01RhYR/MO3V75Cs+QMLbaFjcH3rqDH3hW1Af72MHP7GhbefTHvhHrFUtKAgYUgL2GTT/1He52YtfMgiHlr9nkw9vOA+IxGJMTrzGfA2QhJ2ygbYbET/srzrBzN/wc1S9f/1l11d7SKh0V/wlAyputCtanuhm3zErFBEVVnVmnb4ZuMt0suYkOzi4npFMt9DDbZF0a94Jg/hCpqQpOJyZExpW27Lus6vVtTVMKtGhHKdIWIqQj/sazKIizGDZO/kCkB2B3UZ7Mifr8++HX/JRctca8Xhy2Raq4k4LeLgdfzBJUy8YPk',
    'sus_val': 'bREWstOPzGuz2c6tT/IjoDSm',
    '_naver_usersession_': 'MBrk5lhOc1WegHzbQgb1fDhq',
    'listOffset': '5',
    'lastProductClick': '%EA%B8%B0%EB%8A%A5%EC%84%B1%20%EC%8B%A0%EB%B0%9C%20%EA%B9%94%EC%B0%BD%20%EC%95%84%EC%B9%98%20%EC%97%90%EC%96%B4%20%EC%BF%A0%EC%85%98%20%EC%9A%B4%EB%8F%99%ED%99%94%20%EC%9D%B8%EC%86%94%5E30260232791',
    'ncpa': '6303604|lbmlel28|f29a4f23c2092dd24530afa9813bf4ed43f67312|s_18330e9a745fc|413a3e4b2505b6496c591b071aef2b79c21ebc52:3779889|lbmn6v3s|16785f2b11b65ae4a89a094ca2bc118dc954d556|s_16dc887ee7e95|02e93fa21fc6031ec88bcb6689e7cff18c15665b:5690384|lbmnnr4o|b4c6a1cf1d35097780f135a25c5799f57c65aede|s_19db0b627c733|01e7ef1e0ef2246ac863510bbb29e0a36a0033e3:165568|lbms7ec0|ead632e1b5f336c51e62da580208075272c11459|s_3422b8e1086835|47f818e032ceb6be564e3ff644b883ec359a265e:955149|lbnl92mg|b70c701a1fa1d38c1f581fcc23b30398d4adcc90|s_284221eac5eff|2f245a3ca6021c0dfb29a16bf7c07ec3c04396a7:225323|lbnmb480|2b37c0d8ddc982beacf11112bde0a117667fa270|s_5febb6e0220e1431|00c43677ce145a64eda592347157206e1dbd56d6:114|lbnr0s8w|54bc64a1453b4478949117b8889af402d3167367|s_1f0b6dd34481|165f45347c0201202f68fb81d77ad2bb5359ff6e:17703|lbns2kl4|26027c84c330407b41db82948a4fcaa33e7310d7|s_1f0b6dd345b2|2c70c3b6f7b0fcc2318cc94f852a990489505190:2680908|lbnsxh8o|462894807013beea343e192f633c808c7a42aabb|s_7d847d844c1e|a273843fa1cab15b4f2d4c666bc1fb930901383e:95694|lbntoqq0|d1c8fb101d9c85a5ff547a1b0785720da86641af|95694|477f8709aa2b405357c709a92412bb560da31300',
}

headers = {
    'authority': 'msearch.shopping.naver.com',
    'accept': 'application/json, text/plain, */*',
    'accept-language': 'ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7',
    'logic': 'PART',
    'referer': f'https://msearch.shopping.naver.com/search/all?frm=NVSHPAG&origQuery=%ED%81%B0%EC%8B%A0%EB%B0%9C%EA%B9%94%EC%B0%BD&pagingIndex={page}&pagingSize=40&productSet=total&query=%ED%81%B0%EC%8B%A0%EB%B0%9C%EA%B9%94%EC%B0%BD&sort=rel&viewType=lst',
    'sec-ch-ua': '"Not?A_Brand";v="8", "Chromium";v="108", "Google Chrome";v="108"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': '"Windows"',
    'sec-fetch-dest': 'empty',
    'sec-fetch-mode': 'cors',
    'sec-fetch-site': 'same-origin',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36',
}

params = {
    'query': f'{keyword}',
    'sort': 'rel',
    'pagingIndex': f'{page}',
    'pagingSize': '40',
    'viewType': 'lst',
    'productSet': 'total',
    'frm': 'NVSHPAG',
    'origQuery': f'{keyword}',
}

response = requests.get('https://msearch.shopping.naver.com/api/search/all', params=params, cookies=cookies, headers=headers)

itemlist = json.loads(response.text)  
cnt_1 = 1
print("=============광고 상품==============")
try:
    for i in itemlist['searchAdResult']['products']:
        print(cnt_1)
        print(i["productTitle"])
        print(i["mallProductId"])
        cnt_1 +=1
except:
    print("============광고 상품 없음===================")
    
cnt = 1
print("=============일반 상품==============")
for i in itemlist['shoppingResult']['products']:
    print(cnt)
    print(i["productTitle"])
    try:
        if i["mallProdMblUrl"][0] =="h":
            print(i["mallProdMblUrl"])
            print(i["mallProductId"])
    except:
        print(i["id"]) 
    cnt +=1