A lot of people ask me to write a script to scrape google “people also ask”. This script fetches the data and saves it in a CSV file.
#Project: scraping "people also ask" question/answer and save in csv format
#Code by Sunil
#14-July
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
import csv
import time
# fetch all java related questions and answers
search = "Java"
options = webdriver.ChromeOptions()
options.add_experimental_option('excludeSwitches', ['enable-logging'])
driver = webdriver.Chrome(options=options)
driver.get('https://www.google.com')
driver.implicitly_wait(10)
driver.find_element(By.NAME,"q").send_keys(search + Keys.ENTER)
limit = 20 # no of questions to be fetched
counter = 1
qno=1
header = ["question","answer"]
data = []
while counter <= limit:
qq=aa=""
try:
question = driver.find_element(By.CSS_SELECTOR, 'div[jsname="N760b"] > div:nth-child(' + str(counter) + ')')
qq=question.text
if search not in qq: # this is just to ignore UN RELATED questions
counter+=1 # the keyword must be in questions
continue
except NoSuchElementException:
counter +=1
continue
print (qno, question.text)
#driver.implicitly_wait(60) # wait after each click
# wait for 1 seconds
time.sleep(1)
question.click()
try:
answer = question.find_element(By.CSS_SELECTOR, 'div:nth-child(2) > div > div > div:nth-child(2) > div:nth-child(1) > div > div:nth-child(1) > div > div > span:nth-child(1) > span')
except NoSuchElementException:
answer = ""
if answer == '':
print ('different format')
aa=""
else:
print (answer.text)
aa = answer.text
print ('-------------------------------------')
counter += 1
qno +=1
row = [qq, aa]
data.append(row)
with open('q.csv', 'w', newline='', encoding='utf-8') as file:
writer = csv.writer(file)
writer.writerow(header)
writer.writerows(data)
