I have a code that scrapes oddsportal website.
Sometimes while scraping, I get ValueError("No tables found") and when I manually refresh browser, page loads.
How do I do it via code?
My code is as below:
import pandas as pd
from selenium import webdriver
from bs4 import BeautifulSoup as bsbrowser = webdriver.Chrome()class GameData:def __init__(self):self.date = []self.time = []self.game = []self.score = []self.home_odds = []self.draw_odds = []self.away_odds = []self.country = []self.league = []def parse_data(url):browser.get(url)df = pd.read_html(browser.page_source, header=0)[0]html = browser.page_sourcesoup = bs(html, "lxml")cont = soup.find('div', {'id': 'wrap'})content = cont.find('div', {'id': 'col-content'})content = content.find('table', {'class': 'table-main'}, {'id': 'tournamentTable'})main = content.find('th', {'class': 'first2 tl'})if main is None:return Nonecount = main.findAll('a')country = count[1].textleague = count[2].textgame_data = GameData()game_date = Nonefor row in df.itertuples():if not isinstance(row[1], str):continueelif ':' not in row[1]:game_date = row[1].split('-')[0]continuegame_data.date.append(game_date)game_data.time.append(row[1])game_data.game.append(row[2])game_data.score.append(row[3])game_data.home_odds.append(row[4])game_data.draw_odds.append(row[5])game_data.away_odds.append(row[6])game_data.country.append(country)game_data.league.append(league)return game_dataurls = {
"https://www.oddsportal.com/soccer/africa/africa-cup-of-nations/results/#/",
"https://www.oddsportal.com/soccer/africa/africa-cup-of-nations/results/#/page/2/",
"https://www.oddsportal.com/soccer/africa/africa-cup-of-nations/results/#/page/3/",
"https://www.oddsportal.com/soccer/africa/africa-cup-of-nations/results/#/page/4/",
"https://www.oddsportal.com/soccer/africa/africa-cup-of-nations/results/#/page/5/",
"https://www.oddsportal.com/soccer/africa/africa-cup-of-nations/results/#/page/6/",
"https://www.oddsportal.com/soccer/africa/africa-cup-of-nations/results/#/page/7/",
"https://www.oddsportal.com/soccer/africa/africa-cup-of-nations/results/#/page/8/",
"https://www.oddsportal.com/soccer/africa/africa-cup-of-nations/results/#/page/9/",}if __name__ == '__main__':results = Nonefor url in urls:try:game_data = parse_data(url)if game_data is None:continueresult = pd.DataFrame(game_data.__dict__)if results is None:results = resultelse:results = results.append(result, ignore_index=True)except ValueError:game_data = parse_data(url)if game_data is None:continueresult = pd.DataFrame(game_data.__dict__)if results is None:results = resultelse:results = results.append(result, ignore_index=True)except AttributeError:game_data = parse_data(url)if game_data is None:continueresult = pd.DataFrame(game_data.__dict__)if results is None:results = resultelse:results = results.append(result, ignore_index=True)
Sometimes I get this browser error.
Traceback (most recent call last):File "C:/Users/harsh/AppData/Roaming/JetBrains/PyCharmCE2021.1/scratches/scratch_29.py", line 10098, in <module>game_data = parse_data(url)File "C:/Users/harsh/AppData/Roaming/JetBrains/PyCharmCE2021.1/scratches/scratch_29.py", line 37, in parse_datadf = pd.read_html(browser.page_source, header=0)[0]File "C:\Python\lib\site-packages\pandas\util\_decorators.py", line 299, in wrapperreturn func(*args, **kwargs)File "C:\Python\lib\site-packages\pandas\io\html.py", line 1100, in read_htmldisplayed_only=displayed_only,File "C:\Python\lib\site-packages\pandas\io\html.py", line 913, in _parseraise retainedFile "C:\Python\lib\site-packages\pandas\io\html.py", line 893, in _parsetables = p.parse_tables()File "C:\Python\lib\site-packages\pandas\io\html.py", line 213, in parse_tablestables = self._parse_tables(self._build_doc(), self.match, self.attrs)File "C:\Python\lib\site-packages\pandas\io\html.py", line 543, in _parse_tablesraise ValueError("No tables found")
ValueError: No tables found
My best guess is that I have not built ValueError: No tables found in the code correctly.
How do I handle this?
import pandas as pd
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWaiturls = ["https://www.oddsportal.com/soccer/africa/africa-cup-of-nations/results"
]def main(driver):for url in urls:driver.get(url)waiter = WebDriverWait(driver, 10)try:waiter.until(EC.presence_of_all_elements_located((By.ID, 'tournamentTable')))df = pd.read_html(driver.page_source, attrs={'id': 'tournamentTable'})[0].iloc[:, :-1]df.dropna(inplace=True)df.reset_index(drop=True, inplace=True)return dfexcept TimeoutException:print('Unable to Find Table')finally:driver.quit()if __name__ == "__main__":driver = webdriver.Firefox()df = main(driver)print(df)
Output:
Soccer» Africa»Africa Cup of Nations15 Jun 2021 - Qualification 15 Jun 2021 - Qualification.1 15 Jun 2021 - Qualification.2 1 X 2
0 16:00 Sierra Leone - Benin 1:0 2.85 2.85 2.63
1 30 Mar 2021 - Qualification 30 Mar 2021 - Qualification 30 Mar 2021 - Qualification 1 X 2
2 19:00 Cameroon - Rwanda 0:0 1.51 3.59 7.78
3 19:00 Morocco - Burundi 1:0 1.15 6.63 22.58
4 19:00 Mozambique - Cape Verde 0:1 2.66 2.96 2.81
5 16:00 Guinea Bissau - Congo 3:0 2.64 2.81 3.01
6 16:00 Nigeria - Lesotho 3:0 1.13 7.78 20.49
7 16:00 Senegal - Eswatini 1:1 1.05 12.78 30.79
8 13:00 Central Africa - Mauritania 0:1 1.96 3.12 4.11
9 13:00 Ivory Coast - Ethiopia 3:1 1.61 2.85 9.43
10 13:00 Madagascar - Niger 0:0 1.20 6.36 13.98
11 29 Mar 2021 - Qualification 29 Mar 2021 - Qualification 29 Mar 2021 - Qualification 1 X 2
12 19:00 Algeria - Botswana 5:0 1.11 7.75 26.29
13 19:00 Zimbabwe - Zambia 0:2 2.92 3.02 2.49
14 16:00 Angola - Gabon 2:0 3.08 2.84 2.55
15 16:00 Burkina Faso - South Sudan 1:0 1.16 6.56 18.56
16 16:00 D.R. Congo - Gambia 1:0 1.89 3.25 4.30
17 16:00 Egypt - Comoros 4:0 1.28 4.70 12.98
18 16:00 Togo - Kenya 1:2 2.11 2.70 4.44
19 13:00 Malawi - Uganda 1:0 3.28 2.58 2.62
20 28 Mar 2021 - Qualification 28 Mar 2021 - Qualification 28 Mar 2021 - Qualification 1 X 2
21 16:00 Ghana - Sao Tome and Principe 3:1 1.05 12.65 28.43
22 16:00 Sudan - South Africa 2:0 3.66 3.09 2.10
23 13:00 Namibia - Guinea 2:1 3.32 2.85 2.38
24 13:00 Tanzania - Libya 1:0 2.50 2.80 3.20
25 13:00 Tunisia - Equatorial Guinea 2:1 1.27 5.07 11.87
26 27 Mar 2021 - Qualification 27 Mar 2021 - Qualification 27 Mar 2021 - Qualification 1 X 2
27 16:00 Benin - Nigeria 0:1 4.45 2.37 2.30
28 13:00 Lesotho - Sierra Leone 0:0 2.71 2.50 3.25
29 26 Mar 2021 - Qualification 26 Mar 2021 - Qualification 26 Mar 2021 - Qualification 1 X 2
30 19:00 Mauritania - Morocco 0:0 9.09 3.96 1.40
31 16:00 Cape Verde - Cameroon 3:1 2.92 2.69 2.79
32 16:00 Congo - Senegal 0:0 7.29 3.50 1.54
33 16:00 Niger - Ivory Coast 0:3 11.28 4.38 1.33
34 13:00 Burundi - Central Africa 2:2 1.86 3.16 4.42
35 13:00 Eswatini - Guinea Bissau 1:3 6.42 3.35 1.60
36 25 Mar 2021 - Qualification 25 Mar 2021 - Qualification 25 Mar 2021 - Qualification 1 X 2
37 19:00 Equatorial Guinea - Tanzania 1:0 2.13 2.76 4.10
38 19:00 Libya - Tunisia 2:5 3.98 3.03 2.03
39 19:00 Zambia - Algeria 3:3 3.95 3.06 2.02
40 16:00 Botswana - Zimbabwe 0:1 2.44 2.58 3.64
41 16:00 Gabon - D.R. Congo 3:0 2.33 2.75 3.59
42 16:00 Gambia - Angola 1:0 1.97 2.86 4.58
43 16:00 Kenya - Egypt 1:1 6.24 3.39 1.61
44 16:00 South Africa - Ghana 1:1 2.47 2.77 3.31
45 13:00 Comoros - Togo 0:0 1.56 3.63 6.13
46 24 Mar 2021 - Qualification 24 Mar 2021 - Qualification 24 Mar 2021 - Qualification 1 X 2
47 16:00 Guinea - Mali 1:0 1.66 3.19 6.25
48 13:00 Chad - Namibia award. - - -
49 13:00 Ethiopia - Madagascar 4:0 2.16 2.99 3.59
50 13:00 Rwanda - Mozambique 1:0 2.30 2.78 3.52
51 13:00 Sao Tome and Principe - Sudan 0:2 7.31 3.44 1.54
52 13:00 South Sudan - Malawi 0:1 2.78 2.58 3.06
53 13:00 Uganda - Burkina Faso 0:0 2.75 2.46 3.31
54 17 Nov 2020 - Qualification 17 Nov 2020 - Qualification 17 Nov 2020 - Qualification 1 X 2
55 19:00 Namibia - Mali 1:2 5.22 3.30 1.72
56 19:00 Tanzania - Tunisia 1:1 5.04 3.42 1.71
57 16:00 Angola - D.R. Congo 0:1 3.19 2.84 2.41