I am trying to scrape information from this website example website

I need to get the version 2021 and search by code. Here is my code:

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException
from selenium.webdriver.support.ui import WebDriverWait, Select
from selenium.webdriver.support import expected_conditionsignored_exceptions=(NoSuchElementException,StaleElementReferenceException)options = Options()
options.add_argument('--disable-extensions')
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--remote-debugging-port=9515')
options.add_argument('--disable-setuid-sandbox')
options.add_argument("--start-maximized")driver = webdriver.Chrome(service=Service("/usr/bin/chromedriver"), options=options)
url = "https://noc.esdc.gc.ca/"
driver.get(url)search_by_code = WebDriverWait(driver, 10, ignored_exceptions=ignored_exceptions)\.until(expected_conditions.presence_of_element_located((By.XPATH, "/html/body/main/div[2]/div/div/div/div/div/div/div/div/ul/li[2]/a")))
# click to activate this option
search_by_code.click()text_area = WebDriverWait(driver, 10, ignored_exceptions=ignored_exceptions)\.until(expected_conditions.presence_of_element_located((By.XPATH, "/html/body/main/div[2]/div/div/div/div/div/div/div/div/div/details[2]/div/div/form/div/div[2]/div/input")))version = Select(WebDriverWait(driver, 10, ignored_exceptions=ignored_exceptions)\.until(expected_conditions.presence_of_element_located((By.XPATH, "/html/body/main/div[2]/div/div/div/div/div/div/div/div/div/details[2]/div/div/form/div/div[1]/select"))))search_button = driver.find_element(By.XPATH, '/html/body/main/div[2]/div/div/div/div/div/div/div/div/div/details[2]/div/div/form/div/div[2]/div/div/button')# select version 2021
version.select_by_value('2021.0')# click on text area
text_area.click()# type the text 
text_area.send_keys("10010  –  Financial managers")# click the button
search_button.click()print(source = driver.current_url)

I am not sure what I have missed? I added some comments to describe the logic.

You can use the python-requests library for this website. I have done a little script below to send requests and get the data. You can send your query to SearchCriteria.CodeSearch value. You can use BS4, LXML, or Scrapy Selector class and target the values by XPATH or a CSS selector.

import re
import requestscookies = {'ASP.NET_SessionId': 'u4qffsgfreddkgvdphaxo3pc','BIGipServernoc_esdc_gc_ca_http_443.app~noc_esdc_gc_ca_http_443_pool': '639114412.20480.0000','gpv_pthl': 'blank%20theme','gpv_pc': 'Employment%20and%20Social%20Development%20Canada','gpv_pqs': 'blank%20query%20string','gpv_pu': 'noc.esdc.gc.ca%2FSearch%2FQuickSearchJobTitleResults','gpv_pt': 'Search%20by%20job%20title%20-%20Results%20-%20Canada.ca','s_plt': '9.44','s_tp': '1206','gpv_url': 'noc.esdc.gc.ca%2FSearch%2FQuickSearchJobTitleResults','s_ips': '741',
}headers = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9','Accept-Language': 'en-PK,en;q=0.9,ur-PK;q=0.8,ur;q=0.7,en-GB;q=0.6,en-US;q=0.5,sv;q=0.4,it;q=0.3','Cache-Control': 'no-cache','Connection': 'keep-alive',# Requests sorts cookies= alphabetically# 'Cookie': 'ASP.NET_SessionId=u4qffsgfreddkgvdphaxo3pc; BIGipServernoc_esdc_gc_ca_http_443.app~noc_esdc_gc_ca_http_443_pool=639114412.20480.0000; gpv_pthl=blank%20theme; gpv_pc=Employment%20and%20Social%20Development%20Canada; gpv_pqs=blank%20query%20string; gpv_pu=noc.esdc.gc.ca%2FSearch%2FQuickSearchJobTitleResults; gpv_pt=Search%20by%20job%20title%20-%20Results%20-%20Canada.ca; s_plt=9.44; s_tp=1206; gpv_url=noc.esdc.gc.ca%2FSearch%2FQuickSearchJobTitleResults; s_ips=741','Origin': 'https://noc.esdc.gc.ca','Pragma': 'no-cache','Referer': 'https://noc.esdc.gc.ca/Search/QuickSearchJobTitleResults','Sec-Fetch-Dest': 'document','Sec-Fetch-Mode': 'navigate','Sec-Fetch-Site': 'same-origin','Sec-Fetch-User': '?1','Upgrade-Insecure-Requests': '1','User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36','sec-ch-ua': '".Not/A)Brand";v="99", "Google Chrome";v="103", "Chromium";v="103"','sec-ch-ua-mobile': '?0','sec-ch-ua-platform': '"Windows"',
}data = {'SearchCriteria.CodeVersion': '2021.0','SearchCriteria.CodeSearch': '10010  –  Financial managers','btn-submitSearchNOC': 'Search',
}r = requests.post('https://noc.esdc.gc.ca/Search/QuickSearchJobTitleResults', cookies=cookies, headers=headers, data=data)
id_ = re.findall(r'objectid=(.*?)&', r.text)[0]url = f'https://noc.esdc.gc.ca/Structure/NocProfile?objectid={id_}'
r = requests.get(url=url, headers=headers)print(r.text)