christoph/discord-heidi
1
Fork 0
Code Issues 9 Actions Packages Releases 3 Activity

cleanup

Browse Source
This commit is contained in:
churl
2022-01-19 15:09:43 +01:00
parent 5d6525fed6
commit a522a4aaa2
2 changed files with 4 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
.projectile
View File

19
scraper.py_deprecated → models.py
View File

@ -1,25 +1,15 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
### Models class Models:
class Girls():
def __init__(self): def __init__(self):
url_girls = "https://www.prosieben.de/tv/germanys-next-topmodel/models" url_girls = "https://www.prosieben.de/tv/germanys-next-topmodel/models"
options = Options() html_girls = requests.get(url_girls)
options.headless = True soup_girls = BeautifulSoup(html_girls.text, "html.parser")
driver = webdriver.Firefox(options=options)
driver.get(url_girls)
driver.execute_script("window.scrollTo(0, 1080);") # scroll to load page document.body.scrollHeight for page height
html_girls = driver.page_source
driver.close()
soup_girls = BeautifulSoup(html_girls, "html.parser")
girls_in = soup_girls.findAll("a", class_="candidate-in") girls_in = soup_girls.findAll("a", class_="candidate-in")
girls_out = soup_girls.findAll("a", class_="candidate-out") girls_out = soup_girls.findAll("a", class_="candidate-out")
@ -28,7 +18,6 @@ class Girls():
self.girls_out = {girl.find("h4", class_="candidate-title").text.lower(): girl for girl in girls_out} self.girls_out = {girl.find("h4", class_="candidate-title").text.lower(): girl for girl in girls_out}
self.girls = {**self.girls_in, **self.girls_out} self.girls = {**self.girls_in, **self.girls_out}
def get_in_names(self): def get_in_names(self):
return self.girls_in.keys() return self.girls_in.keys()