241 lines
8.8 KiB
Python
241 lines
8.8 KiB
Python
|
"""Main funda scraper module"""
|
||
|
|
import multiprocessing as mp
|
||
|
|
import os
|
||
|
|
import pandas as pd
|
||
|
|
import requests
|
||
|
|
from bs4 import BeautifulSoup
|
||
|
|
from typing import List, Dict
|
||
|
|
import datetime
|
||
|
|
from funda_scraper.config.core import config
|
||
|
|
from funda_scraper.preprocess import preprocess_data
|
||
|
|
from funda_scraper.utils import logger
|
||
|
|
from tqdm import tqdm
|
||
|
|
from tqdm.contrib.concurrent import process_map
|
||
|
|
|
||
|
|
|
||
|
|
class FundaScraper:
|
||
|
|
"""
|
||
|
|
Handles the main scraping function.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(
|
||
|
|
self,
|
||
|
|
area: str = None,
|
||
|
|
want_to: str = "buy",
|
||
|
|
n_pages: int = 1,
|
||
|
|
find_past: bool = False,
|
||
|
|
):
|
||
|
|
self.area = area.lower().replace(" ", "-") if isinstance(area, str) else area
|
||
|
|
self.want_to = want_to
|
||
|
|
self.find_past = find_past
|
||
|
|
self.n_pages = min(max(n_pages, 1), 999)
|
||
|
|
self.links: List[str] = []
|
||
|
|
self.raw_df = pd.DataFrame()
|
||
|
|
self.clean_df = pd.DataFrame()
|
||
|
|
self.base_url = config.base_url
|
||
|
|
self.selectors = config.css_selector
|
||
|
|
|
||
|
|
def __repr__(self):
|
||
|
|
return (
|
||
|
|
f"FundaScraper(area={self.area}, "
|
||
|
|
f"want_to={self.want_to}, "
|
||
|
|
f"n_pages={self.n_pages}, "
|
||
|
|
f"use_past_data={self.find_past})"
|
||
|
|
)
|
||
|
|
|
||
|
|
@property
|
||
|
|
def site_url(self) -> Dict[str, str]:
|
||
|
|
"""Return the corresponding urls."""
|
||
|
|
if self.to_buy:
|
||
|
|
return {
|
||
|
|
"close": f"{self.base_url}/koop/verkocht/{self.area}/",
|
||
|
|
"open": f"{self.base_url}/koop/{self.area}/",
|
||
|
|
}
|
||
|
|
else:
|
||
|
|
return {
|
||
|
|
"close": f"{self.base_url}/huur/{self.area}/verhuurd/",
|
||
|
|
"open": f"{self.base_url}/huur/{self.area}/",
|
||
|
|
}
|
||
|
|
|
||
|
|
@property
|
||
|
|
def to_buy(self) -> bool:
|
||
|
|
"""Whether to buy or not"""
|
||
|
|
if self.want_to.lower() in ["buy", "koop", "b"]:
|
||
|
|
return True
|
||
|
|
elif self.want_to.lower() in ["rent", "huur", "r"]:
|
||
|
|
return False
|
||
|
|
else:
|
||
|
|
raise ValueError("'want_to' must be 'either buy' or 'rent'.")
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _check_dir() -> None:
|
||
|
|
"""Check whether a temporary directory for data"""
|
||
|
|
if not os.path.exists("data"):
|
||
|
|
os.makedirs("data")
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _get_links_from_one_page(url: str) -> List[str]:
|
||
|
|
"""Scrape all the available housing items from one Funda search page."""
|
||
|
|
response = requests.get(url, headers=config.header)
|
||
|
|
soup = BeautifulSoup(response.text, "lxml")
|
||
|
|
house = soup.find_all(attrs={"data-object-url-tracking": "resultlist"})
|
||
|
|
item_list = [h.get("href") for h in house]
|
||
|
|
return list(set(item_list))
|
||
|
|
|
||
|
|
def init(
|
||
|
|
self,
|
||
|
|
area: str = None,
|
||
|
|
want_to: str = None,
|
||
|
|
n_pages: int = None,
|
||
|
|
find_past: bool = None,
|
||
|
|
) -> None:
|
||
|
|
"""Overwrite or initialise the searching scope."""
|
||
|
|
if area is not None:
|
||
|
|
self.area = area
|
||
|
|
if want_to is not None:
|
||
|
|
self.want_to = want_to
|
||
|
|
if n_pages is not None:
|
||
|
|
self.n_pages = n_pages
|
||
|
|
if find_past is not None:
|
||
|
|
self.find_past = find_past
|
||
|
|
|
||
|
|
def fetch_links(self) -> None:
|
||
|
|
"""Find all the available links across multiple pages. """
|
||
|
|
if self.area is None or self.want_to is None:
|
||
|
|
raise ValueError("You haven't set the area and what you're looking for.")
|
||
|
|
|
||
|
|
logger.info("*** Phase 1: Fetch all the available links from all pages *** ")
|
||
|
|
|
||
|
|
urls = []
|
||
|
|
main_url = self.site_url["close"] if self.find_past else self.site_url["open"]
|
||
|
|
for i in tqdm(range(0, self.n_pages + 1)):
|
||
|
|
item_list = self._get_links_from_one_page(main_url + f"p{i}")
|
||
|
|
if len(item_list) == 0:
|
||
|
|
self.n_pages = i
|
||
|
|
break
|
||
|
|
urls += item_list
|
||
|
|
urls = list(set(urls))
|
||
|
|
logger.info(
|
||
|
|
f"*** Got all the urls. {len(urls)} houses found in {self.n_pages} pages. ***"
|
||
|
|
)
|
||
|
|
self.links = ["https://www.funda.nl" + url for url in urls]
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def get_value(soup: BeautifulSoup, selector: str) -> str:
|
||
|
|
"""Use CSS selector to find certain features."""
|
||
|
|
try:
|
||
|
|
return soup.select(selector)[0].text
|
||
|
|
except IndexError:
|
||
|
|
return "na"
|
||
|
|
|
||
|
|
def scrape_from_url(self, url: str) -> List[str]:
|
||
|
|
"""Scrape all the features from one house item given a link. """
|
||
|
|
|
||
|
|
# Initialize for each page
|
||
|
|
response = requests.get(url, headers=config.header)
|
||
|
|
soup = BeautifulSoup(response.text, "lxml")
|
||
|
|
|
||
|
|
# Get the value according to respective CSS selectors
|
||
|
|
list_since_selector = (
|
||
|
|
self.selectors.listed_since
|
||
|
|
if self.to_buy
|
||
|
|
else ".fd-align-items-center:nth-child(7) span"
|
||
|
|
)
|
||
|
|
result = [
|
||
|
|
url,
|
||
|
|
self.get_value(soup, self.selectors.price),
|
||
|
|
self.get_value(soup, self.selectors.address),
|
||
|
|
self.get_value(soup, self.selectors.descrip),
|
||
|
|
self.get_value(soup, list_since_selector).replace("\n", ""),
|
||
|
|
self.get_value(soup, self.selectors.zip_code)
|
||
|
|
.replace("\n", "")
|
||
|
|
.replace("\r ", ""),
|
||
|
|
self.get_value(soup, self.selectors.size),
|
||
|
|
self.get_value(soup, self.selectors.year),
|
||
|
|
self.get_value(soup, self.selectors.living_area),
|
||
|
|
self.get_value(soup, self.selectors.kind_of_house),
|
||
|
|
self.get_value(soup, self.selectors.building_type),
|
||
|
|
self.get_value(soup, self.selectors.num_of_rooms).replace("\n", ""),
|
||
|
|
self.get_value(soup, self.selectors.num_of_bathrooms).replace("\n", ""),
|
||
|
|
self.get_value(soup, self.selectors.layout),
|
||
|
|
self.get_value(soup, self.selectors.energy_label).replace(
|
||
|
|
"\r\n ", ""
|
||
|
|
),
|
||
|
|
self.get_value(soup, self.selectors.insulation).replace("\n", ""),
|
||
|
|
self.get_value(soup, self.selectors.heating).replace("\n", ""),
|
||
|
|
self.get_value(soup, self.selectors.ownership).replace("\n", ""),
|
||
|
|
self.get_value(soup, self.selectors.exteriors),
|
||
|
|
self.get_value(soup, self.selectors.parking),
|
||
|
|
self.get_value(soup, self.selectors.neighborhood_name),
|
||
|
|
self.get_value(soup, self.selectors.date_list),
|
||
|
|
self.get_value(soup, self.selectors.date_sold),
|
||
|
|
self.get_value(soup, self.selectors.term),
|
||
|
|
self.get_value(soup, self.selectors.price_sold),
|
||
|
|
self.get_value(soup, self.selectors.last_ask_price).replace("\n", ""),
|
||
|
|
self.get_value(soup, self.selectors.last_ask_price_m2).split("\r")[0],
|
||
|
|
]
|
||
|
|
|
||
|
|
return result
|
||
|
|
|
||
|
|
def scrape_pages(self) -> None:
|
||
|
|
"""Scrape all the content acoss multiple pages."""
|
||
|
|
|
||
|
|
logger.info("*** Phase 2: Start scraping results from individual links ***")
|
||
|
|
df = pd.DataFrame({key: [] for key in self.selectors.keys()})
|
||
|
|
|
||
|
|
# Scrape pages with multiprocessing to improve efficiency
|
||
|
|
pools = mp.cpu_count()
|
||
|
|
content = process_map(self.scrape_from_url, self.links, max_workers=pools)
|
||
|
|
|
||
|
|
for i, c in enumerate(content):
|
||
|
|
df.loc[len(df)] = c
|
||
|
|
|
||
|
|
df["city"] = self.area
|
||
|
|
df["log_id"] = datetime.datetime.now().strftime("%Y%m-%d%H-%M%S")
|
||
|
|
logger.info(f"*** All scraping done: {df.shape[0]} results ***")
|
||
|
|
self.raw_df = df
|
||
|
|
|
||
|
|
def save_csv(self, df: pd.DataFrame, filepath: str = None) -> None:
|
||
|
|
"""Save the result to a .csv file."""
|
||
|
|
if filepath is None:
|
||
|
|
self._check_dir()
|
||
|
|
date = str(datetime.datetime.now().date()).replace("-", "")
|
||
|
|
if self.find_past:
|
||
|
|
if self.to_buy:
|
||
|
|
status = "sold"
|
||
|
|
else:
|
||
|
|
status = "rented"
|
||
|
|
else:
|
||
|
|
if self.to_buy:
|
||
|
|
status = "selling"
|
||
|
|
else:
|
||
|
|
status = "renting"
|
||
|
|
filepath = (
|
||
|
|
f"./data/houseprice_{date}_{self.area}_{status}_{len(self.links)}.csv"
|
||
|
|
)
|
||
|
|
df.to_csv(filepath, index=False)
|
||
|
|
logger.info(f"*** File saved: {filepath}. ***")
|
||
|
|
|
||
|
|
def run(self, raw_data: bool = False, save: bool = False, filepath: str = None) -> pd.DataFrame:
|
||
|
|
"""Scrape all links and all content."""
|
||
|
|
self.fetch_links()
|
||
|
|
self.scrape_pages()
|
||
|
|
|
||
|
|
if raw_data:
|
||
|
|
if save:
|
||
|
|
self.save_csv(self.raw_df, filepath)
|
||
|
|
return self.raw_df
|
||
|
|
else:
|
||
|
|
logger.info("*** Cleaning data ***")
|
||
|
|
clean_df = preprocess_data(df=self.raw_df, is_past=self.find_past)
|
||
|
|
self.clean_df = clean_df
|
||
|
|
if save:
|
||
|
|
self.save_csv(self.clean_df, filepath)
|
||
|
|
return clean_df
|
||
|
|
logger.info("*** Done! ***")
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
scraper = FundaScraper(area="amsterdam", want_to="rent", find_past=False, n_pages=1)
|
||
|
|
df = scraper.run()
|
||
|
|
print(df.head())
|