Added docs for tileserver
This commit is contained in:
parent
799fe81f39
commit
445f2400e2
Binary file not shown.
BIN
documentation/build/.doctrees/funda.doctree
Normal file
BIN
documentation/build/.doctrees/funda.doctree
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
documentation/build/_images/usse_castle.webp
Normal file
BIN
documentation/build/_images/usse_castle.webp
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 272 KiB |
31
documentation/build/_sources/funda.rst.txt
Normal file
31
documentation/build/_sources/funda.rst.txt
Normal file
@ -0,0 +1,31 @@
|
|||||||
|
##############
|
||||||
|
Scraping Funda
|
||||||
|
##############
|
||||||
|
``Funda`` is a real estate housing market that tries to keep track of all houses that are currently for sale.
|
||||||
|
Scraping is not allowed, but on github there are several projects that still try to do this.
|
||||||
|
|
||||||
|
A quick test from several github projects landed us with `this project <https://github.com/whchien/funda-scraper>`_.
|
||||||
|
|
||||||
|
This project still works, but is very limited in the filtering methods.
|
||||||
|
A few patches to code allows us to inject a URL that will be used and no other filters will be applied.
|
||||||
|
Next we can setup a basic filter in the browser and copy the URL in order to do scraping.
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
if self.url != "":
|
||||||
|
# https://www.funda.nl/koop/gemeente-huizen/0-350000/tuin/+10km/
|
||||||
|
# gemeente-huizen/0-350000/tuin/+10km/
|
||||||
|
return {
|
||||||
|
"close": f"{self.base_url}/koop/verkocht/{self.url}/",
|
||||||
|
"open": f"{self.base_url}/koop/{self.url}/",
|
||||||
|
}
|
||||||
|
|
||||||
|
Scrape funda with URL:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
def get_funda_data():
|
||||||
|
scraper = FundaScraper(url="nijkerk/beschikbaar/100000-400000/woonhuis/tuin/eengezinswoning/landhuis/+30km/", find_past=False, n_pages=81)
|
||||||
|
df = scraper.run()
|
||||||
|
return df
|
||||||
|
|
||||||
@ -6,18 +6,19 @@
|
|||||||
Project Usse
|
Project Usse
|
||||||
============
|
============
|
||||||
|
|
||||||
|
.. image:: images/usse_castle.webp
|
||||||
|
|
||||||
.. toctree::
|
.. toctree::
|
||||||
:maxdepth: 2
|
:maxdepth: 2
|
||||||
:caption: Contents:
|
:caption: Contents:
|
||||||
|
|
||||||
|
funda.rst
|
||||||
googlemaps.rst
|
googlemaps.rst
|
||||||
osm.rst
|
osm.rst
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Indices and tables
|
Goal
|
||||||
==================
|
====
|
||||||
|
The Goal of this project is to, automatically, scrape `real estate housing website Funda <https://www.funda.nl>`_ for houses with some specifications(garden, non-apartment).
|
||||||
* :ref:`genindex`
|
Next, we must be able to view these houses and filter them based on the distance to several locations(NFI, Hooghstraat etc.).
|
||||||
* :ref:`modindex`
|
|
||||||
* :ref:`search`
|
|
||||||
@ -67,4 +67,26 @@ In order to prevent bots from misusing these interfaces a firewall rule was adde
|
|||||||
.. code-block:: console
|
.. code-block:: console
|
||||||
|
|
||||||
$ sudo ufw allow from <ip_addr> to any port 5998
|
$ sudo ufw allow from <ip_addr> to any port 5998
|
||||||
$ sudo ufw allow from <ip_addr> to any port 5999
|
$ sudo ufw allow from <ip_addr> to any port 5999
|
||||||
|
|
||||||
|
|
||||||
|
Tile Server
|
||||||
|
###########
|
||||||
|
In order to also serve tiles and be fully independent from ``Google Maps`` a tileserver was also started.
|
||||||
|
For this the previously donwloaded .pbf file needs to be imported and a postgresql database was used to *hopefully* speed up the performance.
|
||||||
|
|
||||||
|
First we need to import the data, then start a tileserver.
|
||||||
|
|
||||||
|
.. code-block:: console
|
||||||
|
|
||||||
|
Import data
|
||||||
|
$ docker run -v "${PWD}/data/netherlands-latest.osm.pbf:/data.osm.pbf" -v openstreetmap-data:/var/lib/postgresql/12/main overv/openstreetmap-tile-server:1.3.10 import
|
||||||
|
|
||||||
|
Run tile server
|
||||||
|
$ docker run -p 5997:80 -v openstreetmap-data:/var/lib/postgresql/12/main -d overv/openstreetmap-tile-server:1.3.10 run
|
||||||
|
|
||||||
|
This will open a map server at port **5997**. When navigating to *http://www.herreweb.nl:5997/tile/0/0/0.png*, a world map is shown.
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
|
||||||
|
The import step took over 2 hours on a 6-core *slow* server(Intel Xeon E5-2620 v2). The map server is not fast either, but faster after loading all the map tiles.
|
||||||
130
documentation/build/funda.html
Normal file
130
documentation/build/funda.html
Normal file
@ -0,0 +1,130 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html class="writer-html5" lang="en" >
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||||
|
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<title>Scraping Funda — Usse 1 documentation</title>
|
||||||
|
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
|
||||||
|
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
|
||||||
|
<!--[if lt IE 9]>
|
||||||
|
<script src="_static/js/html5shiv.min.js"></script>
|
||||||
|
<![endif]-->
|
||||||
|
|
||||||
|
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||||
|
<script src="_static/jquery.js"></script>
|
||||||
|
<script src="_static/underscore.js"></script>
|
||||||
|
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||||
|
<script src="_static/doctools.js"></script>
|
||||||
|
<script src="_static/sphinx_highlight.js"></script>
|
||||||
|
<script src="_static/js/theme.js"></script>
|
||||||
|
<link rel="index" title="Index" href="genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="search.html" />
|
||||||
|
<link rel="next" title="Google Maps API" href="googlemaps.html" />
|
||||||
|
<link rel="prev" title="Project Usse" href="index.html" />
|
||||||
|
</head>
|
||||||
|
|
||||||
|
<body class="wy-body-for-nav">
|
||||||
|
<div class="wy-grid-for-nav">
|
||||||
|
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
||||||
|
<div class="wy-side-scroll">
|
||||||
|
<div class="wy-side-nav-search" >
|
||||||
|
<a href="index.html" class="icon icon-home"> Usse
|
||||||
|
</a>
|
||||||
|
<div role="search">
|
||||||
|
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||||
|
<input type="text" name="q" placeholder="Search docs" />
|
||||||
|
<input type="hidden" name="check_keywords" value="yes" />
|
||||||
|
<input type="hidden" name="area" value="default" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
||||||
|
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
||||||
|
<ul class="current">
|
||||||
|
<li class="toctree-l1 current"><a class="current reference internal" href="#">Scraping Funda</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="googlemaps.html">Google Maps API</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="osm.html">Open Street Maps</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
|
||||||
|
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||||||
|
<a href="index.html">Usse</a>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
<div class="wy-nav-content">
|
||||||
|
<div class="rst-content">
|
||||||
|
<div role="navigation" aria-label="Page navigation">
|
||||||
|
<ul class="wy-breadcrumbs">
|
||||||
|
<li><a href="index.html" class="icon icon-home"></a></li>
|
||||||
|
<li class="breadcrumb-item active">Scraping Funda</li>
|
||||||
|
<li class="wy-breadcrumbs-aside">
|
||||||
|
<a href="_sources/funda.rst.txt" rel="nofollow"> View page source</a>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<hr/>
|
||||||
|
</div>
|
||||||
|
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
||||||
|
<div itemprop="articleBody">
|
||||||
|
|
||||||
|
<section id="scraping-funda">
|
||||||
|
<h1>Scraping Funda<a class="headerlink" href="#scraping-funda" title="Permalink to this heading"></a></h1>
|
||||||
|
<p><code class="docutils literal notranslate"><span class="pre">Funda</span></code> is a real estate housing market that tries to keep track of all houses that are currently for sale.
|
||||||
|
Scraping is not allowed, but on github there are several projects that still try to do this.</p>
|
||||||
|
<p>A quick test from several github projects landed us with <a class="reference external" href="https://github.com/whchien/funda-scraper">this project</a>.</p>
|
||||||
|
<p>This project still works, but is very limited in the filtering methods.
|
||||||
|
A few patches to code allows us to inject a URL that will be used and no other filters will be applied.
|
||||||
|
Next we can setup a basic filter in the browser and copy the URL in order to do scraping.</p>
|
||||||
|
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">!=</span> <span class="s2">""</span><span class="p">:</span>
|
||||||
|
<span class="c1"># https://www.funda.nl/koop/gemeente-huizen/0-350000/tuin/+10km/</span>
|
||||||
|
<span class="c1"># gemeente-huizen/0-350000/tuin/+10km/</span>
|
||||||
|
<span class="k">return</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"close"</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">base_url</span><span class="si">}</span><span class="s2">/koop/verkocht/</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="si">}</span><span class="s2">/"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"open"</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">base_url</span><span class="si">}</span><span class="s2">/koop/</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="si">}</span><span class="s2">/"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
<p>Scrape funda with URL:</p>
|
||||||
|
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">get_funda_data</span><span class="p">():</span>
|
||||||
|
<span class="n">scraper</span> <span class="o">=</span> <span class="n">FundaScraper</span><span class="p">(</span><span class="n">url</span><span class="o">=</span><span class="s2">"nijkerk/beschikbaar/100000-400000/woonhuis/tuin/eengezinswoning/landhuis/+30km/"</span><span class="p">,</span> <span class="n">find_past</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">n_pages</span><span class="o">=</span><span class="mi">81</span><span class="p">)</span>
|
||||||
|
<span class="n">df</span> <span class="o">=</span> <span class="n">scraper</span><span class="o">.</span><span class="n">run</span><span class="p">()</span>
|
||||||
|
<span class="k">return</span> <span class="n">df</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
|
||||||
|
<a href="index.html" class="btn btn-neutral float-left" title="Project Usse" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
|
||||||
|
<a href="googlemaps.html" class="btn btn-neutral float-right" title="Google Maps API" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<hr/>
|
||||||
|
|
||||||
|
<div role="contentinfo">
|
||||||
|
<p>© Copyright 2023, Eljakim.</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||||
|
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
|
||||||
|
provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
||||||
|
|
||||||
|
|
||||||
|
</footer>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
<script>
|
||||||
|
jQuery(function () {
|
||||||
|
SphinxRtdTheme.Navigation.enable(true);
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@ -38,6 +38,7 @@
|
|||||||
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
||||||
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
||||||
<ul>
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="funda.html">Scraping Funda</a></li>
|
||||||
<li class="toctree-l1"><a class="reference internal" href="googlemaps.html">Google Maps API</a></li>
|
<li class="toctree-l1"><a class="reference internal" href="googlemaps.html">Google Maps API</a></li>
|
||||||
<li class="toctree-l1"><a class="reference internal" href="osm.html">Open Street Maps</a></li>
|
<li class="toctree-l1"><a class="reference internal" href="osm.html">Open Street Maps</a></li>
|
||||||
</ul>
|
</ul>
|
||||||
|
|||||||
@ -20,7 +20,7 @@
|
|||||||
<script src="_static/js/theme.js"></script>
|
<script src="_static/js/theme.js"></script>
|
||||||
<link rel="index" title="Index" href="genindex.html" />
|
<link rel="index" title="Index" href="genindex.html" />
|
||||||
<link rel="search" title="Search" href="search.html" />
|
<link rel="search" title="Search" href="search.html" />
|
||||||
<link rel="next" title="Google Maps API" href="googlemaps.html" />
|
<link rel="next" title="Scraping Funda" href="funda.html" />
|
||||||
</head>
|
</head>
|
||||||
|
|
||||||
<body class="wy-body-for-nav">
|
<body class="wy-body-for-nav">
|
||||||
@ -40,6 +40,7 @@
|
|||||||
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
||||||
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
||||||
<ul>
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="funda.html">Scraping Funda</a></li>
|
||||||
<li class="toctree-l1"><a class="reference internal" href="googlemaps.html">Google Maps API</a></li>
|
<li class="toctree-l1"><a class="reference internal" href="googlemaps.html">Google Maps API</a></li>
|
||||||
<li class="toctree-l1"><a class="reference internal" href="osm.html">Open Street Maps</a></li>
|
<li class="toctree-l1"><a class="reference internal" href="osm.html">Open Street Maps</a></li>
|
||||||
</ul>
|
</ul>
|
||||||
@ -70,9 +71,11 @@
|
|||||||
|
|
||||||
<section id="project-usse">
|
<section id="project-usse">
|
||||||
<h1>Project Usse<a class="headerlink" href="#project-usse" title="Permalink to this heading"></a></h1>
|
<h1>Project Usse<a class="headerlink" href="#project-usse" title="Permalink to this heading"></a></h1>
|
||||||
|
<img alt="_images/usse_castle.webp" src="_images/usse_castle.webp" />
|
||||||
<div class="toctree-wrapper compound">
|
<div class="toctree-wrapper compound">
|
||||||
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
||||||
<ul>
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="funda.html">Scraping Funda</a></li>
|
||||||
<li class="toctree-l1"><a class="reference internal" href="googlemaps.html">Google Maps API</a><ul>
|
<li class="toctree-l1"><a class="reference internal" href="googlemaps.html">Google Maps API</a><ul>
|
||||||
<li class="toctree-l2"><a class="reference internal" href="googlemaps.html#using-the-google-api">Using the Google API</a></li>
|
<li class="toctree-l2"><a class="reference internal" href="googlemaps.html#using-the-google-api">Using the Google API</a></li>
|
||||||
<li class="toctree-l2"><a class="reference internal" href="googlemaps.html#google-ban">Google Ban</a></li>
|
<li class="toctree-l2"><a class="reference internal" href="googlemaps.html#google-ban">Google Ban</a></li>
|
||||||
@ -82,25 +85,23 @@
|
|||||||
<li class="toctree-l2"><a class="reference internal" href="osm.html#osrm">OSRM</a></li>
|
<li class="toctree-l2"><a class="reference internal" href="osm.html#osrm">OSRM</a></li>
|
||||||
<li class="toctree-l2"><a class="reference internal" href="osm.html#nominatim">Nominatim</a></li>
|
<li class="toctree-l2"><a class="reference internal" href="osm.html#nominatim">Nominatim</a></li>
|
||||||
<li class="toctree-l2"><a class="reference internal" href="osm.html#ip-firewall">IP Firewall</a></li>
|
<li class="toctree-l2"><a class="reference internal" href="osm.html#ip-firewall">IP Firewall</a></li>
|
||||||
|
<li class="toctree-l2"><a class="reference internal" href="osm.html#tile-server">Tile Server</a></li>
|
||||||
</ul>
|
</ul>
|
||||||
</li>
|
</li>
|
||||||
</ul>
|
</ul>
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
<section id="indices-and-tables">
|
<section id="goal">
|
||||||
<h1>Indices and tables<a class="headerlink" href="#indices-and-tables" title="Permalink to this heading"></a></h1>
|
<h1>Goal<a class="headerlink" href="#goal" title="Permalink to this heading"></a></h1>
|
||||||
<ul class="simple">
|
<p>The Goal of this project is to, automatically, scrape <a class="reference external" href="https://www.funda.nl">real estate housing website Funda</a> for houses with some specifications(garden, non-apartment).
|
||||||
<li><p><a class="reference internal" href="genindex.html"><span class="std std-ref">Index</span></a></p></li>
|
Next, we must be able to view these houses and filter them based on the distance to several locations(NFI, Hooghstraat etc.).</p>
|
||||||
<li><p><a class="reference internal" href="py-modindex.html"><span class="std std-ref">Module Index</span></a></p></li>
|
|
||||||
<li><p><a class="reference internal" href="search.html"><span class="std std-ref">Search Page</span></a></p></li>
|
|
||||||
</ul>
|
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
|
<footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
|
||||||
<a href="googlemaps.html" class="btn btn-neutral float-right" title="Google Maps API" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
|
<a href="funda.html" class="btn btn-neutral float-right" title="Scraping Funda" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<hr/>
|
<hr/>
|
||||||
|
|||||||
Binary file not shown.
@ -40,6 +40,7 @@
|
|||||||
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
||||||
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
||||||
<ul class="current">
|
<ul class="current">
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="funda.html">Scraping Funda</a></li>
|
||||||
<li class="toctree-l1"><a class="reference internal" href="googlemaps.html">Google Maps API</a></li>
|
<li class="toctree-l1"><a class="reference internal" href="googlemaps.html">Google Maps API</a></li>
|
||||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Open Street Maps</a><ul>
|
<li class="toctree-l1 current"><a class="current reference internal" href="#">Open Street Maps</a><ul>
|
||||||
<li class="toctree-l2"><a class="reference internal" href="#osrm">OSRM</a><ul>
|
<li class="toctree-l2"><a class="reference internal" href="#osrm">OSRM</a><ul>
|
||||||
@ -48,6 +49,7 @@
|
|||||||
</li>
|
</li>
|
||||||
<li class="toctree-l2"><a class="reference internal" href="#nominatim">Nominatim</a></li>
|
<li class="toctree-l2"><a class="reference internal" href="#nominatim">Nominatim</a></li>
|
||||||
<li class="toctree-l2"><a class="reference internal" href="#ip-firewall">IP Firewall</a></li>
|
<li class="toctree-l2"><a class="reference internal" href="#ip-firewall">IP Firewall</a></li>
|
||||||
|
<li class="toctree-l2"><a class="reference internal" href="#tile-server">Tile Server</a></li>
|
||||||
</ul>
|
</ul>
|
||||||
</li>
|
</li>
|
||||||
</ul>
|
</ul>
|
||||||
@ -139,6 +141,24 @@ In order to prevent bots from misusing these interfaces a firewall rule was adde
|
|||||||
</pre></div>
|
</pre></div>
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
<section id="tile-server">
|
||||||
|
<h2>Tile Server<a class="headerlink" href="#tile-server" title="Permalink to this heading"></a></h2>
|
||||||
|
<p>In order to also serve tiles and be fully independent from <code class="docutils literal notranslate"><span class="pre">Google</span> <span class="pre">Maps</span></code> a tileserver was also started.
|
||||||
|
For this the previously donwloaded .pbf file needs to be imported and a postgresql database was used to <em>hopefully</em> speed up the performance.</p>
|
||||||
|
<p>First we need to import the data, then start a tileserver.</p>
|
||||||
|
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">Import data</span>
|
||||||
|
<span class="gp">$ </span>docker run -v <span class="s2">"</span><span class="si">${</span><span class="nv">PWD</span><span class="si">}</span><span class="s2">/data/netherlands-latest.osm.pbf:/data.osm.pbf"</span> -v openstreetmap-data:/var/lib/postgresql/12/main overv/openstreetmap-tile-server:1.3.10 import
|
||||||
|
|
||||||
|
<span class="go">Run tile server</span>
|
||||||
|
<span class="gp">$ </span>docker run -p <span class="m">5997</span>:80 -v openstreetmap-data:/var/lib/postgresql/12/main -d overv/openstreetmap-tile-server:1.3.10 run
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
<p>This will open a map server at port <strong>5997</strong>. When navigating to <em>http://www.herreweb.nl:5997/tile/0/0/0.png</em>, a world map is shown.</p>
|
||||||
|
<div class="admonition note">
|
||||||
|
<p class="admonition-title">Note</p>
|
||||||
|
<p>The import step took over 2 hours on a 6-core <em>slow</em> server(Intel Xeon E5-2620 v2). The map server is not fast either, but faster after loading all the map tiles.</p>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -41,6 +41,7 @@
|
|||||||
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
||||||
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
||||||
<ul>
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="funda.html">Scraping Funda</a></li>
|
||||||
<li class="toctree-l1"><a class="reference internal" href="googlemaps.html">Google Maps API</a></li>
|
<li class="toctree-l1"><a class="reference internal" href="googlemaps.html">Google Maps API</a></li>
|
||||||
<li class="toctree-l1"><a class="reference internal" href="osm.html">Open Street Maps</a></li>
|
<li class="toctree-l1"><a class="reference internal" href="osm.html">Open Street Maps</a></li>
|
||||||
</ul>
|
</ul>
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
31
documentation/source/funda.rst
Normal file
31
documentation/source/funda.rst
Normal file
@ -0,0 +1,31 @@
|
|||||||
|
##############
|
||||||
|
Scraping Funda
|
||||||
|
##############
|
||||||
|
``Funda`` is a real estate housing market that tries to keep track of all houses that are currently for sale.
|
||||||
|
Scraping is not allowed, but on github there are several projects that still try to do this.
|
||||||
|
|
||||||
|
A quick test from several github projects landed us with `this project <https://github.com/whchien/funda-scraper>`_.
|
||||||
|
|
||||||
|
This project still works, but is very limited in the filtering methods.
|
||||||
|
A few patches to code allows us to inject a URL that will be used and no other filters will be applied.
|
||||||
|
Next we can setup a basic filter in the browser and copy the URL in order to do scraping.
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
if self.url != "":
|
||||||
|
# https://www.funda.nl/koop/gemeente-huizen/0-350000/tuin/+10km/
|
||||||
|
# gemeente-huizen/0-350000/tuin/+10km/
|
||||||
|
return {
|
||||||
|
"close": f"{self.base_url}/koop/verkocht/{self.url}/",
|
||||||
|
"open": f"{self.base_url}/koop/{self.url}/",
|
||||||
|
}
|
||||||
|
|
||||||
|
Scrape funda with URL:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
def get_funda_data():
|
||||||
|
scraper = FundaScraper(url="nijkerk/beschikbaar/100000-400000/woonhuis/tuin/eengezinswoning/landhuis/+30km/", find_past=False, n_pages=81)
|
||||||
|
df = scraper.run()
|
||||||
|
return df
|
||||||
|
|
||||||
BIN
documentation/source/images/usse_castle.webp
Normal file
BIN
documentation/source/images/usse_castle.webp
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 272 KiB |
@ -6,18 +6,19 @@
|
|||||||
Project Usse
|
Project Usse
|
||||||
============
|
============
|
||||||
|
|
||||||
|
.. image:: images/usse_castle.webp
|
||||||
|
|
||||||
.. toctree::
|
.. toctree::
|
||||||
:maxdepth: 2
|
:maxdepth: 2
|
||||||
:caption: Contents:
|
:caption: Contents:
|
||||||
|
|
||||||
|
funda.rst
|
||||||
googlemaps.rst
|
googlemaps.rst
|
||||||
osm.rst
|
osm.rst
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Indices and tables
|
Goal
|
||||||
==================
|
====
|
||||||
|
The Goal of this project is to, automatically, scrape `real estate housing website Funda <https://www.funda.nl>`_ for houses with some specifications(garden, non-apartment).
|
||||||
* :ref:`genindex`
|
Next, we must be able to view these houses and filter them based on the distance to several locations(NFI, Hooghstraat etc.).
|
||||||
* :ref:`modindex`
|
|
||||||
* :ref:`search`
|
|
||||||
@ -67,4 +67,26 @@ In order to prevent bots from misusing these interfaces a firewall rule was adde
|
|||||||
.. code-block:: console
|
.. code-block:: console
|
||||||
|
|
||||||
$ sudo ufw allow from <ip_addr> to any port 5998
|
$ sudo ufw allow from <ip_addr> to any port 5998
|
||||||
$ sudo ufw allow from <ip_addr> to any port 5999
|
$ sudo ufw allow from <ip_addr> to any port 5999
|
||||||
|
|
||||||
|
|
||||||
|
Tile Server
|
||||||
|
###########
|
||||||
|
In order to also serve tiles and be fully independent from ``Google Maps`` a tileserver was also started.
|
||||||
|
For this the previously donwloaded .pbf file needs to be imported and a postgresql database was used to *hopefully* speed up the performance.
|
||||||
|
|
||||||
|
First we need to import the data, then start a tileserver.
|
||||||
|
|
||||||
|
.. code-block:: console
|
||||||
|
|
||||||
|
Import data
|
||||||
|
$ docker run -v "${PWD}/data/netherlands-latest.osm.pbf:/data.osm.pbf" -v openstreetmap-data:/var/lib/postgresql/12/main overv/openstreetmap-tile-server:1.3.10 import
|
||||||
|
|
||||||
|
Run tile server
|
||||||
|
$ docker run -p 5997:80 -v openstreetmap-data:/var/lib/postgresql/12/main -d overv/openstreetmap-tile-server:1.3.10 run
|
||||||
|
|
||||||
|
This will open a map server at port **5997**. When navigating to *http://www.herreweb.nl:5997/tile/0/0/0.png*, a world map is shown.
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
|
||||||
|
The import step took over 2 hours on a 6-core *slow* server(Intel Xeon E5-2620 v2). The map server is not fast either, but faster after loading all the map tiles.
|
||||||
Binary file not shown.
@ -31,7 +31,9 @@ def get_distances(out_dict, destination_location):
|
|||||||
out_dict[key] = distance['routes'][0]
|
out_dict[key] = distance['routes'][0]
|
||||||
|
|
||||||
def generate_json(houses):
|
def generate_json(houses):
|
||||||
|
count = 0
|
||||||
for i in tqdm.tqdm(range(len(houses))):
|
for i in tqdm.tqdm(range(len(houses))):
|
||||||
|
count += 1
|
||||||
out_dict = {}
|
out_dict = {}
|
||||||
zip_code = houses.zip_code.get(i)
|
zip_code = houses.zip_code.get(i)
|
||||||
|
|
||||||
@ -57,7 +59,8 @@ def generate_json(houses):
|
|||||||
destination_location = [destination_location.longitude, destination_location.latitude]
|
destination_location = [destination_location.longitude, destination_location.latitude]
|
||||||
|
|
||||||
# distance_matrix = gmaps.distance_matrix(origin_locations['nfi_location'], destination_location, mode = 'driving')
|
# distance_matrix = gmaps.distance_matrix(origin_locations['nfi_location'], destination_location, mode = 'driving')
|
||||||
out_dict['name'] = address
|
|
||||||
|
out_dict['name'] = f"{address}_{count}" # Fix for duplicate names in dictionary.
|
||||||
out_dict['position'] = destination_location
|
out_dict['position'] = destination_location
|
||||||
for key in houses.keys():
|
for key in houses.keys():
|
||||||
out_dict[key] = houses.__getattr__(key).get(i)
|
out_dict[key] = houses.__getattr__(key).get(i)
|
||||||
|
|||||||
23
react_usse/.gitignore
vendored
Normal file
23
react_usse/.gitignore
vendored
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
||||||
|
|
||||||
|
# dependencies
|
||||||
|
/node_modules
|
||||||
|
/.pnp
|
||||||
|
.pnp.js
|
||||||
|
|
||||||
|
# testing
|
||||||
|
/coverage
|
||||||
|
|
||||||
|
# production
|
||||||
|
/build
|
||||||
|
|
||||||
|
# misc
|
||||||
|
.DS_Store
|
||||||
|
.env.local
|
||||||
|
.env.development.local
|
||||||
|
.env.test.local
|
||||||
|
.env.production.local
|
||||||
|
|
||||||
|
npm-debug.log*
|
||||||
|
yarn-debug.log*
|
||||||
|
yarn-error.log*
|
||||||
15
react_usse/.vscode/launch.json
vendored
Normal file
15
react_usse/.vscode/launch.json
vendored
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
// Use IntelliSense to learn about possible attributes.
|
||||||
|
// Hover to view descriptions of existing attributes.
|
||||||
|
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||||
|
"version": "0.2.0",
|
||||||
|
"configurations": [
|
||||||
|
{
|
||||||
|
"type": "chrome",
|
||||||
|
"request": "launch",
|
||||||
|
"name": "Launch Chrome against localhost",
|
||||||
|
"url": "http://localhost:3000",
|
||||||
|
"webRoot": "${workspaceFolder}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
20
react_usse/.vscode/tasks.json
vendored
Normal file
20
react_usse/.vscode/tasks.json
vendored
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"version": "2.0.0",
|
||||||
|
"tasks": [
|
||||||
|
{
|
||||||
|
"type": "npm",
|
||||||
|
"script": "build",
|
||||||
|
"group": "build",
|
||||||
|
"problemMatcher": [],
|
||||||
|
"label": "npm: build",
|
||||||
|
"detail": "react-scripts build"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "npm",
|
||||||
|
"script": "start",
|
||||||
|
"problemMatcher": [],
|
||||||
|
"label": "npm: start",
|
||||||
|
"detail": "react-scripts start"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
28401
react_usse/package-lock.json
generated
Normal file
28401
react_usse/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
30
react_usse/package.json
Normal file
30
react_usse/package.json
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
{
|
||||||
|
"name": "Project Usse Map",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"private": true,
|
||||||
|
"dependencies": {
|
||||||
|
"escape-string-regexp": "^1.0.5",
|
||||||
|
"leaflet" : "",
|
||||||
|
"react-leaflet" : "4.2.1",
|
||||||
|
"react": "^18.2.0",
|
||||||
|
"react-dom": "^18.2.0",
|
||||||
|
"react-icons": "^3.2.2",
|
||||||
|
"react-scripts": "^5.0.1",
|
||||||
|
"sort-by": "^1.2.0"
|
||||||
|
},
|
||||||
|
"scripts": {
|
||||||
|
"start": "react-scripts start",
|
||||||
|
"build": "react-scripts build",
|
||||||
|
"test": "react-scripts test",
|
||||||
|
"eject": "react-scripts eject"
|
||||||
|
},
|
||||||
|
"eslintConfig": {
|
||||||
|
"extends": "react-app"
|
||||||
|
},
|
||||||
|
"browserslist": [
|
||||||
|
">0.2%",
|
||||||
|
"not dead",
|
||||||
|
"not ie <= 11",
|
||||||
|
"not op_mini all"
|
||||||
|
]
|
||||||
|
}
|
||||||
BIN
react_usse/public/favicon.ico
Normal file
BIN
react_usse/public/favicon.ico
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 3.8 KiB |
40
react_usse/public/index.html
Normal file
40
react_usse/public/index.html
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<link rel="shortcut icon" href="%PUBLIC_URL%/favicon.ico">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
||||||
|
<meta name="theme-color" content="#000000">
|
||||||
|
<!--
|
||||||
|
manifest.json provides metadata used when your web app is added to the
|
||||||
|
homescreen on Android. See https://developers.google.com/web/fundamentals/web-app-manifest/
|
||||||
|
-->
|
||||||
|
<link rel="manifest" href="%PUBLIC_URL%/manifest.json">
|
||||||
|
<!--
|
||||||
|
Notice the use of %PUBLIC_URL% in the tags above.
|
||||||
|
It will be replaced with the URL of the `public` folder during the build.
|
||||||
|
Only files inside the `public` folder can be referenced from the HTML.
|
||||||
|
|
||||||
|
Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
|
||||||
|
work correctly both with client-side routing and a non-root public URL.
|
||||||
|
Learn how to configure a non-root public URL by running `npm run build`.
|
||||||
|
-->
|
||||||
|
<title>React App</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<noscript>
|
||||||
|
You need to enable JavaScript to run this app.
|
||||||
|
</noscript>
|
||||||
|
<div id="root"></div>
|
||||||
|
<!--
|
||||||
|
This HTML file is a template.
|
||||||
|
If you open it directly in the browser, you will see an empty page.
|
||||||
|
|
||||||
|
You can add webfonts, meta tags, or analytics to this file.
|
||||||
|
The build step will place the bundled scripts into the <body> tag.
|
||||||
|
|
||||||
|
To begin the development, run `npm start` or `yarn start`.
|
||||||
|
To create a production bundle, use `npm run build` or `yarn build`.
|
||||||
|
-->
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
15
react_usse/public/manifest.json
Normal file
15
react_usse/public/manifest.json
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"short_name": "React App",
|
||||||
|
"name": "Create React App Sample",
|
||||||
|
"icons": [
|
||||||
|
{
|
||||||
|
"src": "favicon.ico",
|
||||||
|
"sizes": "64x64 32x32 24x24 16x16",
|
||||||
|
"type": "image/x-icon"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"start_url": ".",
|
||||||
|
"display": "standalone",
|
||||||
|
"theme_color": "#000000",
|
||||||
|
"background_color": "#ffffff"
|
||||||
|
}
|
||||||
132
react_usse/src/App.css
Normal file
132
react_usse/src/App.css
Normal file
@ -0,0 +1,132 @@
|
|||||||
|
.App {
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
/* App Header */
|
||||||
|
.App-header {
|
||||||
|
background-color: #282c34;
|
||||||
|
min-height: 10vh;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
font-size: calc(5px + 1vmin);
|
||||||
|
color: white;
|
||||||
|
padding-left: 40px;
|
||||||
|
|
||||||
|
}
|
||||||
|
/* Menu button */
|
||||||
|
.toggle-menu {
|
||||||
|
background-color: #737579;
|
||||||
|
position: absolute;
|
||||||
|
min-width: 55px;
|
||||||
|
min-height: 55px;
|
||||||
|
left: 25px;
|
||||||
|
margin-left: calc(10px+.5vmin);
|
||||||
|
margin-right: calc(10px+.5vmin);
|
||||||
|
top: 10px;
|
||||||
|
font-size: 25px;
|
||||||
|
border: none;
|
||||||
|
border-radius: 5px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.App-link {
|
||||||
|
color: #282c34;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Search div */
|
||||||
|
.list-locations {
|
||||||
|
box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2), 0 6px 20px 0 rgba(0, 0, 0, 0.19);
|
||||||
|
border-radius: 3px;
|
||||||
|
background-color: #ffffff;
|
||||||
|
position: absolute;
|
||||||
|
left: 10px;
|
||||||
|
width: 300px;
|
||||||
|
height: calc(100%-10vmin);
|
||||||
|
top: calc(10px+10vmin);
|
||||||
|
border-bottom: 1px solid #d5d8df;
|
||||||
|
}
|
||||||
|
|
||||||
|
.priceFilterBlock{
|
||||||
|
margin: 5%;
|
||||||
|
}
|
||||||
|
|
||||||
|
.priceFilterValue{
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
/*Search Input*/
|
||||||
|
input.search-locations {
|
||||||
|
position: absolute;
|
||||||
|
top: 25px;
|
||||||
|
height: 25px;
|
||||||
|
width: 80%;
|
||||||
|
margin: 5%;
|
||||||
|
font-size: 15px;
|
||||||
|
left: inherit;
|
||||||
|
}
|
||||||
|
/* Locations list */
|
||||||
|
ol.location-list {
|
||||||
|
border-top: 1px solid #c5c8cf;
|
||||||
|
margin-top: 20%;
|
||||||
|
padding: 35px;
|
||||||
|
padding-bottom: 10px;
|
||||||
|
line-height: 2;
|
||||||
|
|
||||||
|
}
|
||||||
|
/* Location-button*/
|
||||||
|
button.location-list-item {
|
||||||
|
width: 100%;
|
||||||
|
text-align: initial;
|
||||||
|
font-size: medium;
|
||||||
|
border: none;
|
||||||
|
background: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
button.location-list-item:hover {
|
||||||
|
color: #23a6ca;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Location Details */
|
||||||
|
.location-details {
|
||||||
|
box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2), 0 6px 20px 0 rgba(0, 0, 0, 0.19);
|
||||||
|
border-radius: 3px;
|
||||||
|
background-color: #ffffff;
|
||||||
|
position: absolute;
|
||||||
|
width: 300px;
|
||||||
|
border-bottom: 1px solid #d5d8df;
|
||||||
|
font-size: medium;
|
||||||
|
text-align: initial;
|
||||||
|
box-sizing: border-box;
|
||||||
|
padding: 10px;
|
||||||
|
line-height: 5px;
|
||||||
|
|
||||||
|
}
|
||||||
|
/* Restaurant Image */
|
||||||
|
.featured-image {
|
||||||
|
margin-top: 15px;
|
||||||
|
font-size: small;
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (max-width: 450px) {
|
||||||
|
.list-locations {
|
||||||
|
width: 95%;
|
||||||
|
}
|
||||||
|
|
||||||
|
.location-details {
|
||||||
|
width: 100%;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (max-width: 500px) {
|
||||||
|
.App-header {
|
||||||
|
min-height: 15vh;
|
||||||
|
align-items: top;
|
||||||
|
}
|
||||||
|
.toggle-menu {
|
||||||
|
position: absolute;
|
||||||
|
height: 15vh;
|
||||||
|
left: 2px;
|
||||||
|
top: 2px;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
89
react_usse/src/App.js
Normal file
89
react_usse/src/App.js
Normal file
@ -0,0 +1,89 @@
|
|||||||
|
import React, { Component } from 'react';
|
||||||
|
import LeafletMap from './MapContainer';
|
||||||
|
import locations from './locations';
|
||||||
|
|
||||||
|
|
||||||
|
import { FaBars } from 'react-icons/fa';
|
||||||
|
import './App.css';
|
||||||
|
import escapeRegExp from 'escape-string-regexp';
|
||||||
|
import sortBy from 'sort-by';
|
||||||
|
|
||||||
|
|
||||||
|
class App extends Component {
|
||||||
|
state = {
|
||||||
|
allLocations: {},
|
||||||
|
query : '',
|
||||||
|
toggleMenu: false,
|
||||||
|
}
|
||||||
|
|
||||||
|
componentWillMount() {
|
||||||
|
this.setState({ allLocations : locations})
|
||||||
|
}
|
||||||
|
|
||||||
|
onToggleMenu = () => {
|
||||||
|
this.setState({ toggleMenu: !this.state.toggleMenu})
|
||||||
|
}
|
||||||
|
|
||||||
|
updateQuery = (query) => {
|
||||||
|
this.setState({ query: query.trim()});
|
||||||
|
}
|
||||||
|
|
||||||
|
updatePrice = (maxPrice) => {
|
||||||
|
this.setState({ price: maxPrice.trim()})
|
||||||
|
}
|
||||||
|
|
||||||
|
render() {
|
||||||
|
const { allLocations, query, price } = this.state
|
||||||
|
let showingLocations
|
||||||
|
if (query || price) {
|
||||||
|
if(query){
|
||||||
|
const match = new RegExp(escapeRegExp(query), 'i')
|
||||||
|
showingLocations = allLocations.filter((location) =>
|
||||||
|
match.test(location.name)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if(price){
|
||||||
|
const maxPrice = parseInt(price) * 1000
|
||||||
|
showingLocations = []
|
||||||
|
allLocations.map( house =>
|
||||||
|
{if(parseInt(house['price'].split(" ")[1] * 1000) <= maxPrice){
|
||||||
|
// showingLocations[house['name']] = house
|
||||||
|
showingLocations += house
|
||||||
|
}}
|
||||||
|
)
|
||||||
|
// console.log(showingLocations)
|
||||||
|
// const match = new RegExp(escapeRegExp(String((parseInt(price) * 1000))), 'i')
|
||||||
|
// showingLocations = allLocations.filter((location) =>
|
||||||
|
// match.test(location.price)
|
||||||
|
// )
|
||||||
|
}
|
||||||
|
|
||||||
|
} else {
|
||||||
|
showingLocations = allLocations
|
||||||
|
}
|
||||||
|
// showingLocations.sort(sortBy('name'));
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="App">
|
||||||
|
<header className="App-header">
|
||||||
|
<h1>Vind je droomhuis</h1>
|
||||||
|
<button className="toggle-menu" onClick={this.onToggleMenu}><FaBars/></button>
|
||||||
|
</header>
|
||||||
|
<LeafletMap toggleMenu={this.state.toggleMenu}
|
||||||
|
locations={showingLocations}
|
||||||
|
markers={showingLocations}
|
||||||
|
query={this.state.query}
|
||||||
|
onUpdateQuery={this.updateQuery}
|
||||||
|
onUpdatePrice={this.updatePrice}
|
||||||
|
onListItemClick={this.onListItemClick}
|
||||||
|
showDetails={this.state.showDetails}
|
||||||
|
selectedLocation={this.state.selectedLocation}
|
||||||
|
locationData={this.state.locationData}
|
||||||
|
maxPrice={this.state.price}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default App;
|
||||||
9
react_usse/src/App.test.js
Normal file
9
react_usse/src/App.test.js
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
import React from 'react';
|
||||||
|
import ReactDOM from 'react-dom';
|
||||||
|
import App from './App';
|
||||||
|
|
||||||
|
it('renders without crashing', () => {
|
||||||
|
const div = document.createElement('div');
|
||||||
|
ReactDOM.render(<App />, div);
|
||||||
|
ReactDOM.unmountComponentAtNode(div);
|
||||||
|
});
|
||||||
33
react_usse/src/LocationDetails.js
Normal file
33
react_usse/src/LocationDetails.js
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
import React, { Component } from 'react'
|
||||||
|
|
||||||
|
class LocationDetails extends Component {
|
||||||
|
render() {
|
||||||
|
return(
|
||||||
|
<div className="location-details">
|
||||||
|
<h4>{this.props.locationData.name}</h4>
|
||||||
|
<p>Cuisine: {this.props.locationData.cuisines}</p>
|
||||||
|
|
||||||
|
{this.props.locationData.average_cost_for_two ? (
|
||||||
|
<p> Average Cost for two : {this.props.locationData.average_cost_for_two}{this.props.locationData.currency}</p>
|
||||||
|
): ""}
|
||||||
|
|
||||||
|
{this.props.locationData.menu_url ? (
|
||||||
|
<a href={this.props.locationData.menu_url}> Checkout the Menu </a>
|
||||||
|
): ""}
|
||||||
|
|
||||||
|
{this.props.locationData.featured_image ? (
|
||||||
|
<div className="featured-image">
|
||||||
|
<img alt={this.props.locationData.name + 'food picture'}
|
||||||
|
src={this.props.locationData.featured_image} width="100%" height="150" />
|
||||||
|
<p font="small"> Image credit: Zomato</p>
|
||||||
|
</div>
|
||||||
|
): ""}
|
||||||
|
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export default LocationDetails;
|
||||||
100
react_usse/src/MapContainer.js
Normal file
100
react_usse/src/MapContainer.js
Normal file
@ -0,0 +1,100 @@
|
|||||||
|
import React, { Component } from 'react';
|
||||||
|
import { MapContainer, Marker, Popup, TileLayer } from "react-leaflet";
|
||||||
|
// import { MapContainer, Marker, Popup, TileLayer } from 'react-leaflet'
|
||||||
|
import LocationDetails from './LocationDetails';
|
||||||
|
|
||||||
|
class LeafletMap extends Component {
|
||||||
|
state = {
|
||||||
|
map: null,
|
||||||
|
showInfoWindow: false,
|
||||||
|
activeMarker: {},
|
||||||
|
activeMarkerProps: {},
|
||||||
|
showDetails: false,
|
||||||
|
selectedLocation: {},
|
||||||
|
locationData: {}
|
||||||
|
}
|
||||||
|
|
||||||
|
mapReady = (props, map) => {
|
||||||
|
this.setState({map})
|
||||||
|
}
|
||||||
|
|
||||||
|
onClickMarker = (props, marker, e) => {
|
||||||
|
this.setState({ showInfoWindow: true, activeMarker: marker, activeMarkerProps: props})
|
||||||
|
}
|
||||||
|
onInfoWindowClose = () =>
|
||||||
|
this.setState({ activeMarker: {}, showingInfoWindow: false, activeMarkerProps: {} });
|
||||||
|
|
||||||
|
onListItemClick = (location) => {
|
||||||
|
this.setState({ selectedLocation : location , showDetails : true})
|
||||||
|
let {name} = location;
|
||||||
|
let { lng, lat } = location.position;
|
||||||
|
}
|
||||||
|
|
||||||
|
render() {
|
||||||
|
let { markers} = this.props
|
||||||
|
let { maxPrice} = this.props
|
||||||
|
let { activeMarker, activeMarkerProps} = this.state;
|
||||||
|
const position = [51.505, -0.09]
|
||||||
|
return(
|
||||||
|
<div>
|
||||||
|
<div style={{ height: 'calc(100%-10vmin', width: '100%'}}>
|
||||||
|
<MapContainer center={position} zoom={13} scrollWheelZoom={false}>
|
||||||
|
<TileLayer
|
||||||
|
attribution='© <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors'
|
||||||
|
url="https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png"
|
||||||
|
/>
|
||||||
|
<Marker position={position}>
|
||||||
|
<Popup>
|
||||||
|
A pretty CSS3 popup. <br /> Easily customizable.
|
||||||
|
</Popup>
|
||||||
|
</Marker>
|
||||||
|
</MapContainer>
|
||||||
|
</div>
|
||||||
|
{this.props.toggleMenu && (
|
||||||
|
<div className="list-locations">
|
||||||
|
<input
|
||||||
|
className="search-locations"
|
||||||
|
type="text"
|
||||||
|
placeholder="Filter Locations..."
|
||||||
|
value={this.props.query}
|
||||||
|
onChange={(e) => {
|
||||||
|
this.props.onUpdateQuery(e.target.value)
|
||||||
|
this.setState({ showDetails: false })
|
||||||
|
}}
|
||||||
|
|
||||||
|
onClick={this.onNewSearch}/>
|
||||||
|
|
||||||
|
|
||||||
|
<div className='priceFilterBlock'>
|
||||||
|
<input type="range" min="100" max="400" onInput={(e) => {
|
||||||
|
this.props.onUpdatePrice(e.target.value)
|
||||||
|
}}/>
|
||||||
|
<p className='priceFilterValue'>{maxPrice}</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="location-list container">
|
||||||
|
<ol className="location-list">
|
||||||
|
{this.props.locations.map((location) => (
|
||||||
|
<button
|
||||||
|
key={location.name} className="location-list-item"
|
||||||
|
location={location}
|
||||||
|
onClick={() => this.onListItemClick(location)}>
|
||||||
|
{location.name}
|
||||||
|
</button>
|
||||||
|
|
||||||
|
))}
|
||||||
|
</ol>
|
||||||
|
</div>
|
||||||
|
{this.state.showDetails && (
|
||||||
|
<LocationDetails selectedLocation = {this.state.selectedLocation} locationData = {this.state.locationData}/>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
export default LeafletMap;
|
||||||
|
|
||||||
37
react_usse/src/MapNotLoaded.js
Normal file
37
react_usse/src/MapNotLoaded.js
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
import React, { Component } from 'react';
|
||||||
|
|
||||||
|
class MapNotLoaded extends Component {
|
||||||
|
state = {
|
||||||
|
show: false,
|
||||||
|
timeout : null
|
||||||
|
}
|
||||||
|
|
||||||
|
componentDidMount() {
|
||||||
|
let timeout = window.setTimeout(this.showMessage, 1000);
|
||||||
|
this.setState({ timeout });
|
||||||
|
}
|
||||||
|
|
||||||
|
componentWillUnmount() {
|
||||||
|
window.clearTimeout(this.state.timeout);
|
||||||
|
}
|
||||||
|
|
||||||
|
showMessage = () => {
|
||||||
|
this.setState({ show: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
render() {
|
||||||
|
return(
|
||||||
|
<div>
|
||||||
|
{this.state.show ? (
|
||||||
|
<div>
|
||||||
|
<h1>Error occured while loading the map</h1>
|
||||||
|
<p>Map could not load, please check network connectivity</p>
|
||||||
|
</div>
|
||||||
|
): (<div>Loading...</div>)
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default MapNotLoaded;
|
||||||
45
react_usse/src/_locations.json
Normal file
45
react_usse/src/_locations.json
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"name": "JoJo",
|
||||||
|
"address": "160 E 64th St, New York, NY 10065, USA",
|
||||||
|
"url": "http://www.jojorestaurantnyc.com/",
|
||||||
|
"position": {"lat": 52.232427,"lng": 5.211575}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Fig & Olive",
|
||||||
|
"address": "808 Lexington Ave, New York, NY 10065, USA",
|
||||||
|
"url": "http://www.figandolive.com/",
|
||||||
|
"position": {"lat": 40.7644156,"lng": -73.96910717}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "LAVO",
|
||||||
|
"address": "39 E 58th St, New York, NY 10022, USA",
|
||||||
|
"url": "http://www.lavony.com/",
|
||||||
|
"position": {"lat": 40.7629043,"lng": -73.973652417}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Benihana",
|
||||||
|
"address": "47 W 56th St, New York, NY 10019, USA",
|
||||||
|
"url": "https://www.benihana.com/locations/newyorkwest-ny-we/",
|
||||||
|
"position": {"lat": 40.7634566,"lng": -73.978973117}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Patsy",
|
||||||
|
"address": "236 W 56th St, New York, NY 10019, USA",
|
||||||
|
"url": "http://www.patsys.com/",
|
||||||
|
"position": {"lat": 40.7656201,"lng": -73.984939117}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Nougatine at Jean-Georges",
|
||||||
|
"address": "1 Central Park West, New York, NY 10023, USA",
|
||||||
|
"url": "http://www.jean-georgesrestaurant.com/",
|
||||||
|
"position": {"lat": 40.7690295,"lng": -73.983814217}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Landmarc",
|
||||||
|
"address": "10 Columbus Cir #3, New York, NY 10019, USA",
|
||||||
|
"url": "http://landmarc-restaurant.com/",
|
||||||
|
"position": {"lat": 40.7685566,"lng": -73.985375317}
|
||||||
|
}
|
||||||
|
|
||||||
|
]
|
||||||
14
react_usse/src/index.css
Normal file
14
react_usse/src/index.css
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
body {
|
||||||
|
margin: 0;
|
||||||
|
padding: 0;
|
||||||
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Roboto", "Oxygen",
|
||||||
|
"Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue",
|
||||||
|
sans-serif;
|
||||||
|
-webkit-font-smoothing: antialiased;
|
||||||
|
-moz-osx-font-smoothing: grayscale;
|
||||||
|
}
|
||||||
|
|
||||||
|
code {
|
||||||
|
font-family: source-code-pro, Menlo, Monaco, Consolas, "Courier New",
|
||||||
|
monospace;
|
||||||
|
}
|
||||||
12
react_usse/src/index.js
Normal file
12
react_usse/src/index.js
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
import React from 'react';
|
||||||
|
import ReactDOM from 'react-dom';
|
||||||
|
import './index.css';
|
||||||
|
import App from './App';
|
||||||
|
import * as serviceWorker from './serviceWorker';
|
||||||
|
|
||||||
|
ReactDOM.render(<App />, document.getElementById('root'));
|
||||||
|
|
||||||
|
// If you want your app to work offline and load faster, you can change
|
||||||
|
// unregister() to register() below. Note this comes with some pitfalls.
|
||||||
|
// Learn more about service workers: http://bit.ly/CRA-PWA
|
||||||
|
serviceWorker.register();
|
||||||
818354
react_usse/src/locations.json
Normal file
818354
react_usse/src/locations.json
Normal file
File diff suppressed because one or more lines are too long
135
react_usse/src/serviceWorker.js
Normal file
135
react_usse/src/serviceWorker.js
Normal file
@ -0,0 +1,135 @@
|
|||||||
|
// This optional code is used to register a service worker.
|
||||||
|
// register() is not called by default.
|
||||||
|
|
||||||
|
// This lets the app load faster on subsequent visits in production, and gives
|
||||||
|
// it offline capabilities. However, it also means that developers (and users)
|
||||||
|
// will only see deployed updates on subsequent visits to a page, after all the
|
||||||
|
// existing tabs open on the page have been closed, since previously cached
|
||||||
|
// resources are updated in the background.
|
||||||
|
|
||||||
|
// To learn more about the benefits of this model and instructions on how to
|
||||||
|
// opt-in, read http://bit.ly/CRA-PWA
|
||||||
|
|
||||||
|
const isLocalhost = Boolean(
|
||||||
|
window.location.hostname === 'localhost' ||
|
||||||
|
// [::1] is the IPv6 localhost address.
|
||||||
|
window.location.hostname === '[::1]' ||
|
||||||
|
// 127.0.0.1/8 is considered localhost for IPv4.
|
||||||
|
window.location.hostname.match(
|
||||||
|
/^127(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}$/
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
export function register(config) {
|
||||||
|
if (process.env.NODE_ENV === 'production' && 'serviceWorker' in navigator) {
|
||||||
|
// The URL constructor is available in all browsers that support SW.
|
||||||
|
const publicUrl = new URL(process.env.PUBLIC_URL, window.location.href);
|
||||||
|
if (publicUrl.origin !== window.location.origin) {
|
||||||
|
// Our service worker won't work if PUBLIC_URL is on a different origin
|
||||||
|
// from what our page is served on. This might happen if a CDN is used to
|
||||||
|
// serve assets; see https://github.com/facebook/create-react-app/issues/2374
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
window.addEventListener('load', () => {
|
||||||
|
const swUrl = `${process.env.PUBLIC_URL}/service-worker.js`;
|
||||||
|
|
||||||
|
if (isLocalhost) {
|
||||||
|
// This is running on localhost. Let's check if a service worker still exists or not.
|
||||||
|
checkValidServiceWorker(swUrl, config);
|
||||||
|
|
||||||
|
// Add some additional logging to localhost, pointing developers to the
|
||||||
|
// service worker/PWA documentation.
|
||||||
|
navigator.serviceWorker.ready.then(() => {
|
||||||
|
console.log(
|
||||||
|
'This web app is being served cache-first by a service ' +
|
||||||
|
'worker. To learn more, visit http://bit.ly/CRA-PWA'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
// Is not localhost. Just register service worker
|
||||||
|
registerValidSW(swUrl, config);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function registerValidSW(swUrl, config) {
|
||||||
|
navigator.serviceWorker
|
||||||
|
.register(swUrl)
|
||||||
|
.then(registration => {
|
||||||
|
registration.onupdatefound = () => {
|
||||||
|
const installingWorker = registration.installing;
|
||||||
|
if (installingWorker == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
installingWorker.onstatechange = () => {
|
||||||
|
if (installingWorker.state === 'installed') {
|
||||||
|
if (navigator.serviceWorker.controller) {
|
||||||
|
// At this point, the updated precached content has been fetched,
|
||||||
|
// but the previous service worker will still serve the older
|
||||||
|
// content until all client tabs are closed.
|
||||||
|
console.log(
|
||||||
|
'New content is available and will be used when all ' +
|
||||||
|
'tabs for this page are closed. See http://bit.ly/CRA-PWA.'
|
||||||
|
);
|
||||||
|
|
||||||
|
// Execute callback
|
||||||
|
if (config && config.onUpdate) {
|
||||||
|
config.onUpdate(registration);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// At this point, everything has been precached.
|
||||||
|
// It's the perfect time to display a
|
||||||
|
// "Content is cached for offline use." message.
|
||||||
|
console.log('Content is cached for offline use.');
|
||||||
|
|
||||||
|
// Execute callback
|
||||||
|
if (config && config.onSuccess) {
|
||||||
|
config.onSuccess(registration);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.catch(error => {
|
||||||
|
console.error('Error during service worker registration:', error);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function checkValidServiceWorker(swUrl, config) {
|
||||||
|
// Check if the service worker can be found. If it can't reload the page.
|
||||||
|
fetch(swUrl)
|
||||||
|
.then(response => {
|
||||||
|
// Ensure service worker exists, and that we really are getting a JS file.
|
||||||
|
const contentType = response.headers.get('content-type');
|
||||||
|
if (
|
||||||
|
response.status === 404 ||
|
||||||
|
(contentType != null && contentType.indexOf('javascript') === -1)
|
||||||
|
) {
|
||||||
|
// No service worker found. Probably a different app. Reload the page.
|
||||||
|
navigator.serviceWorker.ready.then(registration => {
|
||||||
|
registration.unregister().then(() => {
|
||||||
|
window.location.reload();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
// Service worker found. Proceed as normal.
|
||||||
|
registerValidSW(swUrl, config);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch(() => {
|
||||||
|
console.log(
|
||||||
|
'No internet connection found. App is running in offline mode.'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function unregister() {
|
||||||
|
if ('serviceWorker' in navigator) {
|
||||||
|
navigator.serviceWorker.ready.then(registration => {
|
||||||
|
registration.unregister();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user