EljakimHerrewijnen/usse
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Eljakim 2024-04-15 19:31:08 +00:00
parent c4f8b5444c
commit 56377e8dc5
4 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
dist/ dist/
ghidra_assistant.egg-info/ ghidra_assistant.egg-info/
out.json out.json
*.pyc

10
automate.sh
View File

@ -1,14 +1,16 @@
#! /bin/bash #! /bin/bash
CURR_DIR=$(pwd) CURR_DIR=$(pwd)
SCRAPE_DIR=${CURR_DIR}/scrape APP_DIR=/mnt/wintergreen_ssd1/projects/usse/usse
REACT_DIR=${CURR_DIR}/react_usse SCRAPE_DIR=${APP_DIR}/scrape
REACT_DIR=${APP_DIR}/react_usse
# Move panda_dump.bin # Move panda_dump.bin
mv ${SCRAPE_DIR}/panda_dump.bin ${SCRAPE_DIR}/_panda_dump.bin mv ${SCRAPE_DIR}/panda_dump.bin ${SCRAPE_DIR}/_panda_dump.bin
cd ${SCRAPE_DIR}
source ${SCRAPE_DIR}/venv/bin/activate && python ${SCRAPE_DIR}/usse.py && deactivate source ${SCRAPE_DIR}/venv/bin/activate && python ${SCRAPE_DIR}/usse.py && deactivate
cp ${SCRAPE_DIR}/out.json ${REACT_DIR}/src/locations.json cp ${SCRAPE_DIR}/out.json ${REACT_DIR}/src/locations.json
# Deploy new container # Deploy new container
cd ${REACT_DIR} cd ${REACT_DIR}
./use_docker.sh recreate ./use_docker.sh rebuild

3
react_usse/Dockerfile
View File

@ -7,8 +7,9 @@ WORKDIR /usr/src/app
# Step 3: Copy package.json and package-lock.json (or yarn.lock) # Step 3: Copy package.json and package-lock.json (or yarn.lock)
COPY package*.json ./ COPY package*.json ./
RUN npm config set fetch-retry-maxtimeout 6000000 && npm config set fetch-retry-mintimeout 1000000
# Step 4: Install dependencies # Step 4: Install dependencies
RUN npm install RUN npm install --no-audit
# Step 5: Copy the rest of your app's source code # Step 5: Copy the rest of your app's source code
COPY . . COPY . .

4
scrape/usse.py
View File

@ -24,7 +24,7 @@ out = []
# URL = "https://www.funda.nl/zoeken/koop?selected_area=%5B%22utrecht,15km%22%5D&price=%22-400000%22&object_type=%5B%22house%22%5D" # URL = "https://www.funda.nl/zoeken/koop?selected_area=%5B%22utrecht,15km%22%5D&price=%22-400000%22&object_type=%5B%22house%22%5D"
URL = "https://www.funda.nl/zoeken/koop?selected_area=%5B%22utrecht,30km%22%5D&price=%22-500000%22&object_type=%5B%22house%22%5D" URL = "https://www.funda.nl/zoeken/koop?selected_area=%5B%22utrecht,30km%22%5D&price=%22-500000%22&object_type=%5B%22house%22%5D"
NUM_PAGES = 1 NUM_PAGES = 150
def get_funda_data(): def get_funda_data():
# scraper = FundaScraper(url="nijkerk/beschikbaar/100000-400000/woonhuis/tuin/eengezinswoning/landhuis/+30km/", find_past=False, n_pages=81) # scraper = FundaScraper(url="nijkerk/beschikbaar/100000-400000/woonhuis/tuin/eengezinswoning/landhuis/+30km/", find_past=False, n_pages=81)
scraper = FundaScraper(url=URL, find_past=False, n_pages=NUM_PAGES) scraper = FundaScraper(url=URL, find_past=False, n_pages=NUM_PAGES)
@ -101,4 +101,4 @@ if __name__ == "__main__":
else: else:
data = get_funda_data() data = get_funda_data()
pickle.dump(data, open('panda_dump.bin', 'wb')) pickle.dump(data, open('panda_dump.bin', 'wb'))
generate_json(data) generate_json(data)