feat: add noVNC headed mode for interactive browser sessions
- Dockerfile with Xvfb + x11vnc + noVNC for browser-in-a-URL - search.py auto-detects DISPLAY env to run headed vs headless - pause_for_human() function for CAPTCHA/login handoff - Facebook Marketplace search (headed mode only, needs login) - Updated README with both modes
This commit is contained in:
+29
@@ -0,0 +1,29 @@
|
|||||||
|
FROM python:3.13-slim
|
||||||
|
|
||||||
|
# Install display and VNC deps
|
||||||
|
RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \
|
||||||
|
xvfb x11vnc websockify novnc \
|
||||||
|
libnss3 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 \
|
||||||
|
libxkbcommon0 libxcomposite1 libxdamage1 libxrandr2 libgbm1 \
|
||||||
|
libpango-1.0-0 libcairo2 libasound2 libxshmfence1 libgtk-3-0 \
|
||||||
|
fonts-liberation && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Install Python deps
|
||||||
|
COPY pyproject.toml uv.lock /app/
|
||||||
|
WORKDIR /app
|
||||||
|
RUN pip install uv && uv sync --frozen
|
||||||
|
|
||||||
|
# Install Playwright Chromium
|
||||||
|
RUN uv run playwright install chromium
|
||||||
|
|
||||||
|
# Copy source
|
||||||
|
COPY . /app/
|
||||||
|
|
||||||
|
# Entrypoint script
|
||||||
|
COPY entrypoint.sh /entrypoint.sh
|
||||||
|
RUN chmod +x /entrypoint.sh
|
||||||
|
|
||||||
|
EXPOSE 6080
|
||||||
|
|
||||||
|
ENTRYPOINT ["/entrypoint.sh"]
|
||||||
@@ -2,15 +2,41 @@
|
|||||||
|
|
||||||
Used car search tool for finding Jude a reliable hybrid (HEV) in the Woodinville, WA area. Budget: $8-10K.
|
Used car search tool for finding Jude a reliable hybrid (HEV) in the Woodinville, WA area. Budget: $8-10K.
|
||||||
|
|
||||||
## Setup
|
## Two Modes
|
||||||
|
|
||||||
|
### Headless (fast, no interaction)
|
||||||
|
|
||||||
|
Runs Chromium headless. Good for sites that don't need login. Skips Facebook Marketplace.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv sync
|
uv sync
|
||||||
uv run playwright install chromium
|
uv run playwright install chromium
|
||||||
```
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
```bash
|
|
||||||
uv run python search.py
|
uv run python search.py
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Headed via noVNC (interactive, for logins/CAPTCHAs)
|
||||||
|
|
||||||
|
Runs Chromium inside Docker with a virtual display. Open the browser view in any web browser when you need to interact (login, solve CAPTCHAs, apply filters manually).
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Build once
|
||||||
|
docker build -t car-help .
|
||||||
|
|
||||||
|
# Run (browser visible at http://localhost:6080/vnc.html)
|
||||||
|
docker run -d --name car-help -p 6080:6080 -v $(pwd)/results:/app/results car-help
|
||||||
|
|
||||||
|
# Run a search (headed -- includes Facebook Marketplace)
|
||||||
|
docker exec -it car-help uv run python search.py
|
||||||
|
|
||||||
|
# View the browser live
|
||||||
|
open http://localhost:6080/vnc.html
|
||||||
|
|
||||||
|
# Stop
|
||||||
|
docker stop car-help && docker rm car-help
|
||||||
|
```
|
||||||
|
|
||||||
|
Results are saved to `results/search_<timestamp>.json`.
|
||||||
|
|
||||||
|
## Sites
|
||||||
|
|
||||||
|
See `sites.yaml` for the full list of car search sites, search parameters, and target models.
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Start virtual display
|
||||||
|
export DISPLAY=:99
|
||||||
|
Xvfb :99 -screen 0 1280x720x24 -ac &
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
# Start VNC server on the virtual display (no password, listen on 5900)
|
||||||
|
x11vnc -display :99 -forever -nopw -listen 0.0.0.0 -rfbport 5900 -shared &
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
# Start noVNC web proxy (serves browser view at http://localhost:6080/vnc.html)
|
||||||
|
websockify --web=/usr/share/novnc/ 6080 localhost:5900 &
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
echo "============================================="
|
||||||
|
echo " noVNC ready: http://localhost:6080/vnc.html"
|
||||||
|
echo "============================================="
|
||||||
|
|
||||||
|
# Run whatever command was passed (or drop to shell)
|
||||||
|
if [ $# -eq 0 ]; then
|
||||||
|
echo "No command given. Container is running -- attach to use."
|
||||||
|
echo "Run searches with: docker exec car-help uv run python search.py"
|
||||||
|
tail -f /dev/null
|
||||||
|
else
|
||||||
|
exec "$@"
|
||||||
|
fi
|
||||||
@@ -1,10 +1,20 @@
|
|||||||
"""
|
"""
|
||||||
Car search crawler -- finds used hybrids under $10K near Woodinville WA.
|
Car search crawler -- finds used hybrids under $10K near Woodinville WA.
|
||||||
Uses Playwright (headless Chromium) to handle JS-heavy car sites.
|
Uses Playwright (headless Chromium) to handle JS-heavy car sites.
|
||||||
|
|
||||||
|
Two modes:
|
||||||
|
headless (default): uv run python search.py
|
||||||
|
headed via noVNC: docker run ... (see README)
|
||||||
|
|
||||||
|
When running in the Docker container, the browser is visible at
|
||||||
|
http://localhost:6080/vnc.html -- use this to handle CAPTCHAs,
|
||||||
|
log into Facebook Marketplace, or visually inspect results.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -13,6 +23,26 @@ from playwright.async_api import async_playwright
|
|||||||
RESULTS_DIR = Path("results")
|
RESULTS_DIR = Path("results")
|
||||||
RESULTS_DIR.mkdir(exist_ok=True)
|
RESULTS_DIR.mkdir(exist_ok=True)
|
||||||
|
|
||||||
|
# If DISPLAY is set (noVNC container), run headed
|
||||||
|
HEADED = bool(os.environ.get("DISPLAY"))
|
||||||
|
|
||||||
|
|
||||||
|
async def pause_for_human(page, reason: str):
|
||||||
|
"""Pause and prompt for human interaction via noVNC."""
|
||||||
|
if not HEADED:
|
||||||
|
print(f" SKIP (headless): {reason}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
print(f"\n{'=' * 60}")
|
||||||
|
print(f" HUMAN NEEDED: {reason}")
|
||||||
|
print(f" Open http://localhost:6080/vnc.html to interact")
|
||||||
|
print(f" Press Enter here when done...")
|
||||||
|
print(f"{'=' * 60}")
|
||||||
|
|
||||||
|
# Wait for user to press Enter in the terminal
|
||||||
|
await asyncio.get_event_loop().run_in_executor(None, input)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
async def search_craigslist(page) -> list[dict]:
|
async def search_craigslist(page) -> list[dict]:
|
||||||
"""Craigslist Seattle -- simple HTML, most reliable to scrape."""
|
"""Craigslist Seattle -- simple HTML, most reliable to scrape."""
|
||||||
@@ -24,7 +54,7 @@ async def search_craigslist(page) -> list[dict]:
|
|||||||
"&search_distance=50"
|
"&search_distance=50"
|
||||||
"&sort=date"
|
"&sort=date"
|
||||||
)
|
)
|
||||||
print(f" Searching Craigslist: {url}")
|
print(f" Searching Craigslist...")
|
||||||
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
||||||
await page.wait_for_timeout(2000)
|
await page.wait_for_timeout(2000)
|
||||||
|
|
||||||
@@ -96,6 +126,44 @@ async def search_cargurus(page) -> list[dict]:
|
|||||||
return listings
|
return listings
|
||||||
|
|
||||||
|
|
||||||
|
async def search_facebook(page) -> list[dict]:
|
||||||
|
"""Facebook Marketplace -- needs login (human via noVNC)."""
|
||||||
|
if not HEADED:
|
||||||
|
print(" SKIP Facebook Marketplace (requires login -- run in Docker with noVNC)")
|
||||||
|
return []
|
||||||
|
|
||||||
|
url = "https://www.facebook.com/marketplace/seattle/vehicles/?maxPrice=10000"
|
||||||
|
print(f" Opening Facebook Marketplace...")
|
||||||
|
await page.goto(url, wait_until="networkidle", timeout=60000)
|
||||||
|
|
||||||
|
# Check if login is needed
|
||||||
|
if "login" in page.url.lower():
|
||||||
|
await pause_for_human(page, "Log into Facebook, then press Enter")
|
||||||
|
|
||||||
|
await page.wait_for_timeout(3000)
|
||||||
|
await pause_for_human(page, "Apply filters (hybrid, $10K max, 50mi) and press Enter when ready")
|
||||||
|
|
||||||
|
# Scrape whatever is visible
|
||||||
|
listings = []
|
||||||
|
cards = await page.query_selector_all("[class*='marketplace']")
|
||||||
|
for card in cards[:30]:
|
||||||
|
try:
|
||||||
|
text = await card.inner_text()
|
||||||
|
link_el = await card.query_selector("a[href*='/marketplace/']")
|
||||||
|
link = await link_el.get_attribute("href") if link_el else ""
|
||||||
|
listings.append({
|
||||||
|
"source": "facebook",
|
||||||
|
"title": text[:100].strip(),
|
||||||
|
"price": "see listing",
|
||||||
|
"url": f"https://www.facebook.com{link}" if link else "",
|
||||||
|
})
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
|
||||||
|
print(f" Found {len(listings)} Facebook listings")
|
||||||
|
return listings
|
||||||
|
|
||||||
|
|
||||||
async def search_autotempest(page) -> list[dict]:
|
async def search_autotempest(page) -> list[dict]:
|
||||||
"""AutoTempest -- meta-aggregator, searches multiple sites."""
|
"""AutoTempest -- meta-aggregator, searches multiple sites."""
|
||||||
url = (
|
url = (
|
||||||
@@ -132,7 +200,8 @@ async def search_autotempest(page) -> list[dict]:
|
|||||||
|
|
||||||
|
|
||||||
async def main():
|
async def main():
|
||||||
print(f"Car Search -- {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
|
mode = "HEADED (noVNC)" if HEADED else "HEADLESS"
|
||||||
|
print(f"Car Search [{mode}] -- {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
|
||||||
print(
|
print(
|
||||||
"Looking for: Used hybrids under $10K "
|
"Looking for: Used hybrids under $10K "
|
||||||
"within 50 miles of 98077 (Woodinville WA)"
|
"within 50 miles of 98077 (Woodinville WA)"
|
||||||
@@ -142,7 +211,7 @@ async def main():
|
|||||||
all_listings: list[dict] = []
|
all_listings: list[dict] = []
|
||||||
|
|
||||||
async with async_playwright() as p:
|
async with async_playwright() as p:
|
||||||
browser = await p.chromium.launch(headless=True)
|
browser = await p.chromium.launch(headless=not HEADED)
|
||||||
context = await browser.new_context(
|
context = await browser.new_context(
|
||||||
user_agent=(
|
user_agent=(
|
||||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||||
@@ -152,12 +221,16 @@ async def main():
|
|||||||
)
|
)
|
||||||
page = await context.new_page()
|
page = await context.new_page()
|
||||||
|
|
||||||
for search_fn in [search_craigslist, search_cargurus, search_autotempest]:
|
searches = [search_craigslist, search_cargurus, search_autotempest]
|
||||||
|
if HEADED:
|
||||||
|
searches.append(search_facebook)
|
||||||
|
|
||||||
|
for search_fn in searches:
|
||||||
try:
|
try:
|
||||||
results = await search_fn(page)
|
results = await search_fn(page)
|
||||||
all_listings.extend(results)
|
all_listings.extend(results)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f" ERROR: {e}")
|
print(f" ERROR in {search_fn.__name__}: {e}")
|
||||||
|
|
||||||
await browser.close()
|
await browser.close()
|
||||||
|
|
||||||
@@ -168,6 +241,7 @@ async def main():
|
|||||||
json.dump(
|
json.dump(
|
||||||
{
|
{
|
||||||
"search_date": datetime.now(timezone.utc).isoformat(),
|
"search_date": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"mode": mode,
|
||||||
"params": {
|
"params": {
|
||||||
"zip": "98077",
|
"zip": "98077",
|
||||||
"max_price": 10000,
|
"max_price": 10000,
|
||||||
|
|||||||
Reference in New Issue
Block a user