feat: add noVNC headed mode for interactive browser sessions

- Dockerfile with Xvfb + x11vnc + noVNC for browser-in-a-URL
- search.py auto-detects DISPLAY env to run headed vs headless
- pause_for_human() function for CAPTCHA/login handoff
- Facebook Marketplace search (headed mode only, needs login)
- Updated README with both modes
This commit is contained in:
Ken
2026-05-25 06:56:21 +00:00
parent 290678635e
commit f09317f95b
4 changed files with 168 additions and 11 deletions
+29
View File
@@ -0,0 +1,29 @@
FROM python:3.13-slim
# Install display and VNC deps
RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \
xvfb x11vnc websockify novnc \
libnss3 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 \
libxkbcommon0 libxcomposite1 libxdamage1 libxrandr2 libgbm1 \
libpango-1.0-0 libcairo2 libasound2 libxshmfence1 libgtk-3-0 \
fonts-liberation && \
rm -rf /var/lib/apt/lists/*
# Install Python deps
COPY pyproject.toml uv.lock /app/
WORKDIR /app
RUN pip install uv && uv sync --frozen
# Install Playwright Chromium
RUN uv run playwright install chromium
# Copy source
COPY . /app/
# Entrypoint script
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
EXPOSE 6080
ENTRYPOINT ["/entrypoint.sh"]
+32 -6
View File
@@ -2,15 +2,41 @@
Used car search tool for finding Jude a reliable hybrid (HEV) in the Woodinville, WA area. Budget: $8-10K. Used car search tool for finding Jude a reliable hybrid (HEV) in the Woodinville, WA area. Budget: $8-10K.
## Setup ## Two Modes
### Headless (fast, no interaction)
Runs Chromium headless. Good for sites that don't need login. Skips Facebook Marketplace.
```bash ```bash
uv sync uv sync
uv run playwright install chromium uv run playwright install chromium
```
## Usage
```bash
uv run python search.py uv run python search.py
``` ```
### Headed via noVNC (interactive, for logins/CAPTCHAs)
Runs Chromium inside Docker with a virtual display. Open the browser view in any web browser when you need to interact (login, solve CAPTCHAs, apply filters manually).
```bash
# Build once
docker build -t car-help .
# Run (browser visible at http://localhost:6080/vnc.html)
docker run -d --name car-help -p 6080:6080 -v $(pwd)/results:/app/results car-help
# Run a search (headed -- includes Facebook Marketplace)
docker exec -it car-help uv run python search.py
# View the browser live
open http://localhost:6080/vnc.html
# Stop
docker stop car-help && docker rm car-help
```
Results are saved to `results/search_<timestamp>.json`.
## Sites
See `sites.yaml` for the full list of car search sites, search parameters, and target models.
+28
View File
@@ -0,0 +1,28 @@
#!/bin/bash
set -e
# Start virtual display
export DISPLAY=:99
Xvfb :99 -screen 0 1280x720x24 -ac &
sleep 1
# Start VNC server on the virtual display (no password, listen on 5900)
x11vnc -display :99 -forever -nopw -listen 0.0.0.0 -rfbport 5900 -shared &
sleep 1
# Start noVNC web proxy (serves browser view at http://localhost:6080/vnc.html)
websockify --web=/usr/share/novnc/ 6080 localhost:5900 &
sleep 1
echo "============================================="
echo " noVNC ready: http://localhost:6080/vnc.html"
echo "============================================="
# Run whatever command was passed (or drop to shell)
if [ $# -eq 0 ]; then
echo "No command given. Container is running -- attach to use."
echo "Run searches with: docker exec car-help uv run python search.py"
tail -f /dev/null
else
exec "$@"
fi
+79 -5
View File
@@ -1,10 +1,20 @@
""" """
Car search crawler -- finds used hybrids under $10K near Woodinville WA. Car search crawler -- finds used hybrids under $10K near Woodinville WA.
Uses Playwright (headless Chromium) to handle JS-heavy car sites. Uses Playwright (headless Chromium) to handle JS-heavy car sites.
Two modes:
headless (default): uv run python search.py
headed via noVNC: docker run ... (see README)
When running in the Docker container, the browser is visible at
http://localhost:6080/vnc.html -- use this to handle CAPTCHAs,
log into Facebook Marketplace, or visually inspect results.
""" """
import asyncio import asyncio
import json import json
import os
import sys
from datetime import datetime, timezone from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
@@ -13,6 +23,26 @@ from playwright.async_api import async_playwright
RESULTS_DIR = Path("results") RESULTS_DIR = Path("results")
RESULTS_DIR.mkdir(exist_ok=True) RESULTS_DIR.mkdir(exist_ok=True)
# If DISPLAY is set (noVNC container), run headed
HEADED = bool(os.environ.get("DISPLAY"))
async def pause_for_human(page, reason: str):
"""Pause and prompt for human interaction via noVNC."""
if not HEADED:
print(f" SKIP (headless): {reason}")
return False
print(f"\n{'=' * 60}")
print(f" HUMAN NEEDED: {reason}")
print(f" Open http://localhost:6080/vnc.html to interact")
print(f" Press Enter here when done...")
print(f"{'=' * 60}")
# Wait for user to press Enter in the terminal
await asyncio.get_event_loop().run_in_executor(None, input)
return True
async def search_craigslist(page) -> list[dict]: async def search_craigslist(page) -> list[dict]:
"""Craigslist Seattle -- simple HTML, most reliable to scrape.""" """Craigslist Seattle -- simple HTML, most reliable to scrape."""
@@ -24,7 +54,7 @@ async def search_craigslist(page) -> list[dict]:
"&search_distance=50" "&search_distance=50"
"&sort=date" "&sort=date"
) )
print(f" Searching Craigslist: {url}") print(f" Searching Craigslist...")
await page.goto(url, wait_until="domcontentloaded", timeout=30000) await page.goto(url, wait_until="domcontentloaded", timeout=30000)
await page.wait_for_timeout(2000) await page.wait_for_timeout(2000)
@@ -96,6 +126,44 @@ async def search_cargurus(page) -> list[dict]:
return listings return listings
async def search_facebook(page) -> list[dict]:
"""Facebook Marketplace -- needs login (human via noVNC)."""
if not HEADED:
print(" SKIP Facebook Marketplace (requires login -- run in Docker with noVNC)")
return []
url = "https://www.facebook.com/marketplace/seattle/vehicles/?maxPrice=10000"
print(f" Opening Facebook Marketplace...")
await page.goto(url, wait_until="networkidle", timeout=60000)
# Check if login is needed
if "login" in page.url.lower():
await pause_for_human(page, "Log into Facebook, then press Enter")
await page.wait_for_timeout(3000)
await pause_for_human(page, "Apply filters (hybrid, $10K max, 50mi) and press Enter when ready")
# Scrape whatever is visible
listings = []
cards = await page.query_selector_all("[class*='marketplace']")
for card in cards[:30]:
try:
text = await card.inner_text()
link_el = await card.query_selector("a[href*='/marketplace/']")
link = await link_el.get_attribute("href") if link_el else ""
listings.append({
"source": "facebook",
"title": text[:100].strip(),
"price": "see listing",
"url": f"https://www.facebook.com{link}" if link else "",
})
except Exception:
continue
print(f" Found {len(listings)} Facebook listings")
return listings
async def search_autotempest(page) -> list[dict]: async def search_autotempest(page) -> list[dict]:
"""AutoTempest -- meta-aggregator, searches multiple sites.""" """AutoTempest -- meta-aggregator, searches multiple sites."""
url = ( url = (
@@ -132,7 +200,8 @@ async def search_autotempest(page) -> list[dict]:
async def main(): async def main():
print(f"Car Search -- {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}") mode = "HEADED (noVNC)" if HEADED else "HEADLESS"
print(f"Car Search [{mode}] -- {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
print( print(
"Looking for: Used hybrids under $10K " "Looking for: Used hybrids under $10K "
"within 50 miles of 98077 (Woodinville WA)" "within 50 miles of 98077 (Woodinville WA)"
@@ -142,7 +211,7 @@ async def main():
all_listings: list[dict] = [] all_listings: list[dict] = []
async with async_playwright() as p: async with async_playwright() as p:
browser = await p.chromium.launch(headless=True) browser = await p.chromium.launch(headless=not HEADED)
context = await browser.new_context( context = await browser.new_context(
user_agent=( user_agent=(
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
@@ -152,12 +221,16 @@ async def main():
) )
page = await context.new_page() page = await context.new_page()
for search_fn in [search_craigslist, search_cargurus, search_autotempest]: searches = [search_craigslist, search_cargurus, search_autotempest]
if HEADED:
searches.append(search_facebook)
for search_fn in searches:
try: try:
results = await search_fn(page) results = await search_fn(page)
all_listings.extend(results) all_listings.extend(results)
except Exception as e: except Exception as e:
print(f" ERROR: {e}") print(f" ERROR in {search_fn.__name__}: {e}")
await browser.close() await browser.close()
@@ -168,6 +241,7 @@ async def main():
json.dump( json.dump(
{ {
"search_date": datetime.now(timezone.utc).isoformat(), "search_date": datetime.now(timezone.utc).isoformat(),
"mode": mode,
"params": { "params": {
"zip": "98077", "zip": "98077",
"max_price": 10000, "max_price": 10000,