feat: add noVNC headed mode for interactive browser sessions

- Dockerfile with Xvfb + x11vnc + noVNC for browser-in-a-URL - search.py auto-detects DISPLAY env to run headed vs headless - pause_for_human() function for CAPTCHA/login handoff - Facebook Marketplace search (headed mode only, needs login) - Updated README with both modes
2026-05-25 06:56:21 +00:00
parent 290678635e
commit f09317f95b
4 changed files with 168 additions and 11 deletions
@@ -0,0 +1,29 @@
+FROM python:3.13-slim
+
+# Install display and VNC deps
+RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \
+    xvfb x11vnc websockify novnc \
+    libnss3 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 \
+    libxkbcommon0 libxcomposite1 libxdamage1 libxrandr2 libgbm1 \
+    libpango-1.0-0 libcairo2 libasound2 libxshmfence1 libgtk-3-0 \
+    fonts-liberation && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install Python deps
+COPY pyproject.toml uv.lock /app/
+WORKDIR /app
+RUN pip install uv && uv sync --frozen
+
+# Install Playwright Chromium
+RUN uv run playwright install chromium
+
+# Copy source
+COPY . /app/
+
+# Entrypoint script
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+EXPOSE 6080
+
+ENTRYPOINT ["/entrypoint.sh"]
@@ -2,15 +2,41 @@

 Used car search tool for finding Jude a reliable hybrid (HEV) in the Woodinville, WA area. Budget: $8-10K.

-## Setup
+## Two Modes
+
+### Headless (fast, no interaction)
+
+Runs Chromium headless. Good for sites that don't need login. Skips Facebook Marketplace.

 ```bash
 uv sync
 uv run playwright install chromium
-```
-
-## Usage
-
-```bash
 uv run python search.py
 ```
+
+### Headed via noVNC (interactive, for logins/CAPTCHAs)
+
+Runs Chromium inside Docker with a virtual display. Open the browser view in any web browser when you need to interact (login, solve CAPTCHAs, apply filters manually).
+
+```bash
+# Build once
+docker build -t car-help .
+
+# Run (browser visible at http://localhost:6080/vnc.html)
+docker run -d --name car-help -p 6080:6080 -v $(pwd)/results:/app/results car-help
+
+# Run a search (headed -- includes Facebook Marketplace)
+docker exec -it car-help uv run python search.py
+
+# View the browser live
+open http://localhost:6080/vnc.html
+
+# Stop
+docker stop car-help && docker rm car-help
+```
+
+Results are saved to `results/search_<timestamp>.json`.
+
+## Sites
+
+See `sites.yaml` for the full list of car search sites, search parameters, and target models.
@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+# Start virtual display
+export DISPLAY=:99
+Xvfb :99 -screen 0 1280x720x24 -ac &
+sleep 1
+
+# Start VNC server on the virtual display (no password, listen on 5900)
+x11vnc -display :99 -forever -nopw -listen 0.0.0.0 -rfbport 5900 -shared &
+sleep 1
+
+# Start noVNC web proxy (serves browser view at http://localhost:6080/vnc.html)
+websockify --web=/usr/share/novnc/ 6080 localhost:5900 &
+sleep 1
+
+echo "============================================="
+echo "  noVNC ready: http://localhost:6080/vnc.html"
+echo "============================================="
+
+# Run whatever command was passed (or drop to shell)
+if [ $# -eq 0 ]; then
+    echo "No command given. Container is running -- attach to use."
+    echo "Run searches with: docker exec car-help uv run python search.py"
+    tail -f /dev/null
+else
+    exec "$@"
+fi
@@ -1,10 +1,20 @@
 """
 Car search crawler -- finds used hybrids under $10K near Woodinville WA.
 Uses Playwright (headless Chromium) to handle JS-heavy car sites.
+
+Two modes:
+  headless (default):  uv run python search.py
+  headed via noVNC:    docker run ... (see README)
+
+When running in the Docker container, the browser is visible at
+http://localhost:6080/vnc.html -- use this to handle CAPTCHAs,
+log into Facebook Marketplace, or visually inspect results.
 """

 import asyncio
 import json
+import os
+import sys
 from datetime import datetime, timezone
 from pathlib import Path

@@ -13,6 +23,26 @@ from playwright.async_api import async_playwright
 RESULTS_DIR = Path("results")
 RESULTS_DIR.mkdir(exist_ok=True)

+# If DISPLAY is set (noVNC container), run headed
+HEADED = bool(os.environ.get("DISPLAY"))
+
+
+async def pause_for_human(page, reason: str):
+    """Pause and prompt for human interaction via noVNC."""
+    if not HEADED:
+        print(f"  SKIP (headless): {reason}")
+        return False
+
+    print(f"\n{'=' * 60}")
+    print(f"  HUMAN NEEDED: {reason}")
+    print(f"  Open http://localhost:6080/vnc.html to interact")
+    print(f"  Press Enter here when done...")
+    print(f"{'=' * 60}")
+
+    # Wait for user to press Enter in the terminal
+    await asyncio.get_event_loop().run_in_executor(None, input)
+    return True
+

 async def search_craigslist(page) -> list[dict]:
    """Craigslist Seattle -- simple HTML, most reliable to scrape."""
@@ -24,7 +54,7 @@ async def search_craigslist(page) -> list[dict]:
        "&search_distance=50"
        "&sort=date"
    )
-    print(f"  Searching Craigslist: {url}")
+    print(f"  Searching Craigslist...")
    await page.goto(url, wait_until="domcontentloaded", timeout=30000)
    await page.wait_for_timeout(2000)

@@ -96,6 +126,44 @@ async def search_cargurus(page) -> list[dict]:
    return listings


+async def search_facebook(page) -> list[dict]:
+    """Facebook Marketplace -- needs login (human via noVNC)."""
+    if not HEADED:
+        print("  SKIP Facebook Marketplace (requires login -- run in Docker with noVNC)")
+        return []
+
+    url = "https://www.facebook.com/marketplace/seattle/vehicles/?maxPrice=10000"
+    print(f"  Opening Facebook Marketplace...")
+    await page.goto(url, wait_until="networkidle", timeout=60000)
+
+    # Check if login is needed
+    if "login" in page.url.lower():
+        await pause_for_human(page, "Log into Facebook, then press Enter")
+
+    await page.wait_for_timeout(3000)
+    await pause_for_human(page, "Apply filters (hybrid, $10K max, 50mi) and press Enter when ready")
+
+    # Scrape whatever is visible
+    listings = []
+    cards = await page.query_selector_all("[class*='marketplace']")
+    for card in cards[:30]:
+        try:
+            text = await card.inner_text()
+            link_el = await card.query_selector("a[href*='/marketplace/']")
+            link = await link_el.get_attribute("href") if link_el else ""
+            listings.append({
+                "source": "facebook",
+                "title": text[:100].strip(),
+                "price": "see listing",
+                "url": f"https://www.facebook.com{link}" if link else "",
+            })
+        except Exception:
+            continue
+
+    print(f"  Found {len(listings)} Facebook listings")
+    return listings
+
+
 async def search_autotempest(page) -> list[dict]:
    """AutoTempest -- meta-aggregator, searches multiple sites."""
    url = (
@@ -132,7 +200,8 @@ async def search_autotempest(page) -> list[dict]:


 async def main():
-    print(f"Car Search -- {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
+    mode = "HEADED (noVNC)" if HEADED else "HEADLESS"
+    print(f"Car Search [{mode}] -- {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
    print(
        "Looking for: Used hybrids under $10K "
        "within 50 miles of 98077 (Woodinville WA)"
@@ -142,7 +211,7 @@ async def main():
    all_listings: list[dict] = []

    async with async_playwright() as p:
-        browser = await p.chromium.launch(headless=True)
+        browser = await p.chromium.launch(headless=not HEADED)
        context = await browser.new_context(
            user_agent=(
                "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
@@ -152,12 +221,16 @@ async def main():
        )
        page = await context.new_page()

-        for search_fn in [search_craigslist, search_cargurus, search_autotempest]:
+        searches = [search_craigslist, search_cargurus, search_autotempest]
+        if HEADED:
+            searches.append(search_facebook)
+
+        for search_fn in searches:
            try:
                results = await search_fn(page)
                all_listings.extend(results)
            except Exception as e:
-                print(f"  ERROR: {e}")
+                print(f"  ERROR in {search_fn.__name__}: {e}")

        await browser.close()

@@ -168,6 +241,7 @@ async def main():
        json.dump(
            {
                "search_date": datetime.now(timezone.utc).isoformat(),
+                "mode": mode,
                "params": {
                    "zip": "98077",
                    "max_price": 10000,