From f09317f95b62868b03ccc3193ecc9a9b1a37ddd5 Mon Sep 17 00:00:00 2001 From: Ken Date: Mon, 25 May 2026 06:56:21 +0000 Subject: [PATCH] feat: add noVNC headed mode for interactive browser sessions - Dockerfile with Xvfb + x11vnc + noVNC for browser-in-a-URL - search.py auto-detects DISPLAY env to run headed vs headless - pause_for_human() function for CAPTCHA/login handoff - Facebook Marketplace search (headed mode only, needs login) - Updated README with both modes --- Dockerfile | 29 ++++++++++++++++++ README.md | 38 +++++++++++++++++++---- entrypoint.sh | 28 +++++++++++++++++ search.py | 84 ++++++++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 168 insertions(+), 11 deletions(-) create mode 100644 Dockerfile create mode 100644 entrypoint.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..638afa2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.13-slim + +# Install display and VNC deps +RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \ + xvfb x11vnc websockify novnc \ + libnss3 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 \ + libxkbcommon0 libxcomposite1 libxdamage1 libxrandr2 libgbm1 \ + libpango-1.0-0 libcairo2 libasound2 libxshmfence1 libgtk-3-0 \ + fonts-liberation && \ + rm -rf /var/lib/apt/lists/* + +# Install Python deps +COPY pyproject.toml uv.lock /app/ +WORKDIR /app +RUN pip install uv && uv sync --frozen + +# Install Playwright Chromium +RUN uv run playwright install chromium + +# Copy source +COPY . /app/ + +# Entrypoint script +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +EXPOSE 6080 + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/README.md b/README.md index b1ec704..d290d9d 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,41 @@ Used car search tool for finding Jude a reliable hybrid (HEV) in the Woodinville, WA area. Budget: $8-10K. -## Setup +## Two Modes + +### Headless (fast, no interaction) + +Runs Chromium headless. Good for sites that don't need login. Skips Facebook Marketplace. ```bash uv sync uv run playwright install chromium -``` - -## Usage - -```bash uv run python search.py ``` + +### Headed via noVNC (interactive, for logins/CAPTCHAs) + +Runs Chromium inside Docker with a virtual display. Open the browser view in any web browser when you need to interact (login, solve CAPTCHAs, apply filters manually). + +```bash +# Build once +docker build -t car-help . + +# Run (browser visible at http://localhost:6080/vnc.html) +docker run -d --name car-help -p 6080:6080 -v $(pwd)/results:/app/results car-help + +# Run a search (headed -- includes Facebook Marketplace) +docker exec -it car-help uv run python search.py + +# View the browser live +open http://localhost:6080/vnc.html + +# Stop +docker stop car-help && docker rm car-help +``` + +Results are saved to `results/search_.json`. + +## Sites + +See `sites.yaml` for the full list of car search sites, search parameters, and target models. diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..c2e857b --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -e + +# Start virtual display +export DISPLAY=:99 +Xvfb :99 -screen 0 1280x720x24 -ac & +sleep 1 + +# Start VNC server on the virtual display (no password, listen on 5900) +x11vnc -display :99 -forever -nopw -listen 0.0.0.0 -rfbport 5900 -shared & +sleep 1 + +# Start noVNC web proxy (serves browser view at http://localhost:6080/vnc.html) +websockify --web=/usr/share/novnc/ 6080 localhost:5900 & +sleep 1 + +echo "=============================================" +echo " noVNC ready: http://localhost:6080/vnc.html" +echo "=============================================" + +# Run whatever command was passed (or drop to shell) +if [ $# -eq 0 ]; then + echo "No command given. Container is running -- attach to use." + echo "Run searches with: docker exec car-help uv run python search.py" + tail -f /dev/null +else + exec "$@" +fi diff --git a/search.py b/search.py index 20d54a8..b61786d 100644 --- a/search.py +++ b/search.py @@ -1,10 +1,20 @@ """ Car search crawler -- finds used hybrids under $10K near Woodinville WA. Uses Playwright (headless Chromium) to handle JS-heavy car sites. + +Two modes: + headless (default): uv run python search.py + headed via noVNC: docker run ... (see README) + +When running in the Docker container, the browser is visible at +http://localhost:6080/vnc.html -- use this to handle CAPTCHAs, +log into Facebook Marketplace, or visually inspect results. """ import asyncio import json +import os +import sys from datetime import datetime, timezone from pathlib import Path @@ -13,6 +23,26 @@ from playwright.async_api import async_playwright RESULTS_DIR = Path("results") RESULTS_DIR.mkdir(exist_ok=True) +# If DISPLAY is set (noVNC container), run headed +HEADED = bool(os.environ.get("DISPLAY")) + + +async def pause_for_human(page, reason: str): + """Pause and prompt for human interaction via noVNC.""" + if not HEADED: + print(f" SKIP (headless): {reason}") + return False + + print(f"\n{'=' * 60}") + print(f" HUMAN NEEDED: {reason}") + print(f" Open http://localhost:6080/vnc.html to interact") + print(f" Press Enter here when done...") + print(f"{'=' * 60}") + + # Wait for user to press Enter in the terminal + await asyncio.get_event_loop().run_in_executor(None, input) + return True + async def search_craigslist(page) -> list[dict]: """Craigslist Seattle -- simple HTML, most reliable to scrape.""" @@ -24,7 +54,7 @@ async def search_craigslist(page) -> list[dict]: "&search_distance=50" "&sort=date" ) - print(f" Searching Craigslist: {url}") + print(f" Searching Craigslist...") await page.goto(url, wait_until="domcontentloaded", timeout=30000) await page.wait_for_timeout(2000) @@ -96,6 +126,44 @@ async def search_cargurus(page) -> list[dict]: return listings +async def search_facebook(page) -> list[dict]: + """Facebook Marketplace -- needs login (human via noVNC).""" + if not HEADED: + print(" SKIP Facebook Marketplace (requires login -- run in Docker with noVNC)") + return [] + + url = "https://www.facebook.com/marketplace/seattle/vehicles/?maxPrice=10000" + print(f" Opening Facebook Marketplace...") + await page.goto(url, wait_until="networkidle", timeout=60000) + + # Check if login is needed + if "login" in page.url.lower(): + await pause_for_human(page, "Log into Facebook, then press Enter") + + await page.wait_for_timeout(3000) + await pause_for_human(page, "Apply filters (hybrid, $10K max, 50mi) and press Enter when ready") + + # Scrape whatever is visible + listings = [] + cards = await page.query_selector_all("[class*='marketplace']") + for card in cards[:30]: + try: + text = await card.inner_text() + link_el = await card.query_selector("a[href*='/marketplace/']") + link = await link_el.get_attribute("href") if link_el else "" + listings.append({ + "source": "facebook", + "title": text[:100].strip(), + "price": "see listing", + "url": f"https://www.facebook.com{link}" if link else "", + }) + except Exception: + continue + + print(f" Found {len(listings)} Facebook listings") + return listings + + async def search_autotempest(page) -> list[dict]: """AutoTempest -- meta-aggregator, searches multiple sites.""" url = ( @@ -132,7 +200,8 @@ async def search_autotempest(page) -> list[dict]: async def main(): - print(f"Car Search -- {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}") + mode = "HEADED (noVNC)" if HEADED else "HEADLESS" + print(f"Car Search [{mode}] -- {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}") print( "Looking for: Used hybrids under $10K " "within 50 miles of 98077 (Woodinville WA)" @@ -142,7 +211,7 @@ async def main(): all_listings: list[dict] = [] async with async_playwright() as p: - browser = await p.chromium.launch(headless=True) + browser = await p.chromium.launch(headless=not HEADED) context = await browser.new_context( user_agent=( "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " @@ -152,12 +221,16 @@ async def main(): ) page = await context.new_page() - for search_fn in [search_craigslist, search_cargurus, search_autotempest]: + searches = [search_craigslist, search_cargurus, search_autotempest] + if HEADED: + searches.append(search_facebook) + + for search_fn in searches: try: results = await search_fn(page) all_listings.extend(results) except Exception as e: - print(f" ERROR: {e}") + print(f" ERROR in {search_fn.__name__}: {e}") await browser.close() @@ -168,6 +241,7 @@ async def main(): json.dump( { "search_date": datetime.now(timezone.utc).isoformat(), + "mode": mode, "params": { "zip": "98077", "max_price": 10000,