feat: add noVNC headed mode for interactive browser sessions

- Dockerfile with Xvfb + x11vnc + noVNC for browser-in-a-URL
- search.py auto-detects DISPLAY env to run headed vs headless
- pause_for_human() function for CAPTCHA/login handoff
- Facebook Marketplace search (headed mode only, needs login)
- Updated README with both modes
This commit is contained in:
Ken
2026-05-25 06:56:21 +00:00
parent 290678635e
commit f09317f95b
4 changed files with 168 additions and 11 deletions
+79 -5
View File
@@ -1,10 +1,20 @@
"""
Car search crawler -- finds used hybrids under $10K near Woodinville WA.
Uses Playwright (headless Chromium) to handle JS-heavy car sites.
Two modes:
headless (default): uv run python search.py
headed via noVNC: docker run ... (see README)
When running in the Docker container, the browser is visible at
http://localhost:6080/vnc.html -- use this to handle CAPTCHAs,
log into Facebook Marketplace, or visually inspect results.
"""
import asyncio
import json
import os
import sys
from datetime import datetime, timezone
from pathlib import Path
@@ -13,6 +23,26 @@ from playwright.async_api import async_playwright
RESULTS_DIR = Path("results")
RESULTS_DIR.mkdir(exist_ok=True)
# If DISPLAY is set (noVNC container), run headed
HEADED = bool(os.environ.get("DISPLAY"))
async def pause_for_human(page, reason: str):
"""Pause and prompt for human interaction via noVNC."""
if not HEADED:
print(f" SKIP (headless): {reason}")
return False
print(f"\n{'=' * 60}")
print(f" HUMAN NEEDED: {reason}")
print(f" Open http://localhost:6080/vnc.html to interact")
print(f" Press Enter here when done...")
print(f"{'=' * 60}")
# Wait for user to press Enter in the terminal
await asyncio.get_event_loop().run_in_executor(None, input)
return True
async def search_craigslist(page) -> list[dict]:
"""Craigslist Seattle -- simple HTML, most reliable to scrape."""
@@ -24,7 +54,7 @@ async def search_craigslist(page) -> list[dict]:
"&search_distance=50"
"&sort=date"
)
print(f" Searching Craigslist: {url}")
print(f" Searching Craigslist...")
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
await page.wait_for_timeout(2000)
@@ -96,6 +126,44 @@ async def search_cargurus(page) -> list[dict]:
return listings
async def search_facebook(page) -> list[dict]:
"""Facebook Marketplace -- needs login (human via noVNC)."""
if not HEADED:
print(" SKIP Facebook Marketplace (requires login -- run in Docker with noVNC)")
return []
url = "https://www.facebook.com/marketplace/seattle/vehicles/?maxPrice=10000"
print(f" Opening Facebook Marketplace...")
await page.goto(url, wait_until="networkidle", timeout=60000)
# Check if login is needed
if "login" in page.url.lower():
await pause_for_human(page, "Log into Facebook, then press Enter")
await page.wait_for_timeout(3000)
await pause_for_human(page, "Apply filters (hybrid, $10K max, 50mi) and press Enter when ready")
# Scrape whatever is visible
listings = []
cards = await page.query_selector_all("[class*='marketplace']")
for card in cards[:30]:
try:
text = await card.inner_text()
link_el = await card.query_selector("a[href*='/marketplace/']")
link = await link_el.get_attribute("href") if link_el else ""
listings.append({
"source": "facebook",
"title": text[:100].strip(),
"price": "see listing",
"url": f"https://www.facebook.com{link}" if link else "",
})
except Exception:
continue
print(f" Found {len(listings)} Facebook listings")
return listings
async def search_autotempest(page) -> list[dict]:
"""AutoTempest -- meta-aggregator, searches multiple sites."""
url = (
@@ -132,7 +200,8 @@ async def search_autotempest(page) -> list[dict]:
async def main():
print(f"Car Search -- {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
mode = "HEADED (noVNC)" if HEADED else "HEADLESS"
print(f"Car Search [{mode}] -- {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
print(
"Looking for: Used hybrids under $10K "
"within 50 miles of 98077 (Woodinville WA)"
@@ -142,7 +211,7 @@ async def main():
all_listings: list[dict] = []
async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
browser = await p.chromium.launch(headless=not HEADED)
context = await browser.new_context(
user_agent=(
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
@@ -152,12 +221,16 @@ async def main():
)
page = await context.new_page()
for search_fn in [search_craigslist, search_cargurus, search_autotempest]:
searches = [search_craigslist, search_cargurus, search_autotempest]
if HEADED:
searches.append(search_facebook)
for search_fn in searches:
try:
results = await search_fn(page)
all_listings.extend(results)
except Exception as e:
print(f" ERROR: {e}")
print(f" ERROR in {search_fn.__name__}: {e}")
await browser.close()
@@ -168,6 +241,7 @@ async def main():
json.dump(
{
"search_date": datetime.now(timezone.utc).isoformat(),
"mode": mode,
"params": {
"zip": "98077",
"max_price": 10000,