feat: LLM-powered search with free-text query, filter toggles, and auto-generated guide
- Search box: type anything, toggle filter pills (hybrid, price range, distance, etc.) - Backend calls Anthropic to parse query into search params - Playwright crawls Craigslist, CarGurus, AutoTempest with dynamic URLs - Results scored/ranked with car-specific scoring engine - Second Anthropic call generates a custom buying guide from actual results - Guide sections: top picks, checklist, mechanics, negotiation, budget - All visible live in the browser panel via noVNC
This commit is contained in:
Ken
@@ -1,12 +1,12 @@
|
||||
"""
|
||||
Car-help web app -- FastAPI backend serving the search UI + guide.
|
||||
Car-help web app -- FastAPI backend with Anthropic-powered search + guide generation.
|
||||
|
||||
Runs on port 8080 inside the Docker container. Proxies noVNC from port 6080
|
||||
so the browser iframe works same-origin.
|
||||
Runs on port 8080 inside the Docker container.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
@@ -16,9 +16,7 @@ from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from guide import get_guide
|
||||
|
||||
app = FastAPI(title="car-help", version="0.1.0")
|
||||
app = FastAPI(title="car-help", version="0.2.0")
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
@@ -28,108 +26,193 @@ app.add_middleware(
|
||||
)
|
||||
|
||||
RESULTS_DIR = Path("results")
|
||||
RESULTS_DIR.mkdir(exist_ok=True)
|
||||
NOVNC_UPSTREAM = "http://localhost:6080"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Static files + index (FIRST)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
async def index():
|
||||
html = Path("static/index.html").read_text()
|
||||
return HTMLResponse(html)
|
||||
|
||||
app.mount("/static", StaticFiles(directory="static"), name="static")
|
||||
ANTHROPIC_BASE_URL = os.environ.get("ANTHROPIC_BASE_URL", "https://api.anthropic.com")
|
||||
ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY", "")
|
||||
ANTHROPIC_MODEL = "claude-sonnet-4-20250514"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API routes
|
||||
# Anthropic API helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@app.get("/api/guide")
|
||||
async def guide():
|
||||
return JSONResponse(get_guide())
|
||||
|
||||
|
||||
@app.get("/api/search")
|
||||
async def search():
|
||||
"""Trigger a car search using Playwright."""
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"uv", "run", "python", "search.py",
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
env={
|
||||
"PATH": "/usr/local/bin:/usr/bin:/bin",
|
||||
"HOME": "/root",
|
||||
"DISPLAY": ":99",
|
||||
async def call_anthropic(prompt: str, max_tokens: int = 2000, timeout: int = 60) -> str:
|
||||
"""Call the Anthropic Messages API and return the text response."""
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(
|
||||
f"{ANTHROPIC_BASE_URL}/v1/messages",
|
||||
headers={
|
||||
"x-api-key": ANTHROPIC_API_KEY,
|
||||
"content-type": "application/json",
|
||||
"anthropic-version": "2023-06-01",
|
||||
},
|
||||
json={
|
||||
"model": ANTHROPIC_MODEL,
|
||||
"max_tokens": max_tokens,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
stdout, stderr = await asyncio.wait_for(
|
||||
proc.communicate(), timeout=120
|
||||
)
|
||||
|
||||
# Find the most recent result file
|
||||
result_files = sorted(RESULTS_DIR.glob("search_*.json"))
|
||||
if result_files:
|
||||
data = json.loads(result_files[-1].read_text())
|
||||
return JSONResponse({
|
||||
"status": "ok",
|
||||
"file": result_files[-1].name,
|
||||
"total": data.get("total_listings", 0),
|
||||
"listings": data.get("listings", []),
|
||||
"stdout": stdout.decode()[-500:] if stdout else "",
|
||||
})
|
||||
return JSONResponse({
|
||||
"status": "ok",
|
||||
"total": 0,
|
||||
"listings": [],
|
||||
"stdout": stdout.decode() if stdout else "",
|
||||
"stderr": stderr.decode() if stderr else "",
|
||||
})
|
||||
except asyncio.TimeoutError:
|
||||
return JSONResponse({"status": "error", "error": "Search timed out"}, 504)
|
||||
except Exception as e:
|
||||
return JSONResponse({"status": "error", "error": str(e)}, 500)
|
||||
data = resp.json()
|
||||
if "content" in data and data["content"]:
|
||||
return data["content"][0].get("text", "")
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
@app.get("/api/results")
|
||||
async def list_results():
|
||||
"""List all saved search results."""
|
||||
RESULTS_DIR.mkdir(exist_ok=True)
|
||||
files = sorted(RESULTS_DIR.glob("search_*.json"), reverse=True)
|
||||
results = []
|
||||
for f in files[:20]:
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parse user query into search parameters via LLM
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def parse_query(query: str, filters: list[str]) -> dict:
|
||||
"""Use the LLM to turn a natural language query + filters into search params."""
|
||||
filter_str = ", ".join(filters) if filters else "none"
|
||||
prompt = f"""You are a car search assistant. Parse this user query and active filters into concrete search parameters.
|
||||
|
||||
User query: "{query}"
|
||||
Active filters: {filter_str}
|
||||
|
||||
Return ONLY valid JSON (no markdown, no explanation) with these keys:
|
||||
- "makes": list of car makes to search (e.g. ["Toyota", "Honda"]). Empty list = any make.
|
||||
- "models": list of specific models (e.g. ["Prius", "Camry Hybrid"]). Empty list = any model.
|
||||
- "min_year": minimum year (integer, e.g. 2010). null if not specified.
|
||||
- "max_year": maximum year (integer). null if not specified.
|
||||
- "max_price": maximum price in dollars (integer). Default 10000.
|
||||
- "fuel_type": one of "hybrid", "electric", "any". Default based on filters.
|
||||
- "zip": ZIP code. Default "98077" (Woodinville WA).
|
||||
- "radius": search radius in miles (integer). Default 50.
|
||||
- "seller_type": one of "private", "dealer", "any". Default "any".
|
||||
- "max_miles": maximum mileage (integer). null if not specified.
|
||||
- "search_summary": one-line human-readable summary of what we're searching for.
|
||||
|
||||
Example: {{"makes": ["Toyota"], "models": ["Prius"], "min_year": 2012, "max_year": 2015, "max_price": 8000, "fuel_type": "hybrid", "zip": "98077", "radius": 50, "seller_type": "any", "max_miles": 100000, "search_summary": "Toyota Prius 2012-2015 under $8K within 50mi"}}"""
|
||||
|
||||
text = await call_anthropic(prompt, max_tokens=500, timeout=30)
|
||||
# Extract JSON from response
|
||||
text = text.strip()
|
||||
# Try to find JSON object in the response
|
||||
match = re.search(r"\{.*\}", text, re.DOTALL)
|
||||
if match:
|
||||
try:
|
||||
data = json.loads(f.read_text())
|
||||
results.append({
|
||||
"filename": f.name,
|
||||
"date": data.get("search_date", ""),
|
||||
"total": data.get("total_listings", 0),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
return JSONResponse(results)
|
||||
return json.loads(match.group())
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
# Fallback defaults
|
||||
return {
|
||||
"makes": [],
|
||||
"models": [],
|
||||
"min_year": None,
|
||||
"max_year": None,
|
||||
"max_price": 10000,
|
||||
"fuel_type": "hybrid",
|
||||
"zip": "98077",
|
||||
"radius": 50,
|
||||
"seller_type": "any",
|
||||
"max_miles": None,
|
||||
"search_summary": query,
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/results/{filename}")
|
||||
async def get_result(filename: str):
|
||||
"""Get a specific result file."""
|
||||
path = RESULTS_DIR / filename
|
||||
if not path.exists():
|
||||
return JSONResponse({"error": "not found"}, 404)
|
||||
return JSONResponse(json.loads(path.read_text()))
|
||||
# ---------------------------------------------------------------------------
|
||||
# Build search URLs from parsed parameters
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@app.post("/api/rank")
|
||||
async def rank_listings(request: Request):
|
||||
"""Score and rank car listings."""
|
||||
body = await request.json()
|
||||
listings = body.get("listings", [])
|
||||
def build_craigslist_url(params: dict) -> str:
|
||||
"""Build Craigslist search URL from params."""
|
||||
base = "https://seattle.craigslist.org/search/cta?"
|
||||
parts = []
|
||||
fuel_map = {"hybrid": "4", "electric": "6", "any": ""}
|
||||
fuel = fuel_map.get(params.get("fuel_type", "hybrid"), "")
|
||||
if fuel:
|
||||
parts.append(f"auto_fuel_type={fuel}")
|
||||
if params.get("max_price"):
|
||||
parts.append(f"max_price={params['max_price']}")
|
||||
parts.append(f"postal={params.get('zip', '98077')}")
|
||||
parts.append(f"search_distance={params.get('radius', 50)}")
|
||||
if params.get("min_year"):
|
||||
parts.append(f"min_auto_year={params['min_year']}")
|
||||
if params.get("max_year"):
|
||||
parts.append(f"max_auto_year={params['max_year']}")
|
||||
if params.get("max_miles"):
|
||||
parts.append(f"max_auto_miles={params['max_miles']}")
|
||||
if params.get("seller_type") == "private":
|
||||
parts.append("purveyor=owner")
|
||||
elif params.get("seller_type") == "dealer":
|
||||
parts.append("purveyor=dealer")
|
||||
# Add make/model as search query
|
||||
query_parts = []
|
||||
if params.get("makes"):
|
||||
query_parts.extend(params["makes"])
|
||||
if params.get("models"):
|
||||
query_parts.extend(params["models"])
|
||||
if query_parts:
|
||||
parts.append(f"query={'+'.join(query_parts)}")
|
||||
parts.append("sort=date")
|
||||
return base + "&".join(parts)
|
||||
|
||||
def score(listing: dict) -> int:
|
||||
s = 50 # base score
|
||||
|
||||
def build_cargurus_url(params: dict) -> str:
|
||||
"""Build CarGurus search URL from params."""
|
||||
base = (
|
||||
"https://www.cargurus.com/Cars/inventorylisting/"
|
||||
"viewDetailsFilterViewInventoryListing.action?"
|
||||
)
|
||||
parts = []
|
||||
parts.append(f"zip={params.get('zip', '98077')}")
|
||||
if params.get("max_price"):
|
||||
parts.append(f"maxPrice={params['max_price']}")
|
||||
fuel_map = {"hybrid": "HYBRID", "electric": "ELECTRIC", "any": ""}
|
||||
fuel = fuel_map.get(params.get("fuel_type", "hybrid"), "")
|
||||
if fuel:
|
||||
parts.append(f"fuelTypes={fuel}")
|
||||
parts.append(f"distance={params.get('radius', 50)}")
|
||||
if params.get("max_miles"):
|
||||
parts.append(f"maxMileage={params['max_miles']}")
|
||||
if params.get("min_year"):
|
||||
parts.append(f"startYear={params['min_year']}")
|
||||
if params.get("max_year"):
|
||||
parts.append(f"endYear={params['max_year']}")
|
||||
parts.append("sortDir=ASC&sortType=PRICE")
|
||||
return base + "&".join(parts)
|
||||
|
||||
|
||||
def build_autotempest_url(params: dict) -> str:
|
||||
"""Build AutoTempest search URL from params."""
|
||||
base = "https://www.autotempest.com/results?"
|
||||
parts = []
|
||||
parts.append(f"zip={params.get('zip', '98077')}")
|
||||
if params.get("max_price"):
|
||||
parts.append(f"maxprice={params['max_price']}")
|
||||
fuel_map = {"hybrid": "hybrid", "electric": "electric", "any": ""}
|
||||
fuel = fuel_map.get(params.get("fuel_type", "hybrid"), "")
|
||||
if fuel:
|
||||
parts.append(f"fuel={fuel}")
|
||||
parts.append(f"radius={params.get('radius', 50)}")
|
||||
if params.get("makes"):
|
||||
parts.append(f"make={params['makes'][0].lower()}")
|
||||
if params.get("models"):
|
||||
parts.append(f"model={params['models'][0].lower()}")
|
||||
if params.get("min_year"):
|
||||
parts.append(f"minyear={params['min_year']}")
|
||||
if params.get("max_year"):
|
||||
parts.append(f"maxyear={params['max_year']}")
|
||||
if params.get("max_miles"):
|
||||
parts.append(f"maxmiles={params['max_miles']}")
|
||||
return base + "&".join(parts)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scoring engine
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def score_listing(listing: dict, params: dict) -> int:
|
||||
"""Score a single listing 0-100."""
|
||||
s = 50
|
||||
title = (listing.get("title", "") + " " + listing.get("url", "")).lower()
|
||||
|
||||
# Model bonuses
|
||||
@@ -143,6 +226,10 @@ async def rank_listings(request: Request):
|
||||
s += 20
|
||||
elif "civic hybrid" in title:
|
||||
s += 15
|
||||
elif "corolla" in title:
|
||||
s += 18
|
||||
elif "rav4" in title:
|
||||
s += 15
|
||||
|
||||
# Brand bonuses
|
||||
if "toyota" in title:
|
||||
@@ -151,13 +238,15 @@ async def rank_listings(request: Request):
|
||||
s += 7
|
||||
elif "hyundai" in title:
|
||||
s += 5
|
||||
elif "kia" in title:
|
||||
s += 4
|
||||
|
||||
# Penalties
|
||||
if "nissan" in title and "leaf" in title:
|
||||
s -= 25
|
||||
if "altima" in title:
|
||||
s -= 20
|
||||
if any(w in title for w in ["salvage", "rebuilt", "flood", "junk"]):
|
||||
if any(w in title for w in ["salvage", "rebuilt", "flood", "junk", "parts only"]):
|
||||
s -= 20
|
||||
|
||||
# Price parsing
|
||||
@@ -170,22 +259,235 @@ async def rank_listings(request: Request):
|
||||
s += 10
|
||||
elif 6000 <= price_num <= 8000:
|
||||
s += 5
|
||||
elif price_num > 15000:
|
||||
s -= 10
|
||||
|
||||
# Positive signals
|
||||
if "one owner" in title or "single owner" in title:
|
||||
s += 8
|
||||
if "clean title" in title:
|
||||
s += 8
|
||||
if "low miles" in title or "low mileage" in title:
|
||||
s += 5
|
||||
|
||||
# If user searched for specific makes/models, boost matches
|
||||
for make in params.get("makes", []):
|
||||
if make.lower() in title:
|
||||
s += 5
|
||||
for model in params.get("models", []):
|
||||
if model.lower() in title:
|
||||
s += 5
|
||||
|
||||
return max(0, min(100, s))
|
||||
|
||||
scored = []
|
||||
for listing in listings:
|
||||
listing["score"] = score(listing)
|
||||
scored.append(listing)
|
||||
|
||||
scored.sort(key=lambda x: x["score"], reverse=True)
|
||||
return JSONResponse(scored)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Generate guide via LLM
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def generate_guide(query: str, listings: list[dict], params: dict) -> dict:
|
||||
"""Call the Anthropic API to generate a custom buying guide."""
|
||||
# Prepare top 20 listings summary
|
||||
top_listings = listings[:20]
|
||||
listings_text = "\n".join(
|
||||
f"- {item.get('title', 'Unknown')} | {item.get('price', 'N/A')} | Score: {item.get('score', '?')} | {item.get('url', '')}"
|
||||
for item in top_listings
|
||||
)
|
||||
if not listings_text:
|
||||
listings_text = "(No listings found)"
|
||||
|
||||
prompt = f"""You are a car buying expert helping someone near Woodinville, WA (ZIP 98077).
|
||||
|
||||
The user searched for: "{query}"
|
||||
Search parameters: {json.dumps(params, default=str)}
|
||||
|
||||
Here are the top listings found:
|
||||
{listings_text}
|
||||
|
||||
Generate a comprehensive buying guide tailored to this specific search. Return ONLY valid JSON (no markdown fences, no explanation before/after) with these keys:
|
||||
|
||||
1. "top_picks": Analysis of the best listings found. Which ones look like the best deals and why? Reference specific listings by name/price. If no listings found, give general advice for this type of car. Use markdown formatting (bold, bullet points).
|
||||
|
||||
2. "checklist": What to inspect when looking at THESE specific car models. Be specific to the makes/models found (e.g., if mostly Prius results, talk about hybrid battery health, catalytic converter theft, etc.). Use markdown formatting.
|
||||
|
||||
3. "mechanics": Recommend 3-4 mechanics near Woodinville WA 98077 that would be good for pre-purchase inspections of these types of cars. Include name, approximate location, and why they're good. Use markdown formatting.
|
||||
|
||||
4. "negotiation": Negotiation tips specific to this price range and car type. Include what to look up beforehand (KBB, etc.), how much below asking to start, WA-specific rules (doc fees, etc.). Use markdown formatting.
|
||||
|
||||
5. "budget": Total budget breakdown including the car, registration, insurance, maintenance reserves. Be specific to the price range searched. Use markdown formatting.
|
||||
|
||||
Each value should be a string with markdown formatting for rich display. Be practical, specific, and actionable."""
|
||||
|
||||
text = await call_anthropic(prompt, max_tokens=4000, timeout=90)
|
||||
text = text.strip()
|
||||
|
||||
# Try to parse JSON from response
|
||||
match = re.search(r"\{.*\}", text, re.DOTALL)
|
||||
if match:
|
||||
try:
|
||||
guide = json.loads(match.group())
|
||||
# Ensure all keys exist
|
||||
for key in ["top_picks", "checklist", "mechanics", "negotiation", "budget"]:
|
||||
if key not in guide:
|
||||
guide[key] = "No information available."
|
||||
return guide
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Fallback: return the raw text in top_picks
|
||||
return {
|
||||
"top_picks": text or "Guide generation failed. Please try again.",
|
||||
"checklist": "Run a search to generate a custom checklist.",
|
||||
"mechanics": "Run a search to get mechanic recommendations.",
|
||||
"negotiation": "Run a search to get negotiation tips.",
|
||||
"budget": "Run a search to get a budget breakdown.",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Static files + index
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
async def index():
|
||||
html = Path("static/index.html").read_text()
|
||||
return HTMLResponse(html)
|
||||
|
||||
|
||||
app.mount("/static", StaticFiles(directory="static"), name="static")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API routes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@app.post("/api/search")
|
||||
async def search(request: Request):
|
||||
"""Main search endpoint: parse query, crawl sites, score, generate guide."""
|
||||
body = await request.json()
|
||||
query = body.get("query", "").strip()
|
||||
filters = body.get("filters", [])
|
||||
|
||||
if not query:
|
||||
return JSONResponse({"error": "Please enter a search query"}, 400)
|
||||
|
||||
try:
|
||||
# Step 1: Parse query into search parameters
|
||||
params = await parse_query(query, filters)
|
||||
search_summary = params.get("search_summary", query)
|
||||
|
||||
# Step 2: Build search URLs
|
||||
cl_url = build_craigslist_url(params)
|
||||
cg_url = build_cargurus_url(params)
|
||||
at_url = build_autotempest_url(params)
|
||||
|
||||
# Step 3: Run Playwright crawl via search.py subprocess
|
||||
search_args = json.dumps(
|
||||
{
|
||||
"craigslist_url": cl_url,
|
||||
"cargurus_url": cg_url,
|
||||
"autotempest_url": at_url,
|
||||
"params": params,
|
||||
}
|
||||
)
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"uv",
|
||||
"run",
|
||||
"python",
|
||||
"search.py",
|
||||
"--dynamic",
|
||||
search_args,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
env={
|
||||
"PATH": "/usr/local/bin:/usr/bin:/bin",
|
||||
"HOME": "/root",
|
||||
"DISPLAY": ":99",
|
||||
},
|
||||
)
|
||||
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=180)
|
||||
|
||||
# Step 4: Load results
|
||||
result_files = sorted(RESULTS_DIR.glob("search_*.json"))
|
||||
all_listings = []
|
||||
if result_files:
|
||||
data = json.loads(result_files[-1].read_text())
|
||||
all_listings = data.get("listings", [])
|
||||
|
||||
# Step 5: Score and rank
|
||||
for listing in all_listings:
|
||||
listing["score"] = score_listing(listing, params)
|
||||
all_listings.sort(key=lambda x: x["score"], reverse=True)
|
||||
|
||||
# Step 6: Generate guide via LLM
|
||||
guide = await generate_guide(query, all_listings, params)
|
||||
|
||||
return JSONResponse(
|
||||
{
|
||||
"status": "ok",
|
||||
"search_summary": search_summary,
|
||||
"total": len(all_listings),
|
||||
"listings": all_listings,
|
||||
"guide": guide,
|
||||
"urls_searched": {
|
||||
"craigslist": cl_url,
|
||||
"cargurus": cg_url,
|
||||
"autotempest": at_url,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
return JSONResponse({"error": "Search timed out after 3 minutes"}, 504)
|
||||
except Exception as e:
|
||||
import traceback
|
||||
|
||||
return JSONResponse(
|
||||
{
|
||||
"error": str(e),
|
||||
"traceback": traceback.format_exc(),
|
||||
},
|
||||
500,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Keep existing endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@app.get("/api/results")
|
||||
async def list_results():
|
||||
"""List all saved search results."""
|
||||
RESULTS_DIR.mkdir(exist_ok=True)
|
||||
files = sorted(RESULTS_DIR.glob("search_*.json"), reverse=True)
|
||||
results = []
|
||||
for f in files[:20]:
|
||||
try:
|
||||
data = json.loads(f.read_text())
|
||||
results.append(
|
||||
{
|
||||
"filename": f.name,
|
||||
"date": data.get("search_date", ""),
|
||||
"total": data.get("total_listings", 0),
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
return JSONResponse(results)
|
||||
|
||||
|
||||
@app.get("/api/results/{filename}")
|
||||
async def get_result(filename: str):
|
||||
"""Get a specific result file."""
|
||||
path = RESULTS_DIR / filename
|
||||
if not path.exists():
|
||||
return JSONResponse({"error": "not found"}, 404)
|
||||
return JSONResponse(json.loads(path.read_text()))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1,11 +1,16 @@
|
||||
"""
|
||||
Car search crawler -- finds used hybrids under $10K near Woodinville WA.
|
||||
Car search crawler -- finds used cars based on dynamic parameters.
|
||||
Always runs headed on Xvfb. View live at browser.ampbox.io.
|
||||
|
||||
Usage:
|
||||
python search.py # runs default hardcoded search
|
||||
python search.py --dynamic '{...}' # runs with dynamic URLs/params from app.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
@@ -23,17 +28,9 @@ def extract_title_from_url(url: str) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
async def search_craigslist(page) -> list[dict]:
|
||||
"""Craigslist Seattle -- simple HTML, most reliable to scrape."""
|
||||
url = (
|
||||
"https://seattle.craigslist.org/search/cta"
|
||||
"?auto_fuel_type=4"
|
||||
"&max_price=10000"
|
||||
"&postal=98077"
|
||||
"&search_distance=50"
|
||||
"&sort=date"
|
||||
)
|
||||
print(f" Searching Craigslist...")
|
||||
async def search_craigslist(page, url: str) -> list[dict]:
|
||||
"""Craigslist -- simple HTML, most reliable to scrape."""
|
||||
print(f" Searching Craigslist: {url}")
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
||||
await page.wait_for_timeout(3000)
|
||||
|
||||
@@ -43,29 +40,30 @@ async def search_craigslist(page) -> list[dict]:
|
||||
|
||||
for card in cards[:50]:
|
||||
try:
|
||||
# Get the full text of the card -- title is in there
|
||||
text = await card.inner_text()
|
||||
lines = [l.strip() for l in text.split("\n") if l.strip() and l.strip() != "•"]
|
||||
lines = [
|
||||
ln.strip()
|
||||
for ln in text.split("\n")
|
||||
if ln.strip() and ln.strip() != "\u2022"
|
||||
]
|
||||
|
||||
# Get the link
|
||||
link_el = await card.query_selector("a.main, a.cl-app-anchor, a[href*='/d/']")
|
||||
link_el = await card.query_selector(
|
||||
"a.main, a.cl-app-anchor, a[href*='/d/']"
|
||||
)
|
||||
link = await link_el.get_attribute("href") if link_el else ""
|
||||
|
||||
# Title is usually the longest non-price, non-date text line
|
||||
title = ""
|
||||
price = ""
|
||||
location = ""
|
||||
for line in lines:
|
||||
if line.startswith("$"):
|
||||
price = line
|
||||
elif re.match(r"^\d+/\d+$", line):
|
||||
continue # date like 5/23
|
||||
continue
|
||||
elif re.match(r"^\d+k?\s*mi", line, re.I):
|
||||
continue # mileage
|
||||
continue
|
||||
elif not title and len(line) > 5:
|
||||
title = line
|
||||
|
||||
# Fallback: extract from URL
|
||||
if not title and link:
|
||||
title = extract_title_from_url(link)
|
||||
|
||||
@@ -74,17 +72,18 @@ async def search_craigslist(page) -> list[dict]:
|
||||
if price_el:
|
||||
price = await price_el.inner_text()
|
||||
|
||||
# Get image
|
||||
img_el = await card.query_selector("img[src]")
|
||||
img = await img_el.get_attribute("src") if img_el else ""
|
||||
|
||||
listings.append({
|
||||
listings.append(
|
||||
{
|
||||
"source": "craigslist",
|
||||
"title": title.strip(),
|
||||
"price": price.strip(),
|
||||
"url": link,
|
||||
"image": img,
|
||||
})
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
@@ -92,25 +91,20 @@ async def search_craigslist(page) -> list[dict]:
|
||||
return listings
|
||||
|
||||
|
||||
async def search_cargurus(page) -> list[dict]:
|
||||
async def search_cargurus(page, url: str) -> list[dict]:
|
||||
"""CarGurus -- JS-heavy but good data."""
|
||||
url = (
|
||||
"https://www.cargurus.com/Cars/inventorylisting/"
|
||||
"viewDetailsFilterViewInventoryListing.action"
|
||||
"?zip=98077&maxPrice=10000&fuelTypes=HYBRID"
|
||||
"&distance=50&sortDir=ASC&sortType=PRICE"
|
||||
)
|
||||
print(f" Searching CarGurus...")
|
||||
print(f" Searching CarGurus: {url}")
|
||||
await page.goto(url, wait_until="networkidle", timeout=60000)
|
||||
await page.wait_for_timeout(3000)
|
||||
|
||||
listings = []
|
||||
# Try multiple selector patterns for CarGurus
|
||||
cards = await page.query_selector_all("article, [data-cg-ft='car-blade'], a[href*='/Cars/']")
|
||||
cards = await page.query_selector_all(
|
||||
"article, [data-cg-ft='car-blade'], a[href*='/Cars/']"
|
||||
)
|
||||
for card in cards[:50]:
|
||||
try:
|
||||
text = await card.inner_text()
|
||||
lines = [l.strip() for l in text.split("\n") if l.strip()]
|
||||
lines = [ln.strip() for ln in text.split("\n") if ln.strip()]
|
||||
|
||||
title = lines[0] if lines else "Unknown"
|
||||
price = ""
|
||||
@@ -129,13 +123,15 @@ async def search_cargurus(page) -> list[dict]:
|
||||
img_el = await card.query_selector("img[src*='cargurus']")
|
||||
img = await img_el.get_attribute("src") if img_el else ""
|
||||
|
||||
listings.append({
|
||||
listings.append(
|
||||
{
|
||||
"source": "cargurus",
|
||||
"title": title[:100].strip(),
|
||||
"price": price.strip(),
|
||||
"url": link,
|
||||
"image": img,
|
||||
})
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
@@ -143,25 +139,23 @@ async def search_cargurus(page) -> list[dict]:
|
||||
return listings
|
||||
|
||||
|
||||
async def search_autotempest(page) -> list[dict]:
|
||||
async def search_autotempest(page, url: str) -> list[dict]:
|
||||
"""AutoTempest -- meta-aggregator."""
|
||||
url = (
|
||||
"https://www.autotempest.com/results"
|
||||
"?zip=98077&maxprice=10000&fuel=hybrid&radius=50"
|
||||
)
|
||||
print(f" Searching AutoTempest...")
|
||||
print(f" Searching AutoTempest: {url}")
|
||||
await page.goto(url, wait_until="networkidle", timeout=60000)
|
||||
await page.wait_for_timeout(5000)
|
||||
|
||||
listings = []
|
||||
cards = await page.query_selector_all(".result-list-item, .listing, [class*='result']")
|
||||
cards = await page.query_selector_all(
|
||||
".result-list-item, .listing, [class*='result']"
|
||||
)
|
||||
for card in cards[:50]:
|
||||
try:
|
||||
text = await card.inner_text()
|
||||
if len(text.strip()) < 10:
|
||||
continue
|
||||
|
||||
lines = [l.strip() for l in text.split("\n") if l.strip()]
|
||||
lines = [ln.strip() for ln in text.split("\n") if ln.strip()]
|
||||
title = lines[0] if lines else "Unknown"
|
||||
price = ""
|
||||
for line in lines:
|
||||
@@ -175,13 +169,15 @@ async def search_autotempest(page) -> list[dict]:
|
||||
img_el = await card.query_selector("img[src]")
|
||||
img = await img_el.get_attribute("src") if img_el else ""
|
||||
|
||||
listings.append({
|
||||
listings.append(
|
||||
{
|
||||
"source": "autotempest",
|
||||
"title": title[:100].strip(),
|
||||
"price": price.strip(),
|
||||
"url": link,
|
||||
"image": img,
|
||||
})
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
@@ -191,7 +187,17 @@ async def search_autotempest(page) -> list[dict]:
|
||||
|
||||
async def main():
|
||||
print(f"Car Search -- {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
|
||||
print("Looking for: Used hybrids under $10K within 50mi of 98077 (Woodinville WA)")
|
||||
|
||||
# Parse dynamic args if provided
|
||||
dynamic_config = None
|
||||
if len(sys.argv) >= 3 and sys.argv[1] == "--dynamic":
|
||||
dynamic_config = json.loads(sys.argv[2])
|
||||
params = dynamic_config.get("params", {})
|
||||
search_summary = params.get("search_summary", "dynamic search")
|
||||
print(f"Dynamic search: {search_summary}")
|
||||
else:
|
||||
print("Default search: Used hybrids under $10K within 50mi of 98077")
|
||||
|
||||
print("=" * 70)
|
||||
|
||||
all_listings: list[dict] = []
|
||||
@@ -207,28 +213,66 @@ async def main():
|
||||
)
|
||||
page = await context.new_page()
|
||||
|
||||
for search_fn in [search_craigslist, search_cargurus, search_autotempest]:
|
||||
if dynamic_config:
|
||||
# Dynamic search with provided URLs
|
||||
searches = [
|
||||
(search_craigslist, dynamic_config.get("craigslist_url", "")),
|
||||
(search_cargurus, dynamic_config.get("cargurus_url", "")),
|
||||
(search_autotempest, dynamic_config.get("autotempest_url", "")),
|
||||
]
|
||||
for search_fn, url in searches:
|
||||
if url:
|
||||
try:
|
||||
results = await search_fn(page)
|
||||
results = await search_fn(page, url)
|
||||
all_listings.extend(results)
|
||||
except Exception as e:
|
||||
print(f" ERROR in {search_fn.__name__}: {e}")
|
||||
else:
|
||||
# Default hardcoded search
|
||||
default_urls = {
|
||||
"craigslist": (
|
||||
"https://seattle.craigslist.org/search/cta"
|
||||
"?auto_fuel_type=4&max_price=10000&postal=98077"
|
||||
"&search_distance=50&sort=date"
|
||||
),
|
||||
"cargurus": (
|
||||
"https://www.cargurus.com/Cars/inventorylisting/"
|
||||
"viewDetailsFilterViewInventoryListing.action"
|
||||
"?zip=98077&maxPrice=10000&fuelTypes=HYBRID"
|
||||
"&distance=50&sortDir=ASC&sortType=PRICE"
|
||||
),
|
||||
"autotempest": (
|
||||
"https://www.autotempest.com/results"
|
||||
"?zip=98077&maxprice=10000&fuel=hybrid&radius=50"
|
||||
),
|
||||
}
|
||||
for name, url in default_urls.items():
|
||||
search_fn = {
|
||||
"craigslist": search_craigslist,
|
||||
"cargurus": search_cargurus,
|
||||
"autotempest": search_autotempest,
|
||||
}[name]
|
||||
try:
|
||||
results = await search_fn(page, url)
|
||||
all_listings.extend(results)
|
||||
except Exception as e:
|
||||
print(f" ERROR in {name}: {e}")
|
||||
|
||||
await browser.close()
|
||||
|
||||
# Save results
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M")
|
||||
search_params = (
|
||||
dynamic_config.get("params", {})
|
||||
if dynamic_config
|
||||
else {"zip": "98077", "max_price": 10000, "fuel": "hybrid", "radius_miles": 50}
|
||||
)
|
||||
output_file = RESULTS_DIR / f"search_{timestamp}.json"
|
||||
with open(output_file, "w") as f:
|
||||
json.dump(
|
||||
{
|
||||
"search_date": datetime.now(timezone.utc).isoformat(),
|
||||
"params": {
|
||||
"zip": "98077",
|
||||
"max_price": 10000,
|
||||
"fuel": "hybrid",
|
||||
"radius_miles": 50,
|
||||
},
|
||||
"params": search_params,
|
||||
"total_listings": len(all_listings),
|
||||
"listings": all_listings,
|
||||
},
|
||||
@@ -241,7 +285,9 @@ async def main():
|
||||
print(f"Saved to: {output_file}")
|
||||
|
||||
for listing in all_listings[:10]:
|
||||
print(f" [{listing['source']}] {listing['price']:>8s} {listing['title'][:60]}")
|
||||
print(
|
||||
f" [{listing['source']}] {listing['price']:>8s} {listing['title'][:60]}"
|
||||
)
|
||||
|
||||
if len(all_listings) > 10:
|
||||
print(f" ... and {len(all_listings) - 10} more")
|
||||
|
||||
+488
-501
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user