feat: LLM-powered search with free-text query, filter toggles, and auto-generated guide
- Search box: type anything, toggle filter pills (hybrid, price range, distance, etc.) - Backend calls Anthropic to parse query into search params - Playwright crawls Craigslist, CarGurus, AutoTempest with dynamic URLs - Results scored/ranked with car-specific scoring engine - Second Anthropic call generates a custom buying guide from actual results - Guide sections: top picks, checklist, mechanics, negotiation, budget - All visible live in the browser panel via noVNC
This commit is contained in:
@@ -1,12 +1,12 @@
|
|||||||
"""
|
"""
|
||||||
Car-help web app -- FastAPI backend serving the search UI + guide.
|
Car-help web app -- FastAPI backend with Anthropic-powered search + guide generation.
|
||||||
|
|
||||||
Runs on port 8080 inside the Docker container. Proxies noVNC from port 6080
|
Runs on port 8080 inside the Docker container.
|
||||||
so the browser iframe works same-origin.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -16,9 +16,7 @@ from fastapi.middleware.cors import CORSMiddleware
|
|||||||
from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse
|
from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse
|
||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
|
|
||||||
from guide import get_guide
|
app = FastAPI(title="car-help", version="0.2.0")
|
||||||
|
|
||||||
app = FastAPI(title="car-help", version="0.1.0")
|
|
||||||
|
|
||||||
app.add_middleware(
|
app.add_middleware(
|
||||||
CORSMiddleware,
|
CORSMiddleware,
|
||||||
@@ -28,18 +26,336 @@ app.add_middleware(
|
|||||||
)
|
)
|
||||||
|
|
||||||
RESULTS_DIR = Path("results")
|
RESULTS_DIR = Path("results")
|
||||||
|
RESULTS_DIR.mkdir(exist_ok=True)
|
||||||
NOVNC_UPSTREAM = "http://localhost:6080"
|
NOVNC_UPSTREAM = "http://localhost:6080"
|
||||||
|
|
||||||
|
ANTHROPIC_BASE_URL = os.environ.get("ANTHROPIC_BASE_URL", "https://api.anthropic.com")
|
||||||
|
ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY", "")
|
||||||
|
ANTHROPIC_MODEL = "claude-sonnet-4-20250514"
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Static files + index (FIRST)
|
# Anthropic API helper
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
async def call_anthropic(prompt: str, max_tokens: int = 2000, timeout: int = 60) -> str:
|
||||||
|
"""Call the Anthropic Messages API and return the text response."""
|
||||||
|
async with httpx.AsyncClient() as client:
|
||||||
|
resp = await client.post(
|
||||||
|
f"{ANTHROPIC_BASE_URL}/v1/messages",
|
||||||
|
headers={
|
||||||
|
"x-api-key": ANTHROPIC_API_KEY,
|
||||||
|
"content-type": "application/json",
|
||||||
|
"anthropic-version": "2023-06-01",
|
||||||
|
},
|
||||||
|
json={
|
||||||
|
"model": ANTHROPIC_MODEL,
|
||||||
|
"max_tokens": max_tokens,
|
||||||
|
"messages": [{"role": "user", "content": prompt}],
|
||||||
|
},
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
data = resp.json()
|
||||||
|
if "content" in data and data["content"]:
|
||||||
|
return data["content"][0].get("text", "")
|
||||||
|
return json.dumps(data)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Parse user query into search parameters via LLM
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
async def parse_query(query: str, filters: list[str]) -> dict:
|
||||||
|
"""Use the LLM to turn a natural language query + filters into search params."""
|
||||||
|
filter_str = ", ".join(filters) if filters else "none"
|
||||||
|
prompt = f"""You are a car search assistant. Parse this user query and active filters into concrete search parameters.
|
||||||
|
|
||||||
|
User query: "{query}"
|
||||||
|
Active filters: {filter_str}
|
||||||
|
|
||||||
|
Return ONLY valid JSON (no markdown, no explanation) with these keys:
|
||||||
|
- "makes": list of car makes to search (e.g. ["Toyota", "Honda"]). Empty list = any make.
|
||||||
|
- "models": list of specific models (e.g. ["Prius", "Camry Hybrid"]). Empty list = any model.
|
||||||
|
- "min_year": minimum year (integer, e.g. 2010). null if not specified.
|
||||||
|
- "max_year": maximum year (integer). null if not specified.
|
||||||
|
- "max_price": maximum price in dollars (integer). Default 10000.
|
||||||
|
- "fuel_type": one of "hybrid", "electric", "any". Default based on filters.
|
||||||
|
- "zip": ZIP code. Default "98077" (Woodinville WA).
|
||||||
|
- "radius": search radius in miles (integer). Default 50.
|
||||||
|
- "seller_type": one of "private", "dealer", "any". Default "any".
|
||||||
|
- "max_miles": maximum mileage (integer). null if not specified.
|
||||||
|
- "search_summary": one-line human-readable summary of what we're searching for.
|
||||||
|
|
||||||
|
Example: {{"makes": ["Toyota"], "models": ["Prius"], "min_year": 2012, "max_year": 2015, "max_price": 8000, "fuel_type": "hybrid", "zip": "98077", "radius": 50, "seller_type": "any", "max_miles": 100000, "search_summary": "Toyota Prius 2012-2015 under $8K within 50mi"}}"""
|
||||||
|
|
||||||
|
text = await call_anthropic(prompt, max_tokens=500, timeout=30)
|
||||||
|
# Extract JSON from response
|
||||||
|
text = text.strip()
|
||||||
|
# Try to find JSON object in the response
|
||||||
|
match = re.search(r"\{.*\}", text, re.DOTALL)
|
||||||
|
if match:
|
||||||
|
try:
|
||||||
|
return json.loads(match.group())
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
pass
|
||||||
|
# Fallback defaults
|
||||||
|
return {
|
||||||
|
"makes": [],
|
||||||
|
"models": [],
|
||||||
|
"min_year": None,
|
||||||
|
"max_year": None,
|
||||||
|
"max_price": 10000,
|
||||||
|
"fuel_type": "hybrid",
|
||||||
|
"zip": "98077",
|
||||||
|
"radius": 50,
|
||||||
|
"seller_type": "any",
|
||||||
|
"max_miles": None,
|
||||||
|
"search_summary": query,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Build search URLs from parsed parameters
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def build_craigslist_url(params: dict) -> str:
|
||||||
|
"""Build Craigslist search URL from params."""
|
||||||
|
base = "https://seattle.craigslist.org/search/cta?"
|
||||||
|
parts = []
|
||||||
|
fuel_map = {"hybrid": "4", "electric": "6", "any": ""}
|
||||||
|
fuel = fuel_map.get(params.get("fuel_type", "hybrid"), "")
|
||||||
|
if fuel:
|
||||||
|
parts.append(f"auto_fuel_type={fuel}")
|
||||||
|
if params.get("max_price"):
|
||||||
|
parts.append(f"max_price={params['max_price']}")
|
||||||
|
parts.append(f"postal={params.get('zip', '98077')}")
|
||||||
|
parts.append(f"search_distance={params.get('radius', 50)}")
|
||||||
|
if params.get("min_year"):
|
||||||
|
parts.append(f"min_auto_year={params['min_year']}")
|
||||||
|
if params.get("max_year"):
|
||||||
|
parts.append(f"max_auto_year={params['max_year']}")
|
||||||
|
if params.get("max_miles"):
|
||||||
|
parts.append(f"max_auto_miles={params['max_miles']}")
|
||||||
|
if params.get("seller_type") == "private":
|
||||||
|
parts.append("purveyor=owner")
|
||||||
|
elif params.get("seller_type") == "dealer":
|
||||||
|
parts.append("purveyor=dealer")
|
||||||
|
# Add make/model as search query
|
||||||
|
query_parts = []
|
||||||
|
if params.get("makes"):
|
||||||
|
query_parts.extend(params["makes"])
|
||||||
|
if params.get("models"):
|
||||||
|
query_parts.extend(params["models"])
|
||||||
|
if query_parts:
|
||||||
|
parts.append(f"query={'+'.join(query_parts)}")
|
||||||
|
parts.append("sort=date")
|
||||||
|
return base + "&".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def build_cargurus_url(params: dict) -> str:
|
||||||
|
"""Build CarGurus search URL from params."""
|
||||||
|
base = (
|
||||||
|
"https://www.cargurus.com/Cars/inventorylisting/"
|
||||||
|
"viewDetailsFilterViewInventoryListing.action?"
|
||||||
|
)
|
||||||
|
parts = []
|
||||||
|
parts.append(f"zip={params.get('zip', '98077')}")
|
||||||
|
if params.get("max_price"):
|
||||||
|
parts.append(f"maxPrice={params['max_price']}")
|
||||||
|
fuel_map = {"hybrid": "HYBRID", "electric": "ELECTRIC", "any": ""}
|
||||||
|
fuel = fuel_map.get(params.get("fuel_type", "hybrid"), "")
|
||||||
|
if fuel:
|
||||||
|
parts.append(f"fuelTypes={fuel}")
|
||||||
|
parts.append(f"distance={params.get('radius', 50)}")
|
||||||
|
if params.get("max_miles"):
|
||||||
|
parts.append(f"maxMileage={params['max_miles']}")
|
||||||
|
if params.get("min_year"):
|
||||||
|
parts.append(f"startYear={params['min_year']}")
|
||||||
|
if params.get("max_year"):
|
||||||
|
parts.append(f"endYear={params['max_year']}")
|
||||||
|
parts.append("sortDir=ASC&sortType=PRICE")
|
||||||
|
return base + "&".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def build_autotempest_url(params: dict) -> str:
|
||||||
|
"""Build AutoTempest search URL from params."""
|
||||||
|
base = "https://www.autotempest.com/results?"
|
||||||
|
parts = []
|
||||||
|
parts.append(f"zip={params.get('zip', '98077')}")
|
||||||
|
if params.get("max_price"):
|
||||||
|
parts.append(f"maxprice={params['max_price']}")
|
||||||
|
fuel_map = {"hybrid": "hybrid", "electric": "electric", "any": ""}
|
||||||
|
fuel = fuel_map.get(params.get("fuel_type", "hybrid"), "")
|
||||||
|
if fuel:
|
||||||
|
parts.append(f"fuel={fuel}")
|
||||||
|
parts.append(f"radius={params.get('radius', 50)}")
|
||||||
|
if params.get("makes"):
|
||||||
|
parts.append(f"make={params['makes'][0].lower()}")
|
||||||
|
if params.get("models"):
|
||||||
|
parts.append(f"model={params['models'][0].lower()}")
|
||||||
|
if params.get("min_year"):
|
||||||
|
parts.append(f"minyear={params['min_year']}")
|
||||||
|
if params.get("max_year"):
|
||||||
|
parts.append(f"maxyear={params['max_year']}")
|
||||||
|
if params.get("max_miles"):
|
||||||
|
parts.append(f"maxmiles={params['max_miles']}")
|
||||||
|
return base + "&".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Scoring engine
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def score_listing(listing: dict, params: dict) -> int:
|
||||||
|
"""Score a single listing 0-100."""
|
||||||
|
s = 50
|
||||||
|
title = (listing.get("title", "") + " " + listing.get("url", "")).lower()
|
||||||
|
|
||||||
|
# Model bonuses
|
||||||
|
if "prius" in title:
|
||||||
|
s += 30
|
||||||
|
elif "camry hybrid" in title or "camry" in title:
|
||||||
|
s += 25
|
||||||
|
elif "insight" in title:
|
||||||
|
s += 20
|
||||||
|
elif "ioniq" in title:
|
||||||
|
s += 20
|
||||||
|
elif "civic hybrid" in title:
|
||||||
|
s += 15
|
||||||
|
elif "corolla" in title:
|
||||||
|
s += 18
|
||||||
|
elif "rav4" in title:
|
||||||
|
s += 15
|
||||||
|
|
||||||
|
# Brand bonuses
|
||||||
|
if "toyota" in title:
|
||||||
|
s += 10
|
||||||
|
elif "honda" in title:
|
||||||
|
s += 7
|
||||||
|
elif "hyundai" in title:
|
||||||
|
s += 5
|
||||||
|
elif "kia" in title:
|
||||||
|
s += 4
|
||||||
|
|
||||||
|
# Penalties
|
||||||
|
if "nissan" in title and "leaf" in title:
|
||||||
|
s -= 25
|
||||||
|
if "altima" in title:
|
||||||
|
s -= 20
|
||||||
|
if any(w in title for w in ["salvage", "rebuilt", "flood", "junk", "parts only"]):
|
||||||
|
s -= 20
|
||||||
|
|
||||||
|
# Price parsing
|
||||||
|
price_str = listing.get("price", "")
|
||||||
|
price_num = 0
|
||||||
|
m = re.search(r"\$?([\d,]+)", price_str)
|
||||||
|
if m:
|
||||||
|
price_num = int(m.group(1).replace(",", ""))
|
||||||
|
if 0 < price_num < 6000:
|
||||||
|
s += 10
|
||||||
|
elif 6000 <= price_num <= 8000:
|
||||||
|
s += 5
|
||||||
|
elif price_num > 15000:
|
||||||
|
s -= 10
|
||||||
|
|
||||||
|
# Positive signals
|
||||||
|
if "one owner" in title or "single owner" in title:
|
||||||
|
s += 8
|
||||||
|
if "clean title" in title:
|
||||||
|
s += 8
|
||||||
|
if "low miles" in title or "low mileage" in title:
|
||||||
|
s += 5
|
||||||
|
|
||||||
|
# If user searched for specific makes/models, boost matches
|
||||||
|
for make in params.get("makes", []):
|
||||||
|
if make.lower() in title:
|
||||||
|
s += 5
|
||||||
|
for model in params.get("models", []):
|
||||||
|
if model.lower() in title:
|
||||||
|
s += 5
|
||||||
|
|
||||||
|
return max(0, min(100, s))
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Generate guide via LLM
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
async def generate_guide(query: str, listings: list[dict], params: dict) -> dict:
|
||||||
|
"""Call the Anthropic API to generate a custom buying guide."""
|
||||||
|
# Prepare top 20 listings summary
|
||||||
|
top_listings = listings[:20]
|
||||||
|
listings_text = "\n".join(
|
||||||
|
f"- {item.get('title', 'Unknown')} | {item.get('price', 'N/A')} | Score: {item.get('score', '?')} | {item.get('url', '')}"
|
||||||
|
for item in top_listings
|
||||||
|
)
|
||||||
|
if not listings_text:
|
||||||
|
listings_text = "(No listings found)"
|
||||||
|
|
||||||
|
prompt = f"""You are a car buying expert helping someone near Woodinville, WA (ZIP 98077).
|
||||||
|
|
||||||
|
The user searched for: "{query}"
|
||||||
|
Search parameters: {json.dumps(params, default=str)}
|
||||||
|
|
||||||
|
Here are the top listings found:
|
||||||
|
{listings_text}
|
||||||
|
|
||||||
|
Generate a comprehensive buying guide tailored to this specific search. Return ONLY valid JSON (no markdown fences, no explanation before/after) with these keys:
|
||||||
|
|
||||||
|
1. "top_picks": Analysis of the best listings found. Which ones look like the best deals and why? Reference specific listings by name/price. If no listings found, give general advice for this type of car. Use markdown formatting (bold, bullet points).
|
||||||
|
|
||||||
|
2. "checklist": What to inspect when looking at THESE specific car models. Be specific to the makes/models found (e.g., if mostly Prius results, talk about hybrid battery health, catalytic converter theft, etc.). Use markdown formatting.
|
||||||
|
|
||||||
|
3. "mechanics": Recommend 3-4 mechanics near Woodinville WA 98077 that would be good for pre-purchase inspections of these types of cars. Include name, approximate location, and why they're good. Use markdown formatting.
|
||||||
|
|
||||||
|
4. "negotiation": Negotiation tips specific to this price range and car type. Include what to look up beforehand (KBB, etc.), how much below asking to start, WA-specific rules (doc fees, etc.). Use markdown formatting.
|
||||||
|
|
||||||
|
5. "budget": Total budget breakdown including the car, registration, insurance, maintenance reserves. Be specific to the price range searched. Use markdown formatting.
|
||||||
|
|
||||||
|
Each value should be a string with markdown formatting for rich display. Be practical, specific, and actionable."""
|
||||||
|
|
||||||
|
text = await call_anthropic(prompt, max_tokens=4000, timeout=90)
|
||||||
|
text = text.strip()
|
||||||
|
|
||||||
|
# Try to parse JSON from response
|
||||||
|
match = re.search(r"\{.*\}", text, re.DOTALL)
|
||||||
|
if match:
|
||||||
|
try:
|
||||||
|
guide = json.loads(match.group())
|
||||||
|
# Ensure all keys exist
|
||||||
|
for key in ["top_picks", "checklist", "mechanics", "negotiation", "budget"]:
|
||||||
|
if key not in guide:
|
||||||
|
guide[key] = "No information available."
|
||||||
|
return guide
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Fallback: return the raw text in top_picks
|
||||||
|
return {
|
||||||
|
"top_picks": text or "Guide generation failed. Please try again.",
|
||||||
|
"checklist": "Run a search to generate a custom checklist.",
|
||||||
|
"mechanics": "Run a search to get mechanic recommendations.",
|
||||||
|
"negotiation": "Run a search to get negotiation tips.",
|
||||||
|
"budget": "Run a search to get a budget breakdown.",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Static files + index
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
@app.get("/", response_class=HTMLResponse)
|
@app.get("/", response_class=HTMLResponse)
|
||||||
async def index():
|
async def index():
|
||||||
html = Path("static/index.html").read_text()
|
html = Path("static/index.html").read_text()
|
||||||
return HTMLResponse(html)
|
return HTMLResponse(html)
|
||||||
|
|
||||||
|
|
||||||
app.mount("/static", StaticFiles(directory="static"), name="static")
|
app.mount("/static", StaticFiles(directory="static"), name="static")
|
||||||
|
|
||||||
|
|
||||||
@@ -47,17 +363,44 @@ app.mount("/static", StaticFiles(directory="static"), name="static")
|
|||||||
# API routes
|
# API routes
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
@app.get("/api/guide")
|
|
||||||
async def guide():
|
|
||||||
return JSONResponse(get_guide())
|
|
||||||
|
|
||||||
|
@app.post("/api/search")
|
||||||
|
async def search(request: Request):
|
||||||
|
"""Main search endpoint: parse query, crawl sites, score, generate guide."""
|
||||||
|
body = await request.json()
|
||||||
|
query = body.get("query", "").strip()
|
||||||
|
filters = body.get("filters", [])
|
||||||
|
|
||||||
|
if not query:
|
||||||
|
return JSONResponse({"error": "Please enter a search query"}, 400)
|
||||||
|
|
||||||
@app.get("/api/search")
|
|
||||||
async def search():
|
|
||||||
"""Trigger a car search using Playwright."""
|
|
||||||
try:
|
try:
|
||||||
|
# Step 1: Parse query into search parameters
|
||||||
|
params = await parse_query(query, filters)
|
||||||
|
search_summary = params.get("search_summary", query)
|
||||||
|
|
||||||
|
# Step 2: Build search URLs
|
||||||
|
cl_url = build_craigslist_url(params)
|
||||||
|
cg_url = build_cargurus_url(params)
|
||||||
|
at_url = build_autotempest_url(params)
|
||||||
|
|
||||||
|
# Step 3: Run Playwright crawl via search.py subprocess
|
||||||
|
search_args = json.dumps(
|
||||||
|
{
|
||||||
|
"craigslist_url": cl_url,
|
||||||
|
"cargurus_url": cg_url,
|
||||||
|
"autotempest_url": at_url,
|
||||||
|
"params": params,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
proc = await asyncio.create_subprocess_exec(
|
proc = await asyncio.create_subprocess_exec(
|
||||||
"uv", "run", "python", "search.py",
|
"uv",
|
||||||
|
"run",
|
||||||
|
"python",
|
||||||
|
"search.py",
|
||||||
|
"--dynamic",
|
||||||
|
search_args,
|
||||||
stdout=asyncio.subprocess.PIPE,
|
stdout=asyncio.subprocess.PIPE,
|
||||||
stderr=asyncio.subprocess.PIPE,
|
stderr=asyncio.subprocess.PIPE,
|
||||||
env={
|
env={
|
||||||
@@ -66,32 +409,55 @@ async def search():
|
|||||||
"DISPLAY": ":99",
|
"DISPLAY": ":99",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
stdout, stderr = await asyncio.wait_for(
|
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=180)
|
||||||
proc.communicate(), timeout=120
|
|
||||||
)
|
|
||||||
|
|
||||||
# Find the most recent result file
|
# Step 4: Load results
|
||||||
result_files = sorted(RESULTS_DIR.glob("search_*.json"))
|
result_files = sorted(RESULTS_DIR.glob("search_*.json"))
|
||||||
|
all_listings = []
|
||||||
if result_files:
|
if result_files:
|
||||||
data = json.loads(result_files[-1].read_text())
|
data = json.loads(result_files[-1].read_text())
|
||||||
return JSONResponse({
|
all_listings = data.get("listings", [])
|
||||||
|
|
||||||
|
# Step 5: Score and rank
|
||||||
|
for listing in all_listings:
|
||||||
|
listing["score"] = score_listing(listing, params)
|
||||||
|
all_listings.sort(key=lambda x: x["score"], reverse=True)
|
||||||
|
|
||||||
|
# Step 6: Generate guide via LLM
|
||||||
|
guide = await generate_guide(query, all_listings, params)
|
||||||
|
|
||||||
|
return JSONResponse(
|
||||||
|
{
|
||||||
"status": "ok",
|
"status": "ok",
|
||||||
"file": result_files[-1].name,
|
"search_summary": search_summary,
|
||||||
"total": data.get("total_listings", 0),
|
"total": len(all_listings),
|
||||||
"listings": data.get("listings", []),
|
"listings": all_listings,
|
||||||
"stdout": stdout.decode()[-500:] if stdout else "",
|
"guide": guide,
|
||||||
})
|
"urls_searched": {
|
||||||
return JSONResponse({
|
"craigslist": cl_url,
|
||||||
"status": "ok",
|
"cargurus": cg_url,
|
||||||
"total": 0,
|
"autotempest": at_url,
|
||||||
"listings": [],
|
},
|
||||||
"stdout": stdout.decode() if stdout else "",
|
}
|
||||||
"stderr": stderr.decode() if stderr else "",
|
)
|
||||||
})
|
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
return JSONResponse({"status": "error", "error": "Search timed out"}, 504)
|
return JSONResponse({"error": "Search timed out after 3 minutes"}, 504)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return JSONResponse({"status": "error", "error": str(e)}, 500)
|
import traceback
|
||||||
|
|
||||||
|
return JSONResponse(
|
||||||
|
{
|
||||||
|
"error": str(e),
|
||||||
|
"traceback": traceback.format_exc(),
|
||||||
|
},
|
||||||
|
500,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Keep existing endpoints
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
@app.get("/api/results")
|
@app.get("/api/results")
|
||||||
@@ -103,11 +469,13 @@ async def list_results():
|
|||||||
for f in files[:20]:
|
for f in files[:20]:
|
||||||
try:
|
try:
|
||||||
data = json.loads(f.read_text())
|
data = json.loads(f.read_text())
|
||||||
results.append({
|
results.append(
|
||||||
"filename": f.name,
|
{
|
||||||
"date": data.get("search_date", ""),
|
"filename": f.name,
|
||||||
"total": data.get("total_listings", 0),
|
"date": data.get("search_date", ""),
|
||||||
})
|
"total": data.get("total_listings", 0),
|
||||||
|
}
|
||||||
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
continue
|
continue
|
||||||
return JSONResponse(results)
|
return JSONResponse(results)
|
||||||
@@ -122,72 +490,6 @@ async def get_result(filename: str):
|
|||||||
return JSONResponse(json.loads(path.read_text()))
|
return JSONResponse(json.loads(path.read_text()))
|
||||||
|
|
||||||
|
|
||||||
@app.post("/api/rank")
|
|
||||||
async def rank_listings(request: Request):
|
|
||||||
"""Score and rank car listings."""
|
|
||||||
body = await request.json()
|
|
||||||
listings = body.get("listings", [])
|
|
||||||
|
|
||||||
def score(listing: dict) -> int:
|
|
||||||
s = 50 # base score
|
|
||||||
title = (listing.get("title", "") + " " + listing.get("url", "")).lower()
|
|
||||||
|
|
||||||
# Model bonuses
|
|
||||||
if "prius" in title:
|
|
||||||
s += 30
|
|
||||||
elif "camry hybrid" in title or "camry" in title:
|
|
||||||
s += 25
|
|
||||||
elif "insight" in title:
|
|
||||||
s += 20
|
|
||||||
elif "ioniq" in title:
|
|
||||||
s += 20
|
|
||||||
elif "civic hybrid" in title:
|
|
||||||
s += 15
|
|
||||||
|
|
||||||
# Brand bonuses
|
|
||||||
if "toyota" in title:
|
|
||||||
s += 10
|
|
||||||
elif "honda" in title:
|
|
||||||
s += 7
|
|
||||||
elif "hyundai" in title:
|
|
||||||
s += 5
|
|
||||||
|
|
||||||
# Penalties
|
|
||||||
if "nissan" in title and "leaf" in title:
|
|
||||||
s -= 25
|
|
||||||
if "altima" in title:
|
|
||||||
s -= 20
|
|
||||||
if any(w in title for w in ["salvage", "rebuilt", "flood", "junk"]):
|
|
||||||
s -= 20
|
|
||||||
|
|
||||||
# Price parsing
|
|
||||||
price_str = listing.get("price", "")
|
|
||||||
price_num = 0
|
|
||||||
m = re.search(r"\$?([\d,]+)", price_str)
|
|
||||||
if m:
|
|
||||||
price_num = int(m.group(1).replace(",", ""))
|
|
||||||
if 0 < price_num < 6000:
|
|
||||||
s += 10
|
|
||||||
elif 6000 <= price_num <= 8000:
|
|
||||||
s += 5
|
|
||||||
|
|
||||||
# Positive signals
|
|
||||||
if "one owner" in title or "single owner" in title:
|
|
||||||
s += 8
|
|
||||||
if "clean title" in title:
|
|
||||||
s += 8
|
|
||||||
|
|
||||||
return max(0, min(100, s))
|
|
||||||
|
|
||||||
scored = []
|
|
||||||
for listing in listings:
|
|
||||||
listing["score"] = score(listing)
|
|
||||||
scored.append(listing)
|
|
||||||
|
|
||||||
scored.sort(key=lambda x: x["score"], reverse=True)
|
|
||||||
return JSONResponse(scored)
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# noVNC proxy (LAST -- catch-all)
|
# noVNC proxy (LAST -- catch-all)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -1,11 +1,16 @@
|
|||||||
"""
|
"""
|
||||||
Car search crawler -- finds used hybrids under $10K near Woodinville WA.
|
Car search crawler -- finds used cars based on dynamic parameters.
|
||||||
Always runs headed on Xvfb. View live at browser.ampbox.io.
|
Always runs headed on Xvfb. View live at browser.ampbox.io.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python search.py # runs default hardcoded search
|
||||||
|
python search.py --dynamic '{...}' # runs with dynamic URLs/params from app.py
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
|
import sys
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -23,17 +28,9 @@ def extract_title_from_url(url: str) -> str:
|
|||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
|
||||||
async def search_craigslist(page) -> list[dict]:
|
async def search_craigslist(page, url: str) -> list[dict]:
|
||||||
"""Craigslist Seattle -- simple HTML, most reliable to scrape."""
|
"""Craigslist -- simple HTML, most reliable to scrape."""
|
||||||
url = (
|
print(f" Searching Craigslist: {url}")
|
||||||
"https://seattle.craigslist.org/search/cta"
|
|
||||||
"?auto_fuel_type=4"
|
|
||||||
"&max_price=10000"
|
|
||||||
"&postal=98077"
|
|
||||||
"&search_distance=50"
|
|
||||||
"&sort=date"
|
|
||||||
)
|
|
||||||
print(f" Searching Craigslist...")
|
|
||||||
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
||||||
await page.wait_for_timeout(3000)
|
await page.wait_for_timeout(3000)
|
||||||
|
|
||||||
@@ -43,29 +40,30 @@ async def search_craigslist(page) -> list[dict]:
|
|||||||
|
|
||||||
for card in cards[:50]:
|
for card in cards[:50]:
|
||||||
try:
|
try:
|
||||||
# Get the full text of the card -- title is in there
|
|
||||||
text = await card.inner_text()
|
text = await card.inner_text()
|
||||||
lines = [l.strip() for l in text.split("\n") if l.strip() and l.strip() != "•"]
|
lines = [
|
||||||
|
ln.strip()
|
||||||
|
for ln in text.split("\n")
|
||||||
|
if ln.strip() and ln.strip() != "\u2022"
|
||||||
|
]
|
||||||
|
|
||||||
# Get the link
|
link_el = await card.query_selector(
|
||||||
link_el = await card.query_selector("a.main, a.cl-app-anchor, a[href*='/d/']")
|
"a.main, a.cl-app-anchor, a[href*='/d/']"
|
||||||
|
)
|
||||||
link = await link_el.get_attribute("href") if link_el else ""
|
link = await link_el.get_attribute("href") if link_el else ""
|
||||||
|
|
||||||
# Title is usually the longest non-price, non-date text line
|
|
||||||
title = ""
|
title = ""
|
||||||
price = ""
|
price = ""
|
||||||
location = ""
|
|
||||||
for line in lines:
|
for line in lines:
|
||||||
if line.startswith("$"):
|
if line.startswith("$"):
|
||||||
price = line
|
price = line
|
||||||
elif re.match(r"^\d+/\d+$", line):
|
elif re.match(r"^\d+/\d+$", line):
|
||||||
continue # date like 5/23
|
continue
|
||||||
elif re.match(r"^\d+k?\s*mi", line, re.I):
|
elif re.match(r"^\d+k?\s*mi", line, re.I):
|
||||||
continue # mileage
|
continue
|
||||||
elif not title and len(line) > 5:
|
elif not title and len(line) > 5:
|
||||||
title = line
|
title = line
|
||||||
|
|
||||||
# Fallback: extract from URL
|
|
||||||
if not title and link:
|
if not title and link:
|
||||||
title = extract_title_from_url(link)
|
title = extract_title_from_url(link)
|
||||||
|
|
||||||
@@ -74,17 +72,18 @@ async def search_craigslist(page) -> list[dict]:
|
|||||||
if price_el:
|
if price_el:
|
||||||
price = await price_el.inner_text()
|
price = await price_el.inner_text()
|
||||||
|
|
||||||
# Get image
|
|
||||||
img_el = await card.query_selector("img[src]")
|
img_el = await card.query_selector("img[src]")
|
||||||
img = await img_el.get_attribute("src") if img_el else ""
|
img = await img_el.get_attribute("src") if img_el else ""
|
||||||
|
|
||||||
listings.append({
|
listings.append(
|
||||||
"source": "craigslist",
|
{
|
||||||
"title": title.strip(),
|
"source": "craigslist",
|
||||||
"price": price.strip(),
|
"title": title.strip(),
|
||||||
"url": link,
|
"price": price.strip(),
|
||||||
"image": img,
|
"url": link,
|
||||||
})
|
"image": img,
|
||||||
|
}
|
||||||
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -92,25 +91,20 @@ async def search_craigslist(page) -> list[dict]:
|
|||||||
return listings
|
return listings
|
||||||
|
|
||||||
|
|
||||||
async def search_cargurus(page) -> list[dict]:
|
async def search_cargurus(page, url: str) -> list[dict]:
|
||||||
"""CarGurus -- JS-heavy but good data."""
|
"""CarGurus -- JS-heavy but good data."""
|
||||||
url = (
|
print(f" Searching CarGurus: {url}")
|
||||||
"https://www.cargurus.com/Cars/inventorylisting/"
|
|
||||||
"viewDetailsFilterViewInventoryListing.action"
|
|
||||||
"?zip=98077&maxPrice=10000&fuelTypes=HYBRID"
|
|
||||||
"&distance=50&sortDir=ASC&sortType=PRICE"
|
|
||||||
)
|
|
||||||
print(f" Searching CarGurus...")
|
|
||||||
await page.goto(url, wait_until="networkidle", timeout=60000)
|
await page.goto(url, wait_until="networkidle", timeout=60000)
|
||||||
await page.wait_for_timeout(3000)
|
await page.wait_for_timeout(3000)
|
||||||
|
|
||||||
listings = []
|
listings = []
|
||||||
# Try multiple selector patterns for CarGurus
|
cards = await page.query_selector_all(
|
||||||
cards = await page.query_selector_all("article, [data-cg-ft='car-blade'], a[href*='/Cars/']")
|
"article, [data-cg-ft='car-blade'], a[href*='/Cars/']"
|
||||||
|
)
|
||||||
for card in cards[:50]:
|
for card in cards[:50]:
|
||||||
try:
|
try:
|
||||||
text = await card.inner_text()
|
text = await card.inner_text()
|
||||||
lines = [l.strip() for l in text.split("\n") if l.strip()]
|
lines = [ln.strip() for ln in text.split("\n") if ln.strip()]
|
||||||
|
|
||||||
title = lines[0] if lines else "Unknown"
|
title = lines[0] if lines else "Unknown"
|
||||||
price = ""
|
price = ""
|
||||||
@@ -129,13 +123,15 @@ async def search_cargurus(page) -> list[dict]:
|
|||||||
img_el = await card.query_selector("img[src*='cargurus']")
|
img_el = await card.query_selector("img[src*='cargurus']")
|
||||||
img = await img_el.get_attribute("src") if img_el else ""
|
img = await img_el.get_attribute("src") if img_el else ""
|
||||||
|
|
||||||
listings.append({
|
listings.append(
|
||||||
"source": "cargurus",
|
{
|
||||||
"title": title[:100].strip(),
|
"source": "cargurus",
|
||||||
"price": price.strip(),
|
"title": title[:100].strip(),
|
||||||
"url": link,
|
"price": price.strip(),
|
||||||
"image": img,
|
"url": link,
|
||||||
})
|
"image": img,
|
||||||
|
}
|
||||||
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -143,25 +139,23 @@ async def search_cargurus(page) -> list[dict]:
|
|||||||
return listings
|
return listings
|
||||||
|
|
||||||
|
|
||||||
async def search_autotempest(page) -> list[dict]:
|
async def search_autotempest(page, url: str) -> list[dict]:
|
||||||
"""AutoTempest -- meta-aggregator."""
|
"""AutoTempest -- meta-aggregator."""
|
||||||
url = (
|
print(f" Searching AutoTempest: {url}")
|
||||||
"https://www.autotempest.com/results"
|
|
||||||
"?zip=98077&maxprice=10000&fuel=hybrid&radius=50"
|
|
||||||
)
|
|
||||||
print(f" Searching AutoTempest...")
|
|
||||||
await page.goto(url, wait_until="networkidle", timeout=60000)
|
await page.goto(url, wait_until="networkidle", timeout=60000)
|
||||||
await page.wait_for_timeout(5000)
|
await page.wait_for_timeout(5000)
|
||||||
|
|
||||||
listings = []
|
listings = []
|
||||||
cards = await page.query_selector_all(".result-list-item, .listing, [class*='result']")
|
cards = await page.query_selector_all(
|
||||||
|
".result-list-item, .listing, [class*='result']"
|
||||||
|
)
|
||||||
for card in cards[:50]:
|
for card in cards[:50]:
|
||||||
try:
|
try:
|
||||||
text = await card.inner_text()
|
text = await card.inner_text()
|
||||||
if len(text.strip()) < 10:
|
if len(text.strip()) < 10:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
lines = [l.strip() for l in text.split("\n") if l.strip()]
|
lines = [ln.strip() for ln in text.split("\n") if ln.strip()]
|
||||||
title = lines[0] if lines else "Unknown"
|
title = lines[0] if lines else "Unknown"
|
||||||
price = ""
|
price = ""
|
||||||
for line in lines:
|
for line in lines:
|
||||||
@@ -175,13 +169,15 @@ async def search_autotempest(page) -> list[dict]:
|
|||||||
img_el = await card.query_selector("img[src]")
|
img_el = await card.query_selector("img[src]")
|
||||||
img = await img_el.get_attribute("src") if img_el else ""
|
img = await img_el.get_attribute("src") if img_el else ""
|
||||||
|
|
||||||
listings.append({
|
listings.append(
|
||||||
"source": "autotempest",
|
{
|
||||||
"title": title[:100].strip(),
|
"source": "autotempest",
|
||||||
"price": price.strip(),
|
"title": title[:100].strip(),
|
||||||
"url": link,
|
"price": price.strip(),
|
||||||
"image": img,
|
"url": link,
|
||||||
})
|
"image": img,
|
||||||
|
}
|
||||||
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -191,7 +187,17 @@ async def search_autotempest(page) -> list[dict]:
|
|||||||
|
|
||||||
async def main():
|
async def main():
|
||||||
print(f"Car Search -- {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
|
print(f"Car Search -- {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
|
||||||
print("Looking for: Used hybrids under $10K within 50mi of 98077 (Woodinville WA)")
|
|
||||||
|
# Parse dynamic args if provided
|
||||||
|
dynamic_config = None
|
||||||
|
if len(sys.argv) >= 3 and sys.argv[1] == "--dynamic":
|
||||||
|
dynamic_config = json.loads(sys.argv[2])
|
||||||
|
params = dynamic_config.get("params", {})
|
||||||
|
search_summary = params.get("search_summary", "dynamic search")
|
||||||
|
print(f"Dynamic search: {search_summary}")
|
||||||
|
else:
|
||||||
|
print("Default search: Used hybrids under $10K within 50mi of 98077")
|
||||||
|
|
||||||
print("=" * 70)
|
print("=" * 70)
|
||||||
|
|
||||||
all_listings: list[dict] = []
|
all_listings: list[dict] = []
|
||||||
@@ -207,28 +213,66 @@ async def main():
|
|||||||
)
|
)
|
||||||
page = await context.new_page()
|
page = await context.new_page()
|
||||||
|
|
||||||
for search_fn in [search_craigslist, search_cargurus, search_autotempest]:
|
if dynamic_config:
|
||||||
try:
|
# Dynamic search with provided URLs
|
||||||
results = await search_fn(page)
|
searches = [
|
||||||
all_listings.extend(results)
|
(search_craigslist, dynamic_config.get("craigslist_url", "")),
|
||||||
except Exception as e:
|
(search_cargurus, dynamic_config.get("cargurus_url", "")),
|
||||||
print(f" ERROR in {search_fn.__name__}: {e}")
|
(search_autotempest, dynamic_config.get("autotempest_url", "")),
|
||||||
|
]
|
||||||
|
for search_fn, url in searches:
|
||||||
|
if url:
|
||||||
|
try:
|
||||||
|
results = await search_fn(page, url)
|
||||||
|
all_listings.extend(results)
|
||||||
|
except Exception as e:
|
||||||
|
print(f" ERROR in {search_fn.__name__}: {e}")
|
||||||
|
else:
|
||||||
|
# Default hardcoded search
|
||||||
|
default_urls = {
|
||||||
|
"craigslist": (
|
||||||
|
"https://seattle.craigslist.org/search/cta"
|
||||||
|
"?auto_fuel_type=4&max_price=10000&postal=98077"
|
||||||
|
"&search_distance=50&sort=date"
|
||||||
|
),
|
||||||
|
"cargurus": (
|
||||||
|
"https://www.cargurus.com/Cars/inventorylisting/"
|
||||||
|
"viewDetailsFilterViewInventoryListing.action"
|
||||||
|
"?zip=98077&maxPrice=10000&fuelTypes=HYBRID"
|
||||||
|
"&distance=50&sortDir=ASC&sortType=PRICE"
|
||||||
|
),
|
||||||
|
"autotempest": (
|
||||||
|
"https://www.autotempest.com/results"
|
||||||
|
"?zip=98077&maxprice=10000&fuel=hybrid&radius=50"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
for name, url in default_urls.items():
|
||||||
|
search_fn = {
|
||||||
|
"craigslist": search_craigslist,
|
||||||
|
"cargurus": search_cargurus,
|
||||||
|
"autotempest": search_autotempest,
|
||||||
|
}[name]
|
||||||
|
try:
|
||||||
|
results = await search_fn(page, url)
|
||||||
|
all_listings.extend(results)
|
||||||
|
except Exception as e:
|
||||||
|
print(f" ERROR in {name}: {e}")
|
||||||
|
|
||||||
await browser.close()
|
await browser.close()
|
||||||
|
|
||||||
# Save results
|
# Save results
|
||||||
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M")
|
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M")
|
||||||
|
search_params = (
|
||||||
|
dynamic_config.get("params", {})
|
||||||
|
if dynamic_config
|
||||||
|
else {"zip": "98077", "max_price": 10000, "fuel": "hybrid", "radius_miles": 50}
|
||||||
|
)
|
||||||
output_file = RESULTS_DIR / f"search_{timestamp}.json"
|
output_file = RESULTS_DIR / f"search_{timestamp}.json"
|
||||||
with open(output_file, "w") as f:
|
with open(output_file, "w") as f:
|
||||||
json.dump(
|
json.dump(
|
||||||
{
|
{
|
||||||
"search_date": datetime.now(timezone.utc).isoformat(),
|
"search_date": datetime.now(timezone.utc).isoformat(),
|
||||||
"params": {
|
"params": search_params,
|
||||||
"zip": "98077",
|
|
||||||
"max_price": 10000,
|
|
||||||
"fuel": "hybrid",
|
|
||||||
"radius_miles": 50,
|
|
||||||
},
|
|
||||||
"total_listings": len(all_listings),
|
"total_listings": len(all_listings),
|
||||||
"listings": all_listings,
|
"listings": all_listings,
|
||||||
},
|
},
|
||||||
@@ -241,7 +285,9 @@ async def main():
|
|||||||
print(f"Saved to: {output_file}")
|
print(f"Saved to: {output_file}")
|
||||||
|
|
||||||
for listing in all_listings[:10]:
|
for listing in all_listings[:10]:
|
||||||
print(f" [{listing['source']}] {listing['price']:>8s} {listing['title'][:60]}")
|
print(
|
||||||
|
f" [{listing['source']}] {listing['price']:>8s} {listing['title'][:60]}"
|
||||||
|
)
|
||||||
|
|
||||||
if len(all_listings) > 10:
|
if len(all_listings) > 10:
|
||||||
print(f" ... and {len(all_listings) - 10} more")
|
print(f" ... and {len(all_listings) - 10} more")
|
||||||
|
|||||||
+498
-511
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user