feat: LLM-powered search with free-text query, filter toggles, and auto-generated guide
- Search box: type anything, toggle filter pills (hybrid, price range, distance, etc.) - Backend calls Anthropic to parse query into search params - Playwright crawls Craigslist, CarGurus, AutoTempest with dynamic URLs - Results scored/ranked with car-specific scoring engine - Second Anthropic call generates a custom buying guide from actual results - Guide sections: top picks, checklist, mechanics, negotiation, budget - All visible live in the browser panel via noVNC
This commit is contained in:
Ken
@@ -1,11 +1,16 @@
|
||||
"""
|
||||
Car search crawler -- finds used hybrids under $10K near Woodinville WA.
|
||||
Car search crawler -- finds used cars based on dynamic parameters.
|
||||
Always runs headed on Xvfb. View live at browser.ampbox.io.
|
||||
|
||||
Usage:
|
||||
python search.py # runs default hardcoded search
|
||||
python search.py --dynamic '{...}' # runs with dynamic URLs/params from app.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
@@ -23,17 +28,9 @@ def extract_title_from_url(url: str) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
async def search_craigslist(page) -> list[dict]:
|
||||
"""Craigslist Seattle -- simple HTML, most reliable to scrape."""
|
||||
url = (
|
||||
"https://seattle.craigslist.org/search/cta"
|
||||
"?auto_fuel_type=4"
|
||||
"&max_price=10000"
|
||||
"&postal=98077"
|
||||
"&search_distance=50"
|
||||
"&sort=date"
|
||||
)
|
||||
print(f" Searching Craigslist...")
|
||||
async def search_craigslist(page, url: str) -> list[dict]:
|
||||
"""Craigslist -- simple HTML, most reliable to scrape."""
|
||||
print(f" Searching Craigslist: {url}")
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
||||
await page.wait_for_timeout(3000)
|
||||
|
||||
@@ -43,29 +40,30 @@ async def search_craigslist(page) -> list[dict]:
|
||||
|
||||
for card in cards[:50]:
|
||||
try:
|
||||
# Get the full text of the card -- title is in there
|
||||
text = await card.inner_text()
|
||||
lines = [l.strip() for l in text.split("\n") if l.strip() and l.strip() != "•"]
|
||||
lines = [
|
||||
ln.strip()
|
||||
for ln in text.split("\n")
|
||||
if ln.strip() and ln.strip() != "\u2022"
|
||||
]
|
||||
|
||||
# Get the link
|
||||
link_el = await card.query_selector("a.main, a.cl-app-anchor, a[href*='/d/']")
|
||||
link_el = await card.query_selector(
|
||||
"a.main, a.cl-app-anchor, a[href*='/d/']"
|
||||
)
|
||||
link = await link_el.get_attribute("href") if link_el else ""
|
||||
|
||||
# Title is usually the longest non-price, non-date text line
|
||||
title = ""
|
||||
price = ""
|
||||
location = ""
|
||||
for line in lines:
|
||||
if line.startswith("$"):
|
||||
price = line
|
||||
elif re.match(r"^\d+/\d+$", line):
|
||||
continue # date like 5/23
|
||||
continue
|
||||
elif re.match(r"^\d+k?\s*mi", line, re.I):
|
||||
continue # mileage
|
||||
continue
|
||||
elif not title and len(line) > 5:
|
||||
title = line
|
||||
|
||||
# Fallback: extract from URL
|
||||
if not title and link:
|
||||
title = extract_title_from_url(link)
|
||||
|
||||
@@ -74,17 +72,18 @@ async def search_craigslist(page) -> list[dict]:
|
||||
if price_el:
|
||||
price = await price_el.inner_text()
|
||||
|
||||
# Get image
|
||||
img_el = await card.query_selector("img[src]")
|
||||
img = await img_el.get_attribute("src") if img_el else ""
|
||||
|
||||
listings.append({
|
||||
"source": "craigslist",
|
||||
"title": title.strip(),
|
||||
"price": price.strip(),
|
||||
"url": link,
|
||||
"image": img,
|
||||
})
|
||||
listings.append(
|
||||
{
|
||||
"source": "craigslist",
|
||||
"title": title.strip(),
|
||||
"price": price.strip(),
|
||||
"url": link,
|
||||
"image": img,
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
@@ -92,25 +91,20 @@ async def search_craigslist(page) -> list[dict]:
|
||||
return listings
|
||||
|
||||
|
||||
async def search_cargurus(page) -> list[dict]:
|
||||
async def search_cargurus(page, url: str) -> list[dict]:
|
||||
"""CarGurus -- JS-heavy but good data."""
|
||||
url = (
|
||||
"https://www.cargurus.com/Cars/inventorylisting/"
|
||||
"viewDetailsFilterViewInventoryListing.action"
|
||||
"?zip=98077&maxPrice=10000&fuelTypes=HYBRID"
|
||||
"&distance=50&sortDir=ASC&sortType=PRICE"
|
||||
)
|
||||
print(f" Searching CarGurus...")
|
||||
print(f" Searching CarGurus: {url}")
|
||||
await page.goto(url, wait_until="networkidle", timeout=60000)
|
||||
await page.wait_for_timeout(3000)
|
||||
|
||||
listings = []
|
||||
# Try multiple selector patterns for CarGurus
|
||||
cards = await page.query_selector_all("article, [data-cg-ft='car-blade'], a[href*='/Cars/']")
|
||||
cards = await page.query_selector_all(
|
||||
"article, [data-cg-ft='car-blade'], a[href*='/Cars/']"
|
||||
)
|
||||
for card in cards[:50]:
|
||||
try:
|
||||
text = await card.inner_text()
|
||||
lines = [l.strip() for l in text.split("\n") if l.strip()]
|
||||
lines = [ln.strip() for ln in text.split("\n") if ln.strip()]
|
||||
|
||||
title = lines[0] if lines else "Unknown"
|
||||
price = ""
|
||||
@@ -129,13 +123,15 @@ async def search_cargurus(page) -> list[dict]:
|
||||
img_el = await card.query_selector("img[src*='cargurus']")
|
||||
img = await img_el.get_attribute("src") if img_el else ""
|
||||
|
||||
listings.append({
|
||||
"source": "cargurus",
|
||||
"title": title[:100].strip(),
|
||||
"price": price.strip(),
|
||||
"url": link,
|
||||
"image": img,
|
||||
})
|
||||
listings.append(
|
||||
{
|
||||
"source": "cargurus",
|
||||
"title": title[:100].strip(),
|
||||
"price": price.strip(),
|
||||
"url": link,
|
||||
"image": img,
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
@@ -143,25 +139,23 @@ async def search_cargurus(page) -> list[dict]:
|
||||
return listings
|
||||
|
||||
|
||||
async def search_autotempest(page) -> list[dict]:
|
||||
async def search_autotempest(page, url: str) -> list[dict]:
|
||||
"""AutoTempest -- meta-aggregator."""
|
||||
url = (
|
||||
"https://www.autotempest.com/results"
|
||||
"?zip=98077&maxprice=10000&fuel=hybrid&radius=50"
|
||||
)
|
||||
print(f" Searching AutoTempest...")
|
||||
print(f" Searching AutoTempest: {url}")
|
||||
await page.goto(url, wait_until="networkidle", timeout=60000)
|
||||
await page.wait_for_timeout(5000)
|
||||
|
||||
listings = []
|
||||
cards = await page.query_selector_all(".result-list-item, .listing, [class*='result']")
|
||||
cards = await page.query_selector_all(
|
||||
".result-list-item, .listing, [class*='result']"
|
||||
)
|
||||
for card in cards[:50]:
|
||||
try:
|
||||
text = await card.inner_text()
|
||||
if len(text.strip()) < 10:
|
||||
continue
|
||||
|
||||
lines = [l.strip() for l in text.split("\n") if l.strip()]
|
||||
lines = [ln.strip() for ln in text.split("\n") if ln.strip()]
|
||||
title = lines[0] if lines else "Unknown"
|
||||
price = ""
|
||||
for line in lines:
|
||||
@@ -175,13 +169,15 @@ async def search_autotempest(page) -> list[dict]:
|
||||
img_el = await card.query_selector("img[src]")
|
||||
img = await img_el.get_attribute("src") if img_el else ""
|
||||
|
||||
listings.append({
|
||||
"source": "autotempest",
|
||||
"title": title[:100].strip(),
|
||||
"price": price.strip(),
|
||||
"url": link,
|
||||
"image": img,
|
||||
})
|
||||
listings.append(
|
||||
{
|
||||
"source": "autotempest",
|
||||
"title": title[:100].strip(),
|
||||
"price": price.strip(),
|
||||
"url": link,
|
||||
"image": img,
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
@@ -191,7 +187,17 @@ async def search_autotempest(page) -> list[dict]:
|
||||
|
||||
async def main():
|
||||
print(f"Car Search -- {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
|
||||
print("Looking for: Used hybrids under $10K within 50mi of 98077 (Woodinville WA)")
|
||||
|
||||
# Parse dynamic args if provided
|
||||
dynamic_config = None
|
||||
if len(sys.argv) >= 3 and sys.argv[1] == "--dynamic":
|
||||
dynamic_config = json.loads(sys.argv[2])
|
||||
params = dynamic_config.get("params", {})
|
||||
search_summary = params.get("search_summary", "dynamic search")
|
||||
print(f"Dynamic search: {search_summary}")
|
||||
else:
|
||||
print("Default search: Used hybrids under $10K within 50mi of 98077")
|
||||
|
||||
print("=" * 70)
|
||||
|
||||
all_listings: list[dict] = []
|
||||
@@ -207,28 +213,66 @@ async def main():
|
||||
)
|
||||
page = await context.new_page()
|
||||
|
||||
for search_fn in [search_craigslist, search_cargurus, search_autotempest]:
|
||||
try:
|
||||
results = await search_fn(page)
|
||||
all_listings.extend(results)
|
||||
except Exception as e:
|
||||
print(f" ERROR in {search_fn.__name__}: {e}")
|
||||
if dynamic_config:
|
||||
# Dynamic search with provided URLs
|
||||
searches = [
|
||||
(search_craigslist, dynamic_config.get("craigslist_url", "")),
|
||||
(search_cargurus, dynamic_config.get("cargurus_url", "")),
|
||||
(search_autotempest, dynamic_config.get("autotempest_url", "")),
|
||||
]
|
||||
for search_fn, url in searches:
|
||||
if url:
|
||||
try:
|
||||
results = await search_fn(page, url)
|
||||
all_listings.extend(results)
|
||||
except Exception as e:
|
||||
print(f" ERROR in {search_fn.__name__}: {e}")
|
||||
else:
|
||||
# Default hardcoded search
|
||||
default_urls = {
|
||||
"craigslist": (
|
||||
"https://seattle.craigslist.org/search/cta"
|
||||
"?auto_fuel_type=4&max_price=10000&postal=98077"
|
||||
"&search_distance=50&sort=date"
|
||||
),
|
||||
"cargurus": (
|
||||
"https://www.cargurus.com/Cars/inventorylisting/"
|
||||
"viewDetailsFilterViewInventoryListing.action"
|
||||
"?zip=98077&maxPrice=10000&fuelTypes=HYBRID"
|
||||
"&distance=50&sortDir=ASC&sortType=PRICE"
|
||||
),
|
||||
"autotempest": (
|
||||
"https://www.autotempest.com/results"
|
||||
"?zip=98077&maxprice=10000&fuel=hybrid&radius=50"
|
||||
),
|
||||
}
|
||||
for name, url in default_urls.items():
|
||||
search_fn = {
|
||||
"craigslist": search_craigslist,
|
||||
"cargurus": search_cargurus,
|
||||
"autotempest": search_autotempest,
|
||||
}[name]
|
||||
try:
|
||||
results = await search_fn(page, url)
|
||||
all_listings.extend(results)
|
||||
except Exception as e:
|
||||
print(f" ERROR in {name}: {e}")
|
||||
|
||||
await browser.close()
|
||||
|
||||
# Save results
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M")
|
||||
search_params = (
|
||||
dynamic_config.get("params", {})
|
||||
if dynamic_config
|
||||
else {"zip": "98077", "max_price": 10000, "fuel": "hybrid", "radius_miles": 50}
|
||||
)
|
||||
output_file = RESULTS_DIR / f"search_{timestamp}.json"
|
||||
with open(output_file, "w") as f:
|
||||
json.dump(
|
||||
{
|
||||
"search_date": datetime.now(timezone.utc).isoformat(),
|
||||
"params": {
|
||||
"zip": "98077",
|
||||
"max_price": 10000,
|
||||
"fuel": "hybrid",
|
||||
"radius_miles": 50,
|
||||
},
|
||||
"params": search_params,
|
||||
"total_listings": len(all_listings),
|
||||
"listings": all_listings,
|
||||
},
|
||||
@@ -241,7 +285,9 @@ async def main():
|
||||
print(f"Saved to: {output_file}")
|
||||
|
||||
for listing in all_listings[:10]:
|
||||
print(f" [{listing['source']}] {listing['price']:>8s} {listing['title'][:60]}")
|
||||
print(
|
||||
f" [{listing['source']}] {listing['price']:>8s} {listing['title'][:60]}"
|
||||
)
|
||||
|
||||
if len(all_listings) > 10:
|
||||
print(f" ... and {len(all_listings) - 10} more")
|
||||
|
||||
Reference in New Issue
Block a user