ken/research-workbench
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: extract real titles from Craigslist listings, add image URLs

Browse Source 
- Parse .cl-search-result inner text for title, price, location
- Fallback to URL slug extraction for missing titles
- Capture listing images for display in UI
- AutoTempest scraper also improved with better selectors
This commit is contained in:
Ken
2026-05-25 22:57:14 +00:00
parent 8dfafcc63d
commit f30af4aefd
1 changed files with 95 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
search.py
+95 -71
View File
@@ -1,10 +1,11 @@
"""
Car search crawler -- finds used hybrids under $10K near Woodinville WA.
Playwright Chromium always runs headed on Xvfb. View live at browser.ampbox.io.
Always runs headed on Xvfb. View live at browser.ampbox.io.
"""
import asyncio
import json
import re
from datetime import datetime, timezone
from pathlib import Path
@@ -14,6 +15,14 @@ RESULTS_DIR = Path("results")
RESULTS_DIR.mkdir(exist_ok=True)
def extract_title_from_url(url: str) -> str:
"""Extract a readable title from a Craigslist URL slug."""
m = re.search(r"/d/([^/]+)/", url)
if m:
return m.group(1).replace("-", " ").title()
return ""
async def search_craigslist(page) -> list[dict]:
"""Craigslist Seattle -- simple HTML, most reliable to scrape."""
url = (
@@ -26,30 +35,60 @@ async def search_craigslist(page) -> list[dict]:
)
print(f" Searching Craigslist...")
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
await page.wait_for_timeout(2000)
await page.wait_for_timeout(3000)
listings = []
cards = await page.query_selector_all(
".cl-static-search-result, .result-row, .cl-search-result"
)
cards = await page.query_selector_all(".cl-search-result")
print(f" Found {len(cards)} result elements")
for card in cards[:50]:
try:
title_el = await card.query_selector(".titlestring, .result-title, a")
price_el = await card.query_selector(".priceinfo, .result-price")
title = await title_el.inner_text() if title_el else "Unknown"
price = await price_el.inner_text() if price_el else "N/A"
link_el = await card.query_selector("a[href]")
# Get the full text of the card -- title is in there
text = await card.inner_text()
lines = [l.strip() for l in text.split("\n") if l.strip() and l.strip() != "•"]
# Get the link
link_el = await card.query_selector("a.main, a.cl-app-anchor, a[href*='/d/']")
link = await link_el.get_attribute("href") if link_el else ""
# Title is usually the longest non-price, non-date text line
title = ""
price = ""
location = ""
for line in lines:
if line.startswith("$"):
price = line
elif re.match(r"^\d+/\d+$", line):
continue # date like 5/23
elif re.match(r"^\d+k?\s*mi", line, re.I):
continue # mileage
elif not title and len(line) > 5:
title = line
# Fallback: extract from URL
if not title and link:
title = extract_title_from_url(link)
if not price:
price_el = await card.query_selector(".priceinfo")
if price_el:
price = await price_el.inner_text()
# Get image
img_el = await card.query_selector("img[src]")
img = await img_el.get_attribute("src") if img_el else ""
listings.append({
"source": "craigslist",
"title": title.strip(),
"price": price.strip(),
"url": link,
"image": img,
})
except Exception:
continue
print(f" Found {len(listings)} Craigslist listings")
print(f" Extracted {len(listings)} Craigslist listings")
return listings
@@ -66,28 +105,36 @@ async def search_cargurus(page) -> list[dict]:
await page.wait_for_timeout(3000)
listings = []
cards = await page.query_selector_all(
"[data-cg-ft='car-blade'], .pazLpc, article"
)
# Try multiple selector patterns for CarGurus
cards = await page.query_selector_all("article, [data-cg-ft='car-blade'], a[href*='/Cars/']")
for card in cards[:50]:
try:
title_el = await card.query_selector(
"h4, .pazLpc a, [data-cg-ft='car-blade-link']"
)
price_el = await card.query_selector(
"[data-cg-ft='car-blade-price'], .pazLpc .JzvSTe"
)
title = await title_el.inner_text() if title_el else "Unknown"
price = await price_el.inner_text() if price_el else "N/A"
text = await card.inner_text()
lines = [l.strip() for l in text.split("\n") if l.strip()]
title = lines[0] if lines else "Unknown"
price = ""
for line in lines:
if "$" in line and any(c.isdigit() for c in line):
price = line
break
link_el = await card.query_selector("a[href*='/Cars/']")
if not link_el:
link_el = card if await card.get_attribute("href") else None
link = await link_el.get_attribute("href") if link_el else ""
if link and not link.startswith("http"):
link = f"https://www.cargurus.com{link}"
img_el = await card.query_selector("img[src*='cargurus']")
img = await img_el.get_attribute("src") if img_el else ""
listings.append({
"source": "cargurus",
"title": title.strip(),
"title": title[:100].strip(),
"price": price.strip(),
"url": link,
"image": img,
})
except Exception:
continue
@@ -97,7 +144,7 @@ async def search_cargurus(page) -> list[dict]:
async def search_autotempest(page) -> list[dict]:
"""AutoTempest -- meta-aggregator, searches multiple sites."""
"""AutoTempest -- meta-aggregator."""
url = (
"https://www.autotempest.com/results"
"?zip=98077&maxprice=10000&fuel=hybrid&radius=50"
@@ -107,22 +154,33 @@ async def search_autotempest(page) -> list[dict]:
await page.wait_for_timeout(5000)
listings = []
cards = await page.query_selector_all(
".result-row, .listing-row, [class*='result']"
)
cards = await page.query_selector_all(".result-list-item, .listing, [class*='result']")
for card in cards[:50]:
try:
title_el = await card.query_selector(".title, h3, a")
price_el = await card.query_selector(".price, [class*='price']")
title = await title_el.inner_text() if title_el else "Unknown"
price = await price_el.inner_text() if price_el else "N/A"
text = await card.inner_text()
if len(text.strip()) < 10:
continue
lines = [l.strip() for l in text.split("\n") if l.strip()]
title = lines[0] if lines else "Unknown"
price = ""
for line in lines:
if "$" in line and any(c.isdigit() for c in line):
price = line
break
link_el = await card.query_selector("a[href]")
link = await link_el.get_attribute("href") if link_el else ""
img_el = await card.query_selector("img[src]")
img = await img_el.get_attribute("src") if img_el else ""
listings.append({
"source": "autotempest",
"title": title.strip(),
"title": title[:100].strip(),
"price": price.strip(),
"url": link,
"image": img,
})
except Exception:
continue
@@ -131,43 +189,9 @@ async def search_autotempest(page) -> list[dict]:
return listings
async def search_facebook(page) -> list[dict]:
"""Facebook Marketplace -- view at browser.ampbox.io to log in if needed."""
url = "https://www.facebook.com/marketplace/seattle/vehicles/?maxPrice=10000"
print(f" Opening Facebook Marketplace...")
await page.goto(url, wait_until="networkidle", timeout=60000)
if "login" in page.url.lower():
print(" Facebook needs login -- open https://browser.ampbox.io/vnc.html to log in")
print(" Press Enter when done...")
await asyncio.get_event_loop().run_in_executor(None, input)
await page.wait_for_timeout(3000)
listings = []
cards = await page.query_selector_all("[class*='marketplace']")
for card in cards[:30]:
try:
text = await card.inner_text()
link_el = await card.query_selector("a[href*='/marketplace/']")
link = await link_el.get_attribute("href") if link_el else ""
listings.append({
"source": "facebook",
"title": text[:100].strip(),
"price": "see listing",
"url": f"https://www.facebook.com{link}" if link else "",
})
except Exception:
continue
print(f" Found {len(listings)} Facebook listings")
return listings
async def main():
print(f"Car Search -- {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
print("Looking for: Used hybrids under $10K within 50mi of 98077 (Woodinville WA)")
print("Live browser: https://browser.ampbox.io/vnc.html")
print("=" * 70)
all_listings: list[dict] = []
@@ -183,7 +207,7 @@ async def main():
)
page = await context.new_page()
for search_fn in [search_craigslist, search_cargurus, search_autotempest, search_facebook]:
for search_fn in [search_craigslist, search_cargurus, search_autotempest]:
try:
results = await search_fn(page)
all_listings.extend(results)
@@ -216,11 +240,11 @@ async def main():
print(f"Total: {len(all_listings)} listings found")
print(f"Saved to: {output_file}")
for listing in all_listings[:20]:
print(f" [{listing['source']}] {listing['price']:>8s} {listing['title']}")
for listing in all_listings[:10]:
print(f" [{listing['source']}] {listing['price']:>8s} {listing['title'][:60]}")
if len(all_listings) > 20:
print(f" ... and {len(all_listings) - 20} more (see {output_file})")
if len(all_listings) > 10:
print(f" ... and {len(all_listings) - 10} more")
if __name__ == "__main__":