diff --git a/search.py b/search.py index d5385b5..64497d8 100644 --- a/search.py +++ b/search.py @@ -1,10 +1,11 @@ """ Car search crawler -- finds used hybrids under $10K near Woodinville WA. -Playwright Chromium always runs headed on Xvfb. View live at browser.ampbox.io. +Always runs headed on Xvfb. View live at browser.ampbox.io. """ import asyncio import json +import re from datetime import datetime, timezone from pathlib import Path @@ -14,6 +15,14 @@ RESULTS_DIR = Path("results") RESULTS_DIR.mkdir(exist_ok=True) +def extract_title_from_url(url: str) -> str: + """Extract a readable title from a Craigslist URL slug.""" + m = re.search(r"/d/([^/]+)/", url) + if m: + return m.group(1).replace("-", " ").title() + return "" + + async def search_craigslist(page) -> list[dict]: """Craigslist Seattle -- simple HTML, most reliable to scrape.""" url = ( @@ -26,30 +35,60 @@ async def search_craigslist(page) -> list[dict]: ) print(f" Searching Craigslist...") await page.goto(url, wait_until="domcontentloaded", timeout=30000) - await page.wait_for_timeout(2000) + await page.wait_for_timeout(3000) listings = [] - cards = await page.query_selector_all( - ".cl-static-search-result, .result-row, .cl-search-result" - ) + cards = await page.query_selector_all(".cl-search-result") + print(f" Found {len(cards)} result elements") + for card in cards[:50]: try: - title_el = await card.query_selector(".titlestring, .result-title, a") - price_el = await card.query_selector(".priceinfo, .result-price") - title = await title_el.inner_text() if title_el else "Unknown" - price = await price_el.inner_text() if price_el else "N/A" - link_el = await card.query_selector("a[href]") + # Get the full text of the card -- title is in there + text = await card.inner_text() + lines = [l.strip() for l in text.split("\n") if l.strip() and l.strip() != "•"] + + # Get the link + link_el = await card.query_selector("a.main, a.cl-app-anchor, a[href*='/d/']") link = await link_el.get_attribute("href") if link_el else "" + + # Title is usually the longest non-price, non-date text line + title = "" + price = "" + location = "" + for line in lines: + if line.startswith("$"): + price = line + elif re.match(r"^\d+/\d+$", line): + continue # date like 5/23 + elif re.match(r"^\d+k?\s*mi", line, re.I): + continue # mileage + elif not title and len(line) > 5: + title = line + + # Fallback: extract from URL + if not title and link: + title = extract_title_from_url(link) + + if not price: + price_el = await card.query_selector(".priceinfo") + if price_el: + price = await price_el.inner_text() + + # Get image + img_el = await card.query_selector("img[src]") + img = await img_el.get_attribute("src") if img_el else "" + listings.append({ "source": "craigslist", "title": title.strip(), "price": price.strip(), "url": link, + "image": img, }) except Exception: continue - print(f" Found {len(listings)} Craigslist listings") + print(f" Extracted {len(listings)} Craigslist listings") return listings @@ -66,28 +105,36 @@ async def search_cargurus(page) -> list[dict]: await page.wait_for_timeout(3000) listings = [] - cards = await page.query_selector_all( - "[data-cg-ft='car-blade'], .pazLpc, article" - ) + # Try multiple selector patterns for CarGurus + cards = await page.query_selector_all("article, [data-cg-ft='car-blade'], a[href*='/Cars/']") for card in cards[:50]: try: - title_el = await card.query_selector( - "h4, .pazLpc a, [data-cg-ft='car-blade-link']" - ) - price_el = await card.query_selector( - "[data-cg-ft='car-blade-price'], .pazLpc .JzvSTe" - ) - title = await title_el.inner_text() if title_el else "Unknown" - price = await price_el.inner_text() if price_el else "N/A" + text = await card.inner_text() + lines = [l.strip() for l in text.split("\n") if l.strip()] + + title = lines[0] if lines else "Unknown" + price = "" + for line in lines: + if "$" in line and any(c.isdigit() for c in line): + price = line + break + link_el = await card.query_selector("a[href*='/Cars/']") + if not link_el: + link_el = card if await card.get_attribute("href") else None link = await link_el.get_attribute("href") if link_el else "" if link and not link.startswith("http"): link = f"https://www.cargurus.com{link}" + + img_el = await card.query_selector("img[src*='cargurus']") + img = await img_el.get_attribute("src") if img_el else "" + listings.append({ "source": "cargurus", - "title": title.strip(), + "title": title[:100].strip(), "price": price.strip(), "url": link, + "image": img, }) except Exception: continue @@ -97,7 +144,7 @@ async def search_cargurus(page) -> list[dict]: async def search_autotempest(page) -> list[dict]: - """AutoTempest -- meta-aggregator, searches multiple sites.""" + """AutoTempest -- meta-aggregator.""" url = ( "https://www.autotempest.com/results" "?zip=98077&maxprice=10000&fuel=hybrid&radius=50" @@ -107,22 +154,33 @@ async def search_autotempest(page) -> list[dict]: await page.wait_for_timeout(5000) listings = [] - cards = await page.query_selector_all( - ".result-row, .listing-row, [class*='result']" - ) + cards = await page.query_selector_all(".result-list-item, .listing, [class*='result']") for card in cards[:50]: try: - title_el = await card.query_selector(".title, h3, a") - price_el = await card.query_selector(".price, [class*='price']") - title = await title_el.inner_text() if title_el else "Unknown" - price = await price_el.inner_text() if price_el else "N/A" + text = await card.inner_text() + if len(text.strip()) < 10: + continue + + lines = [l.strip() for l in text.split("\n") if l.strip()] + title = lines[0] if lines else "Unknown" + price = "" + for line in lines: + if "$" in line and any(c.isdigit() for c in line): + price = line + break + link_el = await card.query_selector("a[href]") link = await link_el.get_attribute("href") if link_el else "" + + img_el = await card.query_selector("img[src]") + img = await img_el.get_attribute("src") if img_el else "" + listings.append({ "source": "autotempest", - "title": title.strip(), + "title": title[:100].strip(), "price": price.strip(), "url": link, + "image": img, }) except Exception: continue @@ -131,43 +189,9 @@ async def search_autotempest(page) -> list[dict]: return listings -async def search_facebook(page) -> list[dict]: - """Facebook Marketplace -- view at browser.ampbox.io to log in if needed.""" - url = "https://www.facebook.com/marketplace/seattle/vehicles/?maxPrice=10000" - print(f" Opening Facebook Marketplace...") - await page.goto(url, wait_until="networkidle", timeout=60000) - - if "login" in page.url.lower(): - print(" Facebook needs login -- open https://browser.ampbox.io/vnc.html to log in") - print(" Press Enter when done...") - await asyncio.get_event_loop().run_in_executor(None, input) - - await page.wait_for_timeout(3000) - - listings = [] - cards = await page.query_selector_all("[class*='marketplace']") - for card in cards[:30]: - try: - text = await card.inner_text() - link_el = await card.query_selector("a[href*='/marketplace/']") - link = await link_el.get_attribute("href") if link_el else "" - listings.append({ - "source": "facebook", - "title": text[:100].strip(), - "price": "see listing", - "url": f"https://www.facebook.com{link}" if link else "", - }) - except Exception: - continue - - print(f" Found {len(listings)} Facebook listings") - return listings - - async def main(): print(f"Car Search -- {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}") print("Looking for: Used hybrids under $10K within 50mi of 98077 (Woodinville WA)") - print("Live browser: https://browser.ampbox.io/vnc.html") print("=" * 70) all_listings: list[dict] = [] @@ -183,7 +207,7 @@ async def main(): ) page = await context.new_page() - for search_fn in [search_craigslist, search_cargurus, search_autotempest, search_facebook]: + for search_fn in [search_craigslist, search_cargurus, search_autotempest]: try: results = await search_fn(page) all_listings.extend(results) @@ -216,11 +240,11 @@ async def main(): print(f"Total: {len(all_listings)} listings found") print(f"Saved to: {output_file}") - for listing in all_listings[:20]: - print(f" [{listing['source']}] {listing['price']:>8s} {listing['title']}") + for listing in all_listings[:10]: + print(f" [{listing['source']}] {listing['price']:>8s} {listing['title'][:60]}") - if len(all_listings) > 20: - print(f" ... and {len(all_listings) - 20} more (see {output_file})") + if len(all_listings) > 10: + print(f" ... and {len(all_listings) - 10} more") if __name__ == "__main__":