URL to Markdown for SEO — Competitive Content Analysis

SERP-to-brief Python pipeline (OSS — no MDisBetter API needed)

# Install: pip install trafilatura httpx
import asyncio
import httpx
import trafilatura

async def url_to_md(client: httpx.AsyncClient, url: str) -> str:
    r = await client.get(url, timeout=30, follow_redirects=True,
                         headers={"User-Agent": "Mozilla/5.0 (compatible; SERP-brief/1.0)"})
    md = trafilatura.extract(r.text, output_format="markdown",
                             include_links=True, include_tables=True)
    return md or ""

async def main(serp_urls):
    async with httpx.AsyncClient() as client:
        mds = await asyncio.gather(*[url_to_md(client, u) for u in serp_urls])
    corpus = "\n\n---\n\n".join(f"## {u}\n\n{md}" for u, md in zip(serp_urls, mds))
    open("brief-corpus.md", "w", encoding="utf-8").write(corpus)
    # Now feed brief-corpus.md to Claude with: "Extract common H2 patterns and gaps"

asyncio.run(main([
    "https://example.com/article-1",
    "https://example.com/article-2",
    # ...top 10 SERP results
]))
# For ad-hoc URLs you don't want to script, paste them into
# mdisbetter.com/convert/url-to-markdown and skip the code path entirely.