How To Find All URLs on a Domain

<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <sitemap>
    <loc>https://example.com/sitemap-posts.xml</loc>
    <lastmod>2026-01-20</lastmod>
  </sitemap>
  <sitemap>
    <loc>https://example.com/sitemap-products.xml</loc>
    <lastmod>2026-01-19</lastmod>
  </sitemap>
  <sitemap>
    <loc>https://example.com/sitemap-categories.xml</loc>
    <lastmod>2026-01-18</lastmod>
  </sitemap>
</sitemapindex>

import requests
from bs4 import BeautifulSoup
import csv
import sys
import gzip


def fetch_sitemap_urls(sitemap_url, all_urls=None, visited=None):
    if all_urls is None:
        all_urls = []
    if visited is None:
        visited = set()

    if sitemap_url in visited:
        return all_urls
    visited.add(sitemap_url)

    headers = {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36",
        "Accept-Encoding": "gzip, deflate",
    }

    try:
        response = requests.get(sitemap_url, headers=headers, timeout=30)
        response.raise_for_status()
    except requests.RequestException as e:
        print(f"Error fetching {sitemap_url}: {e}")
        return all_urls

    content = response.content
    if (
        sitemap_url.endswith(".gz")
        or response.headers.get("Content-Encoding") == "gzip"
    ):
        try:
            content = gzip.decompress(content)
        except gzip.BadGzipFile:
            pass

    soup = BeautifulSoup(content, "lxml-xml")

    sitemap_tags = soup.find_all("sitemap")
    if sitemap_tags:
        print(f"Found sitemap index with {len(sitemap_tags)} sub-sitemaps")
        for sitemap in sitemap_tags:
            loc = sitemap.find("loc")
            if loc and loc.text:
                print(f"  Processing: {loc.text.strip()}")
                fetch_sitemap_urls(loc.text.strip(), all_urls, visited)
    else:
        url_tags = soup.find_all("url")
        for url in url_tags:
            loc = url.find("loc")
            if loc and loc.text:
                all_urls.append(loc.text.strip())
        print(f"Found {len(url_tags)} URLs in {sitemap_url}")

    return all_urls


def save_to_csv(urls, filename="urls.csv"):
    with open(filename, "w", newline="", encoding="utf-8") as f:
        writer = csv.writer(f)
        writer.writerow(["URL"])
        for url in urls:
            writer.writerow([url])
    print(f"Saved {len(urls)} URLs to {filename}")


if __name__ == "__main__":
    args = sys.argv[1:]
    sitemap_url = args[0] if args else "https://example.com/sitemap.xml"
    output_file = args[1] if len(args) > 1 else "urls.csv"

    urls = fetch_sitemap_urls(sitemap_url)
    if urls:
        save_to_csv(urls, output_file)
    print(f"\nTotal URLs found: {len(urls)}")

python sitemap_parser.py <sitemap_url> [output_file] 

python sitemap_parser.py https://example.com/sitemap.xml 
python sitemap_parser.py https://example.com/sitemap.xml output.csv 
python sitemap_parser.py https://example.com/sitemap.xml.gz compressed.csv

import { writeFileSync } from 'fs';
import { gunzipSync } from 'zlib';
import * as cheerio from 'cheerio';

const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36';

async function fetchUrl(url) {
    const response = await fetch(url, {
        headers: { 'User-Agent': USER_AGENT, 'Accept-Encoding': 'gzip, deflate' },
        signal: AbortSignal.timeout(30000)
    });

    if (!response.ok) throw new Error(`HTTP ${response.status}`);

    const buffer = await response.arrayBuffer();
    let content;

    if (url.endsWith('.gz') || response.headers.get('content-encoding') === 'gzip') {
        try {
            content = gunzipSync(Buffer.from(buffer)).toString('utf-8');
        } catch {
            content = Buffer.from(buffer).toString('utf-8');
        }
    } else {
        content = Buffer.from(buffer).toString('utf-8');
    }

    return content;
}

async function fetchSitemapUrls(sitemapUrl, allUrls = [], visited = new Set()) {
    if (visited.has(sitemapUrl)) return allUrls;
    visited.add(sitemapUrl);

    let content;
    try {
        content = await fetchUrl(sitemapUrl);
    } catch (err) {
        console.log(`Error fetching ${sitemapUrl}: ${err.message}`);
        return allUrls;
    }

    const $ = cheerio.load(content, { xmlMode: true });

    const sitemaps = $('sitemap loc');
    if (sitemaps.length > 0) {
        console.log(`Found sitemap index with ${sitemaps.length} sub-sitemaps`);
        for (const el of sitemaps.toArray()) {
            const loc = $(el).text().trim();
            if (loc) {
                console.log(`  Processing: ${loc}`);
                await fetchSitemapUrls(loc, allUrls, visited);
            }
        }
    } else {
        const urls = $('url loc');
        urls.each((_, el) => {
            const loc = $(el).text().trim();
            if (loc) allUrls.push(loc);
        });
        console.log(`Found ${urls.length} URLs in ${sitemapUrl}`);
    }

    return allUrls;
}

function saveToCsv(urls, filename = 'urls.csv') {
    writeFileSync(filename, 'URL\n' + urls.join('\n'));
    console.log(`Saved ${urls.length} URLs to ${filename}`);
}

const args = process.argv.slice(2);
const url = args[0] || 'https://example.com/sitemap.xml';
const output = args[1] || 'urls.csv';

const urls = await fetchSitemapUrls(url);
if (urls.length > 0) saveToCsv(urls, output);
console.log(`\nTotal URLs found: ${urls.length}`);

export { fetchSitemapUrls };

import scrapy
from scrapy.crawler import CrawlerProcess
from urllib.parse import urlparse
import csv
import time
import sys


def crawl_site(start_url, max_pages=500, output_file="crawled.csv"):
    found_urls = []

    class CollectorSpider(scrapy.Spider):
        name = "collector"

        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self.start_urls = [start_url]
            self.domain = urlparse(start_url).netloc
            self.visited = set()
            self.page_count = 0

        def parse(self, response):
            if self.page_count >= max_pages:
                return

            if response.url in self.visited:
                return

            self.visited.add(response.url)
            self.page_count += 1
            found_urls.append(response.url)
            print(f"[{self.page_count}/{max_pages}] {response.status} - {response.url}")

            for href in response.css("a::attr(href)").getall():
                if href and not href.startswith(
                    ("javascript:", "mailto:", "tel:", "#")
                ):
                    full_url = response.urljoin(href)
                    parsed = urlparse(full_url)

                    if parsed.netloc != self.domain:
                        continue

                    skip_ext = [
                        ".jpg",
                        ".jpeg",
                        ".png",
                        ".gif",
                        ".pdf",
                        ".css",
                        ".js",
                        ".ico",
                        ".svg",
                        ".woff",
                        ".woff2",
                        ".ttf",
                        ".mp4",
                        ".webp",
                    ]
                    if any(parsed.path.lower().endswith(ext) for ext in skip_ext):
                        continue

                    skip_paths = [
                        "/wp-admin",
                        "/wp-includes",
                        "/cart",
                        "/checkout",
                        "/api/",
                    ]
                    if any(skip in parsed.path.lower() for skip in skip_paths):
                        continue

                    if full_url not in self.visited and self.page_count < max_pages:
                        yield scrapy.Request(full_url, callback=self.parse)

    print(f"Starting crawl of {start_url}")
    print(f"Max pages: {max_pages}")
    print("-" * 50)

    start_time = time.time()

    process = CrawlerProcess(
        settings={
            "LOG_ENABLED": False,
            "USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36",
            "CONCURRENT_REQUESTS": 16,
            "CONCURRENT_REQUESTS_PER_DOMAIN": 16,
            "DOWNLOAD_DELAY": 0.1,
            "ROBOTSTXT_OBEY": False,
            "COOKIES_ENABLED": False,
        }
    )

    process.crawl(CollectorSpider)
    process.start()

    elapsed = time.time() - start_time

    print("-" * 50)
    print(f"Crawl complete. Found {len(found_urls)} pages in {elapsed:.2f}s")

    if found_urls:
        with open(output_file, "w", newline="", encoding="utf-8") as f:
            writer = csv.writer(f)
            writer.writerow(["URL"])
            for url in sorted(found_urls):
                writer.writerow([url])
        print(f"Saved to {output_file}")

    return found_urls


if __name__ == "__main__":
    args = sys.argv[1:]
    url = args[0] if args else "https://example.com"
    max_pages = int(args[1]) if len(args) > 1 else 500
    output = args[2] if len(args) > 2 else "crawled.csv"
    crawl_site(url, max_pages, output)

import asyncio
from urllib.parse import urlparse
from crawlee.crawlers import PlaywrightCrawler
from crawlee import ConcurrencySettings
import csv
import time
import sys


async def crawl_js_site(start_url, max_pages=100, output_file="js_crawled.csv"):
    domain = urlparse(start_url).netloc
    found_urls = []
    visited = set()

    print(f"Starting JS crawl of {start_url}")
    print(f"Max pages: {max_pages}")
    print("-" * 50)

    start_time = time.time()

    crawler = PlaywrightCrawler(
        max_requests_per_crawl=max_pages,
        concurrency_settings=ConcurrencySettings(min_concurrency=5, max_concurrency=20),
        headless=True,
        browser_type="chromium",
    )

    @crawler.router.default_handler
    async def handler(context):
        url = context.request.url
        parsed = urlparse(url)
        normalized = (
            f"{parsed.scheme}://{parsed.netloc}{parsed.path.rstrip('/') or '/'}"
        )

        if normalized in visited:
            return
        visited.add(normalized)

        found_urls.append(normalized)
        print(f"[{len(found_urls)}/{max_pages}] {url}")

        links = await context.page.eval_on_selector_all(
            "a[href]", "els => els.map(e => e.href)"
        )

        for link in links:
            try:
                p = urlparse(link)
                if p.netloc != domain:
                    continue

                skip = [
                    ".jpg",
                    ".jpeg",
                    ".png",
                    ".gif",
                    ".pdf",
                    ".css",
                    ".js",
                    ".ico",
                    ".svg",
                    ".mp4",
                    ".webp",
                    ".zip",
                ]
                if any(p.path.lower().endswith(ext) for ext in skip):
                    continue

                clean = f"{p.scheme}://{p.netloc}{p.path.rstrip('/') or '/'}"
                if clean not in visited and len(found_urls) < max_pages:
                    await context.add_requests([clean])
            except:
                pass

    await crawler.run([start_url])

    elapsed = time.time() - start_time
    print("-" * 50)
    print(f"Crawl complete. Found {len(found_urls)} pages in {elapsed:.2f}s")
    print(f"Speed: {len(found_urls)/elapsed:.2f} pages/sec")

    if found_urls:
        with open(output_file, "w", newline="", encoding="utf-8") as f:
            writer = csv.writer(f)
            writer.writerow(["URL"])
            for url in sorted(found_urls):
                writer.writerow([url])
        print(f"Saved to {output_file}")

    return found_urls


if __name__ == "__main__":
    args = sys.argv[1:]
    url = args[0] if args else "https://example.com"
    max_pages = int(args[1]) if len(args) > 1 else 100
    output = args[2] if len(args) > 2 else "js_crawled.csv"
    asyncio.run(crawl_js_site(url, max_pages, output))