Give your MCP client a web-fetching tool.

import os
from urllib.parse import quote, urlsplit, urlunsplit
import httpx
from mcp.server.fastmcp import FastMCP

server = urlsplit(os.environ['PROXY_SERVER'])
if server.scheme not in ('http', 'https') or not server.hostname or server.username or server.password or server.query or server.fragment or server.path not in ('', '/'):
    raise ValueError('Use an HTTP(S) proxy server URL without credentials, a path or a query.')
username = quote(os.environ['PROXY_USERNAME'], safe='')
password = quote(os.environ['PROXY_PASSWORD'], safe='')
proxy_url = urlunsplit((server.scheme, f'{username}:{password}@{server.netloc}', '', '', ''))
target = os.environ.get('TARGET_URL', 'https://example.com/')
if urlsplit(target).scheme not in ('http', 'https'):
    raise ValueError('TARGET_URL must be an HTTP(S) page.')

mcp = FastMCP('Crawlhub page reader')

@mcp.tool()
def fetch_configured_page() -> dict:
    """Fetch the public page configured by the server operator."""
    with httpx.Client(proxy=proxy_url, timeout=30, trust_env=False,
                      follow_redirects=False) as client:
        with client.stream('GET', target) as response:
            response.raise_for_status()
            # Keep model context and download size bounded.
            parts = bytearray()
            limit = 20000
            truncated = False
            for chunk in response.iter_bytes():
                remaining = limit - len(parts)
                parts.extend(chunk[:remaining])
                if len(chunk) > remaining:
                    truncated = True
                    break
            return {
                'url': str(response.url),
                'status': response.status_code,
                'content_type': response.headers.get('content-type', ''),
                'body': parts.decode(response.encoding or 'utf-8', errors='replace'),
                'truncated': truncated,
            }

if __name__ == '__main__':
    mcp.run(transport='stdio')