設計網路爬蟲（Web Crawler）

public class UrlFrontier
{
    // 前端 Queue：按優先級分層
    private readonly PriorityQueue<CrawlUrl, int> _priorityQueue = new();
 
    // 後端 Queue：按域名分組，確保 Politeness
    private readonly Dictionary<string, Queue<CrawlUrl>> _hostQueues = new();
    private readonly Dictionary<string, DateTime> _lastAccessTime = new();
    private readonly TimeSpan _politenessDelay = TimeSpan.FromSeconds(1);
 
    public void Add(CrawlUrl url, int priority)
    {
        _priorityQueue.Enqueue(url, priority);
    }
 
    public CrawlUrl? GetNext()
    {
        // 找到一個尚未被 Politeness 限制的域名
        while (_priorityQueue.Count > 0)
        {
            var url = _priorityQueue.Dequeue();
            var host = new Uri(url.Url).Host;
 
            if (_lastAccessTime.TryGetValue(host, out var lastTime)
                && DateTime.UtcNow - lastTime < _politenessDelay)
            {
                // 還在冷卻期，放回後端 Queue 稍後處理
                EnqueueToHostQueue(host, url);
                continue;
            }
 
            _lastAccessTime[host] = DateTime.UtcNow;
            return url;
        }
        return null;
    }
}

class BloomFilter {
  private bitArray: Uint8Array;
  private hashCount: number;
  private size: number;
 
  constructor(expectedItems: number, falsePositiveRate: number) {
    // m = -(n * ln(p)) / (ln2)^2
    this.size = Math.ceil(
      -(expectedItems * Math.log(falsePositiveRate)) / Math.pow(Math.LN2, 2)
    );
    this.hashCount = Math.ceil((this.size / expectedItems) * Math.LN2);
    this.bitArray = new Uint8Array(Math.ceil(this.size / 8));
  }
 
  add(url: string): void {
    for (const pos of this.getHashPositions(url)) {
      this.bitArray[Math.floor(pos / 8)] |= 1 << (pos % 8);
    }
  }
 
  mightContain(url: string): boolean {
    return this.getHashPositions(url).every(
      (pos) => (this.bitArray[Math.floor(pos / 8)] & (1 << (pos % 8))) !== 0
    );
  }
}
 
// 10 億 URL、0.01% false positive rate → 約 2.4 GB 記憶體
const filter = new BloomFilter(1_000_000_000, 0.0001);

from urllib.robotparser import RobotFileParser
from functools import lru_cache
import time
 
class RobotsChecker:
    def __init__(self):
        self._cache: dict[str, tuple[RobotFileParser, float]] = {}
        self._ttl = 86400  # 24 小時快取
 
    def is_allowed(self, url: str, user_agent: str = "MyBot") -> bool:
        from urllib.parse import urlparse
        parsed = urlparse(url)
        robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
 
        parser = self._get_parser(robots_url)
        return parser.can_fetch(user_agent, url)
 
    def _get_parser(self, robots_url: str) -> RobotFileParser:
        if robots_url in self._cache:
            parser, timestamp = self._cache[robots_url]
            if time.time() - timestamp < self._ttl:
                return parser
 
        parser = RobotFileParser()
        parser.set_url(robots_url)
        parser.read()
        self._cache[robots_url] = (parser, time.time())
        return parser