# Rule 1: Delay all the bots User-agent: * Crawl-delay: 5 # Rule 2: Block search results from being indexed User-agent: * Disallow: /*?* # Rule 3: Allow all the bots User-agent: * Allow: / # Rule 4: Disallow unwanted bots User-agent: AddThis.com Disallow: / User-agent: AhrefsBot Disallow: / User-agent: BuiltWith Disallow:/ User-agent: GPTBot Disallow: / User-agent: PetalBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: SemrushBot-SA Disallow: / User-agent: SemrushBot/1.2~bl Disallow: / User-agent: YandexBot Disallow: / sitemap: https://www.wisconsinhistory.org/sitemap/AHI_sitemapindex.xml sitemap: https://www.wisconsinhistory.org/sitemap/Birth_sitemapindex.xml sitemap: https://www.wisconsinhistory.org/sitemap/CMS_sitemapindex.xml sitemap: https://www.wisconsinhistory.org/sitemap/Death_sitemapindex.xml sitemap: https://www.wisconsinhistory.org/sitemap/Marriage_sitemapindex.xml sitemap: https://www.wisconsinhistory.org/sitemap/NR_sitemapindex.xml sitemap: https://www.wisconsinhistory.org/sitemap/WHI_sitemapindex.xml sitemap: https://www.wisconsinhistory.org/sitemap/WLHBA_sitemapindex.xml sitemap: https://www.wisconsinhistory.org/sitemap/WNI_sitemapindex.xml