# The content site is meant to be crawled — that's its whole job. User-agent: * Allow: / # The app and the admin console are not content: one is behind auth and renders nothing # useful to a crawler, the other is an ops surface. Disallow: /admin Disallow: /app # The empty frame the gateway fills for the forecast pages — never a page itself. Disallow: /page-shell # Cloudflare's own addresses — its email protection turns every address on a page into a link here, which a # crawler then reports as a missing page on every page. Disallow: /cdn-cgi/ Sitemap: https://kundaly.com/sitemap-index.xml Sitemap: https://kundaly.com/forecast/sitemap.xml