robots.txt

robots.txt is a small text file located in the main directory of your website that tells search engine bots (such as Googlebot, Bingbot) which pages of your site they can or cannot crawl.

net core robots.txt

Kopyala

                        public class RobotsController : Controller
{
    private readonly IHostEnvironment _env;

    public RobotsController(IHostEnvironment env)
    {
        _env = env;
    }
   
    [Route("robots.txt")]
    public async Task<IActionResult> Index()
    {
        var sb = new StringBuilder();

        if (_env.IsDevelopment())
        {
            // Local (Development)
            sb.AppendLine("User-agent: *");
            sb.AppendLine("Disallow: /admin/");
            sb.AppendLine("Disallow: /login/");
            sb.AppendLine("Disallow: /private/");
            sb.AppendLine("Disallow: /config/");
            sb.AppendLine("# Local environment robots.txt");
        }
        else
        {
            // Server
            sb.AppendLine("User-agent: *");
            sb.AppendLine("Disallow: /admin/");
            sb.AppendLine("Disallow: /login/");
            sb.AppendLine("Disallow: /private/");
            sb.AppendLine("Disallow: /config/");
            sb.AppendLine("# Production environment robots.txt");
        }

        var content = sb.ToString();

        return Content(content, "text/plain", Encoding.UTF8);
    }
}
                    

Local and Server link

• https://www.localhost:4045/robots.txt
• https://www.yourdomain.com/robots.txt