
{
    "version": "1.5.0",
    "item": {
        "id": "blog_v92",
        "title": "Website Indexation Strategies for Large Sites (2026 Guide)",
        "url": "https://www.dbeta.co.uk/blog/website-indexation-strategies-large-sites/",
        "summary": "Indexation on large websites is a governance problem about index quality, not just crawling. Unmanaged faceted navigation is a major risk, and bot-specific controls, such as treating OAI-SearchBot differently from GPTBot, now matter for AI-driven discovery.",
        "category": "SEO Foundations",
        "image": "https://dbeta.b-cdn.net/blog/large-site-indexation-strategy-governance.jpg",
        "author": {
            "name": "Donatas Tranauskis",
            "url": "https://www.donatastranauskis.com"
        },
        "published_at": "2026-02-20T22:00:00+00:00",
        "updated_at": "2026-03-28T22:00:00+00:00",
        "faqs": [
            {
                "question": "Why is Google not indexing all the pages on my large website?",
                "answer": "Google does not automatically index every URL it finds. For large sites, indexation is a quality assessment. If your site has thousands of thin, duplicate, or unmanaged filtered URLs, Google may decide that your 'Index Quality' is too low and stop indexing new pages."
            },
            {
                "question": "What is the difference between OAI-SearchBot and GPTBot?",
                "answer": "OAI-SearchBot is OpenAI's crawler for ChatGPT's search features (showing your site as a cited source). GPTBot is their crawler for training AI models. You can choose to allow one while blocking the other in your robots.txt file."
            },
            {
                "question": "How do I fix 'Discovered - currently not indexed' in Search Console?",
                "answer": "This usually means Google found your URLs but decided not to crawl them yet because your 'Crawl Budget' is being wasted on lower-value pages. You need to improve your internal linking to key pages and block junk URLs (like search results or filters) in robots.txt."
            },
            {
                "question": "Is content pruning good for SEO?",
                "answer": "Yes, when done strategically. Pruning is about removing or merging weak, outdated, or duplicate content so that search engines can focus their attention on your highest-quality, most commercially relevant pages."
            }
        ],
        "sources": [
            {
                "label": "Google Search Central - Managing Crawl Budget",
                "url": "https://developers.google.com/search/docs/crawling-indexing/large-site-managing-crawl-budget"
            },
            {
                "label": "OpenAI - Bot Documentation (OAI-SearchBot)",
                "url": "https://platform.openai.com/docs/bots"
            },
            {
                "label": "Google Search Central - Soft 404 errors",
                "url": "https://developers.google.com/search/docs/crawling-indexing/http-network-errors#soft-404"
            },
            {
                "label": "Google Search Central - JavaScript SEO Basics",
                "url": "https://developers.google.com/search/docs/crawling-indexing/javascript/javascript-seo-basics"
            }
        ]
    }
}