[{"data":1,"prerenderedAt":268},["ShallowReactive",2],{"blog-robots-txt-common-mistakes":3,"blog-prev-robots-txt-common-mistakes":152,"blog-next-robots-txt-common-mistakes":132},{"id":4,"title":5,"author":6,"body":7,"canonical":132,"category":132,"cover":132,"date":132,"description":133,"draft":134,"extension":135,"keywords":132,"meta":136,"navigation":137,"noindex":134,"ogImage":132,"path":138,"seo":139,"seoDescription":132,"seoTitle":132,"stem":149,"tags":150,"updated":132,"__hash__":151},"blog\u002Fblog\u002Frobots-txt-common-mistakes.md","robots.txt 最常写错的 7 处，每一处都有真实事故","元析智能",{"type":8,"value":9,"toc":120},"minimark",[10,14,19,31,34,38,45,48,52,55,59,66,70,92,96,103,107,110,114,117],[11,12,13],"p",{},"robots.txt 是 SEO 里性价比最高的文件，也是最容易一错误终身的文件。我们每年在客户站上至少碰到三四次 robots 事故，下面 7 处按出现频次排，全是真实见过的。",[15,16,18],"h2",{"id":17},"一上线忘删-disallow","一、上线忘删 Disallow: \u002F",[11,20,21,22,26,27,30],{},"2025 年 4 月，一个 SaaS 客户的测试规则被带到了生产环境：",[23,24,25],"code",{},"User-agent: *"," 下面一行 ",[23,28,29],{},"Disallow: \u002F","。上线三天，百度收录从 2.1 万掉到 1.4 万，GSC 里\"已被 robots.txt 屏蔽\"的页面占到全站 92%。他们市场负责人是在周报里看到自然流量腰斩才发现的。",[11,32,33],{},"查法：直接 curl 生产环境的 \u002Frobots.txt，不要在后台配置面板里看，面板里显示的经常不是线上那一份。",[15,35,37],{"id":36},"二屏蔽-js-和-css","二、屏蔽 \u002Fjs\u002F 和 \u002Fcss\u002F",[11,39,40,41,44],{},"老教程留下的做法，说屏蔽这些能省抓取预算。现在搜索引擎要执行 JS 才能渲染页面，屏蔽资源的结果是渲染残缺，正文和内链一起丢。2025 年还有人在这么写，我上个月刚在一个客户站见到 ",[23,42,43],{},"Disallow: \u002Fstatic\u002F","。",[11,46,47],{},"查法：robots 里搜 Disallow，看到 \u002Fjs、\u002Fassets、\u002Fstatic 这类路径就删掉。",[15,49,51],{"id":50},"三用-robots-代替-noindex","三、用 robots 代替 noindex",[11,53,54],{},"想让页面不被索引，写 robots 是错的。被屏蔽的 URL 搜索引擎不会去读页面上的 noindex，如果别处有链接指向它，它照样可能被收录；页面上的内链也传不出去。要 noindex 就在 HTML 或响应头里写，别绕路。",[15,56,58],{"id":57},"四忘了写-sitemap-声明","四、忘了写 Sitemap 声明",[11,60,61,62,65],{},"robots.txt 里一行 ",[23,63,64],{},"Sitemap: https:\u002F\u002Fexample.com\u002Fsitemap.xml"," 是最省事的提交方式，对百度尤其明显。漏了不致命，但新页面的发现会慢几天到几周。",[15,67,69],{"id":68},"五通配符和结尾符搞混","五、通配符和结尾符搞混",[11,71,72,75,76,79,80,83,84,87,88,91],{},[23,73,74],{},"Disallow: \u002Fproduct\u002F"," 和 ",[23,77,78],{},"Disallow: \u002Fproduct"," 是两回事，后者会把 ",[23,81,82],{},"\u002Fproducts\u002F"," 一起挡掉。只想匹配结尾用 ",[23,85,86],{},"$","，想匹配中间参数用 ",[23,89,90],{},"*","。这类错误不报错，只安静地挡掉不该挡的页面。",[15,93,95],{"id":94},"六抄了-crawl-delay","六、抄了 Crawl-delay",[11,97,98,99,102],{},"很多模板自带 ",[23,100,101],{},"Crawl-delay: 10","，本意是减轻服务器压力。Googlebot 不认这条指令，百度认，但对抓取频次没有好处。服务器扛不住应该优化响应时间或上 CDN，而不是让蜘蛛慢下来——抓取预算本来就不宽裕。",[15,104,106],{"id":105},"七robotstxt-放错位置或被缓存","七、robots.txt 放错位置或被缓存",[11,108,109],{},"robots.txt 只能放在域名根目录，子目录里的无效。换 CDN 之后旧文件被缓存 24 小时以上很常见，改完必须刷缓存再验证。",[15,111,113],{"id":112},"我们检查-robots-的三个动作","我们检查 robots 的三个动作",[11,115,116],{},"curl 线上文件，用 GSC 的 robots.txt 测试工具验一个 URL，再翻日志确认没有大面积 403。三步十分钟，能挡掉大部分事故。",[11,118,119],{},"我们的立场很简单：robots.txt 只用来拦真正不需要抓的资源，索引控制交给 noindex，别让一个文件兼任两件事。",{"title":121,"searchDepth":122,"depth":122,"links":123},"",2,[124,125,126,127,128,129,130,131],{"id":17,"depth":122,"text":18},{"id":36,"depth":122,"text":37},{"id":50,"depth":122,"text":51},{"id":57,"depth":122,"text":58},{"id":68,"depth":122,"text":69},{"id":94,"depth":122,"text":95},{"id":105,"depth":122,"text":106},{"id":112,"depth":122,"text":113},null,"[object Object]",false,"md",{},true,"\u002Fblog\u002Frobots-txt-common-mistakes",{"title":5,"description":140},{"robots":141,"date":143,"category":144,"tags":145,"author":6,"draft":134},{"txt 只有几条语法，写错一次能让整站从搜索结果里消失。这 7 处错误我们在客户站上都见过，包括上线忘删 Disallow":142},"\u002F、屏蔽 JS、用 robots 代替 noindex，每条给出查法。","2025-09-21","技术SEO",[146,147,148],"robots.txt","抓取","网站诊断","blog\u002Frobots-txt-common-mistakes",[],"LlauDRQEXiKxsJJRZ0wwwSwqZvaZpr85k1Kq9nTbLq0",{"id":153,"title":154,"author":6,"body":155,"canonical":132,"category":256,"cover":132,"date":257,"description":258,"draft":134,"extension":135,"keywords":132,"meta":259,"navigation":137,"noindex":134,"ogImage":132,"path":260,"seo":261,"seoDescription":132,"seoTitle":132,"stem":262,"tags":263,"updated":132,"__hash__":267},"blog\u002Fblog\u002Fcitation-click-landing.md","被 AI 引用之后，用户点进来落在哪一页",{"type":8,"value":156,"toc":247},[157,160,163,166,169,172,175,178,181,184,188,191,194,197,201,204,207,210,214,217,220,223,226,229,232,235,238,241,244],[11,158,159],{},"2026 年 11 月我们查一个客户的 AI 引用数据，发现有内容的页面被引用最多，但那些页面的留资几乎为零。引用带来了访问，访问走到了死路。",[15,161,162],{"id":162},"引用点击会落在哪",[11,164,165],{},"大部分情况落在被引用的那个页面，因为用户点的是答案里的来源链接。",[11,167,168],{},"少数情况用户不会点，而是看完回答直接搜品牌名。这部分流量会被记成品牌词搜索或直接访问。",[11,170,171],{},"两种情况都要求被引用页面本身能承接。",[15,173,174],{"id":174},"承接断在哪",[11,176,177],{},"被引用的页面通常是文档、博客、术语解释，这类页面的设计目标是讲清楚一个问题，不是促成转化。",[11,179,180],{},"于是出现一个断点：用户读完，觉得有用，但不知道下一步做什么。",[11,182,183],{},"页面上没有入口、没有相关产品、没有更多资料，用户就走了。",[15,185,187],{"id":186},"第一步找到被引用的页面","第一步：找到被引用的页面",[11,189,190],{},"从引用记录里整理页面清单，按被引用次数排序。",[11,192,193],{},"再和流量数据交叉，看这些页面实际有多少访问。",[11,195,196],{},"两个数据都靠前的页面，优先改。",[15,198,200],{"id":199},"第二步加一个不打扰的下一步","第二步：加一个不打扰的下一步",[11,202,203],{},"在文章末尾加一句，指向更详细的内容或者试用入口，比如\"想看完整的配置步骤，这里有文档\"。",[11,205,206],{},"在正文中间插一个轻量入口，比如\"这一节涉及的具体参数，我们的产品页有完整列表\"。",[11,208,209],{},"不要弹窗，不要遮内容。被引用页面的读者是来找答案的，打扰就流失。",[15,211,213],{"id":212},"第三步别把文档页改成销售页","第三步：别把文档页改成销售页",[11,215,216],{},"这是最容易犯的错。看到文档页流量大，就往上加各种 CTA 和表单。",[11,218,219],{},"结果内容变得难读，引用减少，流量也没了。",[11,221,222],{},"我们的原则是内容主体不动，只在合适的节点加导航，加完之后自己读一遍，不别扭才算过。",[15,224,225],{"id":225},"怎么测量",[11,227,228],{},"AI 引用的来源流量很难精确归因，因为很多引擎不传 referrer。",[11,230,231],{},"我们的做法是间接看三个数：被引用页面的总访问、这些页面的站内跳转率、品牌词搜索变化。",[11,233,234],{},"站内跳转率是最直接的指标。加了下一步之后，从文档页跳到产品页的比例应该上升。",[15,236,237],{"id":237},"一个态度",[11,239,240],{},"GEO 的终点不是被引用，是被引用之后有人愿意继续了解你。",[11,242,243],{},"引用是入口，承接是转化。只做引用不做承接，等于把流量引到一个没有门的房间。",[11,245,246],{},"这件事的投入很小，改几个页面的末尾就行。但它决定了 GEO 的投入能不能变成生意。",{"title":121,"searchDepth":122,"depth":122,"links":248},[249,250,251,252,253,254,255],{"id":162,"depth":122,"text":162},{"id":174,"depth":122,"text":174},{"id":186,"depth":122,"text":187},{"id":199,"depth":122,"text":200},{"id":212,"depth":122,"text":213},{"id":225,"depth":122,"text":225},{"id":237,"depth":122,"text":237},"SEO","2026-11-07","引用带来的点击通常落在被引用页面，而那一页往往没有转化入口。这篇写承接断在哪、怎么在不打扰阅读的前提下加下一步，以及怎么测量。",{},"\u002Fblog\u002Fcitation-click-landing",{"title":154,"description":258},"blog\u002Fcitation-click-landing",[264,265,266],"GEO","转化路径","落地页","59VysvmXMIaHC1faOdEZ615TAhHXhzDH2Ig14Ey9ZV4",1794088112746]