[{"data":1,"prerenderedAt":452},["ShallowReactive",2],{"blog-crawl-log-analysis-commands":3,"blog-prev-crawl-log-analysis-commands":261,"blog-next-crawl-log-analysis-commands":339},{"id":4,"title":5,"author":6,"body":7,"canonical":245,"category":246,"cover":245,"date":247,"description":248,"draft":249,"extension":250,"keywords":245,"meta":251,"navigation":252,"noindex":249,"ogImage":245,"path":253,"seo":254,"seoDescription":245,"seoTitle":245,"stem":255,"tags":256,"updated":245,"__hash__":260},"blog\u002Fblog\u002Fcrawl-log-analysis-commands.md","日志分析入门：三个命令看蜘蛛在抓什么","元析智能",{"type":8,"value":9,"toc":237},"minimark",[10,14,19,93,96,100,137,140,144,199,202,209,212,215,218,221,224,227,230,233],[11,12,13],"p",{},"日志是唯一能直接看到蜘蛛行为的数据。GSC 的抓取统计只给汇总，百度搜索资源平台给得更少。要看细节，还是得自己读 access.log。",[15,16,18],"h2",{"id":17},"命令一数请求","命令一：数请求",[20,21,26],"pre",{"className":22,"code":23,"language":24,"meta":25,"style":25},"language-bash shiki shiki-themes github-light github-dark","grep -c \"Googlebot\" access.log\ngrep \"Googlebot\" access.log | awk '{print $4}' | cut -d: -f1 | sort | uniq -c\n","bash","",[27,28,29,49],"code",{"__ignoreMap":25},[30,31,34,38,42,46],"span",{"class":32,"line":33},"line",1,[30,35,37],{"class":36},"sScJk","grep",[30,39,41],{"class":40},"sj4cs"," -c",[30,43,45],{"class":44},"sZZnC"," \"Googlebot\"",[30,47,48],{"class":44}," access.log\n",[30,50,52,54,56,59,63,66,69,71,74,77,80,82,85,87,90],{"class":32,"line":51},2,[30,53,37],{"class":36},[30,55,45],{"class":44},[30,57,58],{"class":44}," access.log",[30,60,62],{"class":61},"szBVR"," |",[30,64,65],{"class":36}," awk",[30,67,68],{"class":44}," '{print $4}'",[30,70,62],{"class":61},[30,72,73],{"class":36}," cut",[30,75,76],{"class":40}," -d:",[30,78,79],{"class":40}," -f1",[30,81,62],{"class":61},[30,83,84],{"class":36}," sort",[30,86,62],{"class":61},[30,88,89],{"class":36}," uniq",[30,91,92],{"class":40}," -c\n",[11,94,95],{},"第一条看总量，第二条按天分组看趋势。我们习惯把百度蜘蛛（Baiduspider）也一起数，两条曲线放一起看，一边涨一边跌通常意味着抓取预算被某个方向吃掉了。",[15,97,99],{"id":98},"命令二看状态码分布","命令二：看状态码分布",[20,101,103],{"className":22,"code":102,"language":24,"meta":25,"style":25},"grep \"Googlebot\" access.log | awk '{print $9}' | sort | uniq -c | sort -rn\n",[27,104,105],{"__ignoreMap":25},[30,106,107,109,111,113,115,117,120,122,124,126,128,130,132,134],{"class":32,"line":33},[30,108,37],{"class":36},[30,110,45],{"class":44},[30,112,58],{"class":44},[30,114,62],{"class":61},[30,116,65],{"class":36},[30,118,119],{"class":44}," '{print $9}'",[30,121,62],{"class":61},[30,123,84],{"class":36},[30,125,62],{"class":61},[30,127,89],{"class":36},[30,129,41],{"class":40},[30,131,62],{"class":61},[30,133,84],{"class":36},[30,135,136],{"class":40}," -rn\n",[11,138,139],{},"健康站点 200 占 85% 以上，5xx 应该低于 0.5%。404 占比高说明死链多，301 占比高说明重定向链没收敛。5xx 是最要命的，蜘蛛连续遇到几次服务端错误会主动降频。",[15,141,143],{"id":142},"命令三按路径聚合","命令三：按路径聚合",[20,145,147],{"className":22,"code":146,"language":24,"meta":25,"style":25},"grep \"Googlebot\" access.log | awk '{print $7}' | cut -d\u002F -f2 | sort | uniq -c | sort -rn | head -20\n",[27,148,149],{"__ignoreMap":25},[30,150,151,153,155,157,159,161,164,166,168,171,174,176,178,180,182,184,186,188,191,193,196],{"class":32,"line":33},[30,152,37],{"class":36},[30,154,45],{"class":44},[30,156,58],{"class":44},[30,158,62],{"class":61},[30,160,65],{"class":36},[30,162,163],{"class":44}," '{print $7}'",[30,165,62],{"class":61},[30,167,73],{"class":36},[30,169,170],{"class":40}," -d\u002F",[30,172,173],{"class":40}," -f2",[30,175,62],{"class":61},[30,177,84],{"class":36},[30,179,62],{"class":61},[30,181,89],{"class":36},[30,183,41],{"class":40},[30,185,62],{"class":61},[30,187,84],{"class":36},[30,189,190],{"class":40}," -rn",[30,192,62],{"class":61},[30,194,195],{"class":36}," head",[30,197,198],{"class":40}," -20\n",[11,200,201],{},"看哪个一级路径吃掉最多请求。2025 年 10 月一个站日抓 1.2 万次，排第一的是 \u002Fapi\u002F，占 41%，全部返回 401。robots.txt 里没屏蔽 API 路径，蜘蛛每天白跑 5000 次。屏蔽后产品页的抓取从每天 800 次涨到 2600 次。",[11,203,204,205,208],{},"顺手还能算响应时间：把 ",[27,206,207],{},"$request_time"," 聚合一下，平均超过 1 秒就要查服务器，蜘蛛抓取频次和响应速度强相关。",[15,210,211],{"id":211},"日志怎么留",[11,213,214],{},"nginx 默认按天切分，保留 7 天，做抓取分析不够用。我们一般让运维保留 90 天并压缩归档，或者直接推到日志服务里。做一次完整诊断要看连续 4 周的数据，单看一天容易被大促、发版这种异常带偏。",[11,216,217],{},"如果站点在 CDN 后面，源站日志里只有回源请求，蜘蛛直接命中缓存的请求看不到。这种情况要用 CDN 厂商的访问日志，或者把蜘蛛 UA 的回源规则调整一下。我们遇到过一次，客户以为自己日抓只有 300 次，换到 CDN 日志一看实际是 4000 次，诊断结论完全反了。",[15,219,220],{"id":220},"三个常见误判",[11,222,223],{},"把 UA 当身份。UA 可以伪造，日志里出现自称 Googlebot 的请求不一定是真的，验证方法是反查 IP 段。把缓存命中当没抓。前面说过，CDN 后面的日志会少一大截。把周末的低抓取当异常。蜘蛛也有自己的节奏，按周看比按天看稳。",[15,225,226],{"id":226},"什么时候不用做",[11,228,229],{},"日抓取量 1000 次以下的站，先别碰日志分析。抓取量太小，结论全是噪声，先把内容做起来、把收录问题修好。抓取预算这件事，是流量上来了才值得抠的细节。",[11,231,232],{},"工具方面，GoAccess 能出一份可视化报表，Screaming Frog Log File Analyser 适合和爬虫数据对照，命令行足够应付 80% 的场景。我们做诊断先用三个命令看一遍，判断要不要上工具。",[234,235,236],"style",{},"html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .szBVR, html code.shiki .szBVR{--shiki-default:#D73A49;--shiki-dark:#F97583}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":25,"searchDepth":51,"depth":51,"links":238},[239,240,241,242,243,244],{"id":17,"depth":51,"text":18},{"id":98,"depth":51,"text":99},{"id":142,"depth":51,"text":143},{"id":211,"depth":51,"text":211},{"id":220,"depth":51,"text":220},{"id":226,"depth":51,"text":226},null,"技术SEO","2025-12-13","不用装工具也能做日志分析。我们用 awk 和 grep 从 nginx 日志里拉出蜘蛛请求量、状态码分布和最费预算的路径，附一个 2025 年 10 月的诊断记录，日抓 1.2 万次里 41% 是无效请求。",false,"md",{},true,"\u002Fblog\u002Fcrawl-log-analysis-commands",{"title":5,"description":248},"blog\u002Fcrawl-log-analysis-commands",[257,258,259],"日志分析","抓取预算","爬虫","0jXqh1Xu0qIIJacHBvtC-GCTZ9P3T1s6qFtljpRgYHs",{"id":262,"title":263,"author":6,"body":264,"canonical":245,"category":327,"cover":245,"date":328,"description":329,"draft":249,"extension":250,"keywords":245,"meta":330,"navigation":252,"noindex":249,"ogImage":245,"path":331,"seo":332,"seoDescription":245,"seoTitle":245,"stem":333,"tags":334,"updated":245,"__hash__":338},"blog\u002Fblog\u002Fai-writing-software-details.md","让 AI 写出软件行业的细节，靠三份材料",{"type":8,"value":265,"toc":320},[266,269,273,276,279,283,286,289,293,296,299,302,305,308,311,314,317],[11,267,268],{},"2025 年 12 月我用同一套提示词让 AI 写「日志分析」这个主题，两次结果差得很远。第二次只多做了一件事：先喂材料。想要 AI 写出软件行业的细节，靠的不是把提示词写得更花，而是给它三份东西。",[15,270,272],{"id":271},"第一份产品文档和更新日志","第一份：产品文档和更新日志",[11,274,275],{},"AI 不知道你的产品里那个字段叫什么。它会把「采样率」写成「采样比例」，把「事件配额」写成「数据上限」，看着像那么回事，实际一查就错。",[11,277,278],{},"我们把产品文档、API 文档、近半年的更新日志整理成一个文件，写到哪类主题就喂对应的部分。更新日志尤其有用，里面有真实的版本号、发布时间、配置项变更，这些东西写进文章里，懂行的人一眼能看出作者是真在用。",[15,280,282],{"id":281},"第二份真实问答","第二份：真实问答",[11,284,285],{},"客户工单、售前咨询、GitHub issue 里的原话，是最有信息量的素材。把公司名和敏感信息去掉，保留问题本身。比如「为什么日志里百度蜘蛛的 UA 是空的」「GSC 里显示已抓取未编入索引但页面明明有内容」，这类问题写进文章，比任何方法论都具体。",[11,287,288],{},"我的习惯是每篇至少带一个真实问题的原话转述。读者看到会想，这个问题我也遇到过。",[15,290,292],{"id":291},"第三份术语和禁区","第三份：术语和禁区",[11,294,295],{},"术语表解决两件事：这个行业的标准叫法是什么，我们内部怎么叫。禁区表列清楚不能出现的表述，比如不承诺排名、不写具体客户的业绩、不引用没核过的数据。",[11,297,298],{},"还有一层是文风禁区。我们不用的词、不用的句式，也写进文件里。AI 的默认写法很爱用排比和总结升华，这些都要在材料里明确禁止，不然它每次都给你来一段。",[15,300,301],{"id":301},"提示词的结构",[11,303,304],{},"角色一句话就够，不用铺垫。材料分块贴，标明哪块是文档、哪块是问答。要求写 1200 字，每个二级标题只讲一个论点，禁止编造数字，遇到不确定的地方留空让我补。",[11,306,307],{},"最后加一句自查：输出前问自己三遍，这段里有没有只有我们知道的信息？没有就重写。",[15,309,310],{"id":310},"材料要跟着产品更新",[11,312,313],{},"材料会过期。产品改了定价、加了功能、换了界面，旧材料喂进去就会写出错的细节。我们每月花半小时更新一次材料文件，重点改三处：版本号、菜单路径、限制数字。",[11,315,316],{},"也别一次性给 AI 太多。三份文件加起来超过一万字，它反而抓不住重点。写到哪个主题就摘出对应的 1500 字左右，宁可少给，不要给一坨。",[11,318,319],{},"我们每篇给 AI 的材料在 2000 字以上。材料不够的时候，出来的稿子读着通顺，但没有一句是别人写不出来的。",{"title":25,"searchDepth":51,"depth":51,"links":321},[322,323,324,325,326],{"id":271,"depth":51,"text":272},{"id":281,"depth":51,"text":282},{"id":291,"depth":51,"text":292},{"id":301,"depth":51,"text":301},{"id":310,"depth":51,"text":310},"内容运营","2025-12-12","同一套提示词，给不给材料，出来的稿子差一个档。我们每篇给 AI 准备三份材料：产品文档、真实问答、术语禁区，文章才能有具体参数和真实场景。",{},"\u002Fblog\u002Fai-writing-software-details",{"title":263,"description":329},"blog\u002Fai-writing-software-details",[335,336,337],"AI写作","提示词","内容质量","SaZwAY-8neI-KdAL3xKL7HybK26bUDlOsPw2-7W8GDY",{"id":340,"title":341,"author":6,"body":342,"canonical":245,"category":246,"cover":245,"date":441,"description":442,"draft":249,"extension":250,"keywords":245,"meta":443,"navigation":252,"noindex":249,"ogImage":245,"path":444,"seo":445,"seoDescription":245,"seoTitle":245,"stem":446,"tags":447,"updated":245,"__hash__":451},"blog\u002Fblog\u002Fttfb-network-vs-server.md","TTFB 高先分清是网络还是服务端，别急着换机房",{"type":8,"value":343,"toc":434},[344,347,350,378,384,388,391,394,398,401,404,407,410,413,416,419,422,425,428,431],[11,345,346],{},"TTFB 高的时候，先别换机房。今年 1 月我们查过一个 SaaS 的首页，移动端首字节 1.4 秒，团队第一反应是\"服务器不行\"，准备换云厂商。查下来 800 毫秒花在服务端自己身上，换到哪儿都一样。",[11,348,349],{},"先把 TTFB 拆开看。用 curl 的耗时分解，一个请求能拆成五段：",[20,351,353],{"className":22,"code":352,"language":24,"meta":25,"style":25},"curl -o \u002Fdev\u002Fnull -s -w \"dns:%{time_namelookup} tcp:%{time_connect} tls:%{time_appconnect} ttfb:%{time_starttransfer}\\n\" https:\u002F\u002Fexample.com\u002F\n",[27,354,355],{"__ignoreMap":25},[30,356,357,360,363,366,369,372,375],{"class":32,"line":33},[30,358,359],{"class":36},"curl",[30,361,362],{"class":40}," -o",[30,364,365],{"class":44}," \u002Fdev\u002Fnull",[30,367,368],{"class":40}," -s",[30,370,371],{"class":40}," -w",[30,373,374],{"class":44}," \"dns:%{time_namelookup} tcp:%{time_connect} tls:%{time_appconnect} ttfb:%{time_starttransfer}\\n\"",[30,376,377],{"class":44}," https:\u002F\u002Fexample.com\u002F\n",[11,379,380,383],{},[27,381,382],{},"time_starttransfer"," 减去前面几段，剩下的才是服务端处理时间。这个站 DNS 30 毫秒、TLS 120 毫秒、传输 90 毫秒，剩下 800 毫秒就是后端在算。",[15,385,387],{"id":386},"服务端那-800-毫秒在干什么","服务端那 800 毫秒在干什么",[11,389,390],{},"那个站一次页面请求串行调了 6 个接口，最慢的一个 340 毫秒。6 个加起来 1.1 秒，有并发，实际 800 毫秒。前端一直以为是网络问题，因为监控上只看到\"页面慢\"三个字。",[11,392,393],{},"查法是看应用侧的链路追踪，或者最土的办法：在服务端入口加时间日志，记每个接口的耗时。这个客户加完日志第二天就定位了，比讨论一周有用。",[15,395,397],{"id":396},"改法能并行的并行能缓存的缓存","改法：能并行的并行，能缓存的缓存",[11,399,400],{},"我们做了两件事：6 个接口里 4 个改成并行，总耗时从 800 降到 260 毫秒；首页那块\"最新公告\"从实时查库改成 60 秒缓存。TTFB 从 1.4 秒降到 480 毫秒，同一批机器。",[15,402,403],{"id":403},"什么情况才该怀疑网络",[11,405,406],{},"三种情况我才会建议查链路：TTFB 在 200 到 400 毫秒之间但各地差异大；静态资源也慢；TLS 握手占 300 毫秒以上。这三种通常指向线路或者 CDN 节点问题，和服务端代码无关。",[11,408,409],{},"还有个粗糙但好用的判断：如果你的静态 HTML 的 TTFB 就很高，那是基础设施；如果只有动态页慢，那基本是代码。这个判断一分钟能做完，比开会讨论有用。",[15,411,412],{"id":412},"一个容易误判的情况",[11,414,415],{},"首页慢、产品页快，这种情况很常见，因为首页的接口最多。我们查过一个站，产品页 TTFB 210 毫秒，首页 1.3 秒，团队按首页的平均值去投诉机房，方向完全错了。",[11,417,418],{},"正确做法是按页面类型分组统计 TTFB，每种取 P75。营销页、文档页、后台页分开看，差异往往比你想的大。",[15,420,421],{"id":421},"我们给客户的检查顺序",[11,423,424],{},"第一步，挑三个页面（首页、一个产品页、一个文档页）用 curl 各跑五次，记下五段耗时。第二步，把服务端那一段和接口日志对上。第三步，才决定是改代码、加缓存还是查线路。",[11,426,427],{},"这个顺序的好处是成本低。三步做完，半小时，能排除八成误判。最怕的是先入为主地认为是网络问题，换了一圈机房，最后发现是接口没并行。",[11,429,430],{},"最后一句：换机房是成本最高、见效最不确定的一招。我们接手过的性能项目里，真正靠换机房解决问题的只有两次，都是国内单线机房那种情况。",[234,432,433],{},"html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":25,"searchDepth":51,"depth":51,"links":435},[436,437,438,439,440],{"id":386,"depth":51,"text":387},{"id":396,"depth":51,"text":397},{"id":403,"depth":51,"text":403},{"id":412,"depth":51,"text":412},{"id":421,"depth":51,"text":421},"2025-12-14","一个 SaaS 首页移动端首字节 1.4 秒，团队准备换云厂商，其实 800 毫秒花在服务端串行接口上。给一条 curl 拆时间的命令，和三个才该怀疑网络的情况。",{},"\u002Fblog\u002Fttfb-network-vs-server",{"title":341,"description":442},"blog\u002Fttfb-network-vs-server",[448,449,450],"TTFB","服务端性能","性能诊断","BR70gBacE8JBbLjWfYgmkRwPIixD9DwNRj0tqZXvyIE",1794088110080]