{"id":1839,"date":"2017-02-26T15:23:11","date_gmt":"2017-02-26T07:23:11","guid":{"rendered":"http:\/\/cn.hostease.com\/xueyuan\/?p=1839"},"modified":"2017-02-26T15:23:11","modified_gmt":"2017-02-26T07:23:11","slug":"robots-txt%e6%96%87%e4%bb%b6%e6%98%af%e4%bb%80%e4%b9%88%ef%bc%9f","status":"publish","type":"post","link":"https:\/\/cn.hostease.com\/xueyuan\/jishu\/seo\/robots-txt%e6%96%87%e4%bb%b6%e6%98%af%e4%bb%80%e4%b9%88%ef%bc%9f\/","title":{"rendered":"robots.txt\u6587\u4ef6\u662f\u4ec0\u4e48\uff1f"},"content":{"rendered":"<p>robots.txt\u662f\u4e00\u4e2a\u7b80\u5355\u7684\u4ee5.txt\u7ed3\u5c3e\u7684\u6587\u672c\u6587\u4ef6\uff0c\u662f\u641c\u7d22\u5f15\u64ceRobot\uff08\u4e5f\u53eb\u641c\u7d22\u5f15\u64ce\u673a\u5668\u4eba\uff09\u7a0b\u5e8f\u6293\u53d6\u7f51\u9875\u65f6\u8981\u8bbf\u95ee\u7684\u7b2c\u4e00\u4e2a\u6587\u4ef6\u3002<\/p>\n<p>\u901a\u8fc7robots.txt\u6587\u4ef6\uff0c\u53ef\u4ee5\u548c\u5404\u5927\u641c\u7d22\u5f15\u64ce\u5f88\u53cb\u597d\u7684\u5bf9\u8bdd\uff0c\u5f15\u5bfc\u641c\u7d22\u5f15\u64ce\u673a\u5668\u4eba\u6293\u53d6\u4f60\u63a8\u8350\u7684\u7f51\u9875\uff0c\u907f\u514d\u4e00\u4e9b\u610f\u4e49\u4e0d\u5927\u6216\u65e0\u7528\u7f51\u9875\uff0c\u4f8b\u5982\u7f51\u7ad9\u540e \u53f0\u3001\u4f1a\u5458\u4ea4\u4e92\u529f\u80fd\u7b49\uff0c\u8fd9\u5728\u4e00\u5b9a\u7a0b\u5ea6\u4e0a\u4e5f\u8282\u7701\u670d\u52a1\u5668\u7f51\u7edc\u8d44\u6e90\u3002\u53e6\u5916\uff0crobots.txt\u6587\u4ef6\u5bf9SEO\u7684\u610f\u4e49\u4e5f\u5f88\u91cd\u8981\uff0c\u53ef\u4ee5\u5f88\u597d\u7684\u907f\u514d\u91cd\u590d\u3001\u76f8\u4f3c\u7f51\u9875\uff0c\u4ee5\u53ca \u4e00\u4e9b\u5173\u952e\u5b57\u6743\u91cd\u6d41\u5931\uff1b\u5199\u597drobots.txt\u6587\u4ef6\uff0c\u662f\u6bcf\u4e2aSEOer\u5fc5\u505a\u7684\u529f\u8bfe\u4e4b\u4e00\u3002<\/p>\n<p><strong>\u5982\u4f55\u914d\u7f6erobots.txt\u6587\u4ef6\uff1f<\/strong><\/p>\n<p>\u914d\u7f6erobots.txt\u6587\u4ef6\u5f88\u7b80\u5355\uff0c\u65b0\u5efa\u6587\u672c\u6587\u6863\uff0c\u547d\u540d\u4e3arobots.txt\uff0c\u6ce8\u610f\u8fd9\u4e2a\u6587\u6863\u662f\u4ee5\u201c.txt\u201d\u7ed3\u5c3e\u3002\u5728 robots.txt\u6587\u4ef6\u91cc\u5199\u597d\u89c4\u5219\u5373\u53ef\u3002\u60f3\u68c0\u67e5robots.txt\u6587\u4ef6\u914d\u7f6e\u662f\u5426\u6b63\u786e\uff0c\u53ea\u7528\u5728\u6d4f\u89c8\u5668\u8f93\u5165\u7f51 \u5740:https:\/\/\u4f60\u7684\u57df\u540d\/robots.txt\uff0c\u770b\u80fd\u5426\u6b63\u5e38\u8bbf\u95ee\u5373\u53ef\u3002<\/p>\n<p>robots.txt\u6587\u4ef6\u5199\u6cd5\u975e\u5e38\u7b80\u5355\uff0c\u901a\u5e38\u4ee5User-agent\u5f00\u59cb\uff0c\u5305\u542b\u4e00\u884c\u6216\u591a\u884cDisallow\u6216Allow\u8bb0\u5f55\uff1b\u5728 robots.txt\u6587\u4ef6\u4e2d\u53ef\u4ee5\u4f7f\u7528\u201c#\u201d\u8fdb\u884c\u6ce8\u91ca\uff0c\u5177\u4f53\u4f7f\u7528\u65b9\u6cd5\u548cPHP\u4e00\u6837\uff1b\u53e6\u5916\uff0crobots.txt\u6587\u4ef6\u652f\u6301*|$|?\u7b49\u901a\u914d\u7b26\u3002\u770b\u4e2a\u7b80\u5355\u7684\u4f8b\u5b50<\/p>\n<pre># robots.txt\u6587\u4ef6\u600e\u4e48\u5199\uff1f\r\nUser-agent: *\u00a0 #\u5bf9\u6240\u6709\u641c\u7d22\u5f15\u64ceRobot\u5f00\u653e\r\nDisallow: \/wp-content\/ #\u7981\u6b62\u8bbf\u95eewp-content\u76ee\u5f55\u4e0b\u4efb\u4f55\u5185\u5bb9<\/pre>\n<p>\u5173\u4e8erobots.txt\u6587\u4ef6\u51fd\u6570\u8bf4\u660e\uff1a<\/p>\n<p><strong>User-agent<\/strong><\/p>\n<p>\u8be5\u9879\u7684\u503c\u7528\u4e8e\u63cf\u8ff0\u641c\u7d22\u5f15\u64ceRobot\u7684\u540d\u79f0\uff0c\u81f3\u5c11\u8981\u6709\u4e00\u6761User-agent\u8bb0\u5f55\uff1b\u5982\u679cUser-agent\u7684\u503c\u4e3a*\uff0c\u5219\u8868\u793a\u8be5\u534f\u8bae\u5bf9\u6240\u6709\u641c\u7d22\u5f15\u64ceRobot\u90fd\u6709\u6548\uff1b<\/p>\n<p>\u5728robots.txt\u6587\u4ef6\u4e2d\u201cUser-agent: *\u201d\u53ea\u80fd\u6709\u4e00\u6761\uff0c\u53ef\u4ee5\u540c\u65f6\u51fa\u73b0\u201cUser-agent: Baiduspider\u201d\u548c\u201cUser-agent: Googlebot\u201d\u7684\u60c5\u51b5\u3002<\/p>\n<p><strong>\u9644\uff1a\u5e38\u89c1\u641c\u7d22\u5f15\u64ce\u8718\u86dbRobots\u540d\u79f0<\/strong><\/p>\n<pre>Baiduspider https:\/\/www.baidu.com\r\nia_archiver https:\/\/www.alexa.com\r\nGooglebot https:\/\/www.google.com\r\nScooter https:\/\/www.altavista.com\r\nFAST-WebCrawler https:\/\/www.alltheweb.com\r\nSlurp https:\/\/www.inktomi.com\r\nMSNBOT https:\/\/search.msn.com<\/pre>\n<p><strong>Disallow<\/strong><\/p>\n<p>\u8be5\u9879\u7684\u503c\u7528\u4e8e\u63cf\u8ff0\u4e0d\u5e0c\u671b\u88abRobot\u8bbf\u95ee\u5230\u7684\u4e00\u4e2aURL\uff0c\u8fd9\u4e2aURL\u53ef\u4ee5\u662f\u4e00\u6761\u5b8c\u6574\u7684\u8def\u5f84\uff0c\u4e5f\u53ef\u4ee5\u662f\u90e8\u5206\u7684\uff0c\u4efb\u4f55\u4ee5Disallow\u5f00\u5934\u7684URL\u5747\u4e0d\u4f1a\u88abRobot\u8bbf\u95ee\u5230\uff1b<\/p>\n<p>\u6ce8\u610f\u201cDisallow: \/test\u548c\u201cDisallow: \/test\/\u201d\u7684\u533a\u522b\uff0c\u867d\u8bf4\u53ea\u533a\u522b\u4e8e\u4e00\u4e2a\u53cd\u659c\u6760\u201c\/\u201d\uff0c\u4e0d\u8fc7\u610f\u4e49\u5b8c\u5168\u4e0d\u540c\u3002\u201cDisallow: \/test\u201d\u8868\u793a\u53ef\u4ee5\u7981\u6b62\u7684URL\u5305\u62ec\uff1a\/test\u3001\/testabc.html\u3001\/test\/abc\u8fd9\u4e09\u79cd\u5f62\u5f0f\uff1b\u201cDisallow: \/test\/\u201d\u5219\u5141\u8bb8Robot\u8bbf\u95ee\/test\u3001\/testabc.html\uff0c\u7981\u6b62\u8bbf\u95ee\/test\/abc\u8fd9\u79cd\u5f62\u5f0f\u3002<\/p>\n<p>\u5982\u679cDisallow\u8bb0\u5f55\u7684\u503c\u4e3a\u7a7a\uff0c\u5373\u201cDisallow:\u201d\u683c\u5f0f\uff0c\u5219\u8bf4\u660e\u8be5\u7f51\u7ad9\u7684\u6240\u6709\u5185\u5bb9\u53ef\u4ee5\u88ab\u4efb\u4f55\u641c\u7d22\u5f15\u64ceRobot\u6293\u53d6\uff1b\u5728robots.txt\u6587\u4ef6\uff0c\u5982\u679c\u6709\u58f0\u660eUser-agent\uff0c\u81f3\u5c11\u8981\u6709\u4e00\u6761Disallow\u8bb0\u5f55\u3002<\/p>\n<p>&nbsp;<\/p>\n<p><strong>Allow<\/strong><\/p>\n<p>\u8be5\u9879\u548cDisallow\u5bf9\u7acb\uff0c\u8868\u793a\u5141\u8bb8\u641c\u7d22\u5f15\u64ceRobot\u8bbf\u95ee\u6307\u5b9a\u5185\u5bb9\u3002<\/p>\n<p>&nbsp;<\/p>\n<p><strong>robots.txt\u5b9e\u4f8b<\/strong><\/p>\n<p><strong>\u5141\u8bb8\u6240\u6709\u641c\u7d22\u5f15\u64ceRobot\u8bbf\u95ee<\/strong><\/p>\n<pre>User-agent: *\r\nDisallow:<\/pre>\n<p>\u6216\u8005robots.txt\u6587\u4ef6\u4e3a\u7a7a\uff0c\u4ec0\u4e48\u4e5f\u4e0d\u5199\uff0c\u4e5f\u80fd\u8fbe\u5230\u540c\u6837\u7684\u6548\u679c\u3002<\/p>\n<p><strong>\u7981\u6b62\u6240\u6709\u641c\u7d22\u5f15\u64ceRobot\u8bbf\u95ee\u7f51\u7ad9\u7684\u4efb\u4f55\u5185\u5bb9<\/strong><\/p>\n<pre>User-agent: *\r\nDisallow: \/<\/pre>\n<p><strong>\u7981\u6b62\u6240\u6709\u641c\u7d22\u5f15\u64ceRobot\u8bbf\u95ee\u7f51\u7ad9\u7684\u6307\u5b9a\u5185\u5bb9<\/strong>\uff08\u5982\u4e0b\u4f8b\u4e2d\u7684aaa\u3001bbb\u3001ccc\u76ee\u5f55\uff09<\/p>\n<pre>User-agent: *\r\nDisallow: \/aaa\/\r\nDisallow: \/bbb\/\r\nDisallow: \/cccc\/<\/pre>\n<p><strong>\u7981\u6b62\u6307\u5b9a\u641c\u7d22\u5f15\u64ceRobot\u8bbf\u95ee<\/strong>\uff08\u5982\u4e0b\u4f8b\u4e2d\u7684Google\u673a\u5668\u4eba\uff1aGooglebot\uff09<\/p>\n<pre>User-agent: Googlebot\r\nDisallow: \/<\/pre>\n<p><strong>\u53ea\u5141\u8bb8\u6307\u5b9a\u641c\u7d22\u5f15\u64ceRobot\u8bbf\u95ee<\/strong>\uff08\u5982\u4e0b\u4f8b\u4e2d\u7684\u767e\u5ea6\u673a\u5668\u4eba\uff1aBaiduspider\uff09<\/p>\n<pre>User-agent: Baiduspider\r\nDisallow:\r\nUser-agent: *\r\nDisallow: \/<\/pre>\n<p><strong>\u5411\u641c\u7d22\u5f15\u64ce\u63d0\u4ea4\u7ad9\u70b9\u5730\u56fe<\/strong><\/p>\n<pre>https:\/\/\u4f60\u7684\u57df\u540d\/sitemap.xml<\/pre>\n<p>\u76ee\u524d\u5404\u5927\u641c\u7d22\u5f15\u64ce\u5bf9\u7ad9\u70b9\u5730\u56fe\u652f\u6301\u826f\u597d\uff0c\u9664\u4e86sitemap.xml\u8fd9\u79cd\u683c\u5f0f\u5916\uff0c\u8fd8\u63a8\u8350sitemap.html\u8fd9\u79cd\u9759\u6001\u683c\u5f0f\uff0c\u597d\u5904\u591a\u591a\uff0c\u4e0d\u518d\u7d2f\u8ff0\u3002<\/p>\n<p>&nbsp;<\/p>\n<p><strong>robots.txt\u6587\u4ef6\u5341\u4e2a\u6ce8\u610f\u4e8b\u9879<\/strong><\/p>\n<p>1\uff09robots.txt\u6587\u4ef6\u540d\u5fc5\u987b<strong>\u5168\u90e8\u5c0f\u5199<\/strong>\uff0c\u4e0d\u8981\u4e3a\u8ffd\u6c42\u4e2a\u6027\u5199\u6210RoboTs.txt\u7b49\u5f62\u5f0f\uff1b<\/p>\n<p>2\uff09robots.txt\u6587\u4ef6\u53ea\u80fd\u653e\u5728\u7f51\u7ad9<strong>\u6839\u76ee\u5f55<\/strong>\u4e0b\u9762\uff0c\u5176\u5b83\u4f4d\u7f6e\u65e0\u6548\uff1b<\/p>\n<p>3\uff09robots.txt\u6587\u4ef6\u5185\u5bb9\u5bf9<strong>\u5927\u5c0f\u5199\u654f\u611f<\/strong>\uff0c\u5c24\u5176\u662f\u5305\u542b\u7f51\u5740\u7684\u90e8\u5206\u3002\u867d\u8bf4\u5f88\u591a\u641c\u7d22\u5f15\u64ce\u5bf9\u7f51\u5740\u4e4b\u5916\u7684\u90e8\u5206\u6ca1\u6709\u4e25\u683c\u7684\u5927\u5c0f\u5199\u9650\u5236\uff0c\u4f46\u662f\u4ece\u641c\u7d22\u5f15\u64ce\u7684\u4e25\u8c28\u6027\u548c\u4e2a\u4eba\u5bf9\u7f51\u7ad9\u7684\u8981\u6c42\u65b9\u9762\u6765\u8bf4\uff0c\u8fd8\u662f\u5efa\u8bae\u533a\u5206\u5927\u5c0f\u5199\uff1b<\/p>\n<p>4\uff09\u901a\u5e38\uff0crobots.txt\u6587\u4ef6\u91cc\u53ea\u6709\u4e09\u4e2a\u51fd\u6570\uff1aUser-agent\u3001Disallow\u548cAllow\uff0c\u901a\u5e38\u6211\u4eec\u4f7f\u7528\u524d\u4e24\u4e2a\u7ec4\u5408\u9650\u5236\u6293\u53d6\u5c31\u53ef\u4ee5\u4e86\uff0c\u4e2a\u4eba\u8ba4\u4e3a\u6ca1\u6709\u5fc5\u8981\u4f7f\u7528Allow\uff0c\u9ed8\u8ba4\u641c\u7d22\u5f15\u64ce\u662f\u4f1a\u6293\u53d6\u4f60\u7f51\u7ad9\u6240\u6709\u5185\u5bb9\u7684\uff1b<\/p>\n<p>5\uff09robots.txt\u6587\u4ef6\u91cc\u4e0d\u8981\u653e\u5176\u5b83\u65e0\u5173\u5185\u5bb9\uff0c\u8981\u4e25\u683c\u9075\u4ece\u89c4\u8303\u3002<\/p>\n<p>6\uff09robots.txt\u6587\u4ef6\u53ea\u662f\u548c\u641c\u7d22\u5f15\u64ce\u7684\u4e00\u4e2a\u201c\u541b\u5b50\u534f\u8bae\u201d\uff0c\u5e76\u4e0d\u610f\u5473\u7740\u6bcf\u4e2a\u641c\u7d22\u5f15\u64ce\u90fd\u4e25\u683c\u9075\u5b88robots.txt\u7684\u89c4\u5219\uff0c\u6309\u7167\u4f60\u7684\u610f\u56fe\u6765\u6293\u53d6\u7f51\u9875\uff0c\u8981\u6e05\u695a\uff0c\u4f60\u7684\u7f51\u7ad9\u5728\u641c\u7d22\u5f15\u64ce\u9762\u524d\u6ca1\u6709\u4ec0\u4e48\u201c\u79d8\u5bc6\u201d\uff1b<\/p>\n<p>7\uff09\u8981\u68c0\u67e5robots.txt\u6587\u4ef6\u662f\u5426\u8fbe\u5230\u4e86\u9884\u671f\u7684\u6548\u679c\uff0c\u53ef\u4ee5\u901a\u8fc7\u8c37\u6b4c\u7f51\u7ad9\u7ad9\u957f\u5de5\u5177\u4e2d\u7684robots.txt\u5de5\u5177\u6d4b\u8bd5\uff1b<\/p>\n<p>8\uff09\u7981\u6b62\u641c\u7d22\u5f15\u64ce\u6536\u5f55\u5355\u72ec\u9875\u9762\u3002robots.txt\u6587\u4ef6\u4e3b\u8981\u662f\u9650\u5236\u641c\u7d22\u5f15\u64ce\u5bf9\u6574\u4e2a\u7ad9\u70b9\u6216\u8005\u76ee\u5f55\u7684\u8bbf\u95ee\u60c5\u51b5\uff0c\u800c<strong>Robots Meta<\/strong>\u6807\u7b7e\u5219\u662f\u9488\u5bf9\u4e00\u4e2a\u5177\u4f53\u7684\u9875\u9762\u3002\u901a\u5e38Robots Meta\u6807\u7b7e\u7684\u9ed8\u8ba4\u503c\u662findex\u548cfollow\uff0c\u53ea\u6709inktomi\u9664\u5916\uff0c\u5bf9\u4e8e\u5b83\uff0c\u9ed8\u8ba4\u503c\u662findex,nofollow\u3002Robots Meta\u6807\u7b7e\u603b\u5171\u6709\u56db\u79cd\u7ec4\u5408\uff1a<\/p>\n<pre>&lt;meta name=\"robots\" content=\"index,follow\"&gt;\u6216&lt;meta name=\"robots\" content=\"all\"&gt;\r\n&lt;meta name=\"robots\" content=\"noindex,follow\"&gt;\r\n&lt;meta name=\"robots\" content=\"index,nofollow\"&gt;\r\n&lt;meta name=\"robots\" content=\"noindex,nofollow\"&gt;\u6216&lt;meta name=\"robots\" content=\"none\"&gt;<\/pre>\n<p>9\uff09\u76ee\u524d\u7edd\u5927\u591a\u6570\u641c\u7d22\u5f15\u64ce\u662f\u652f\u6301robots.txt\u6587\u4ef6\u89c4\u5219\u7684\uff0cRobots META\u6807\u7b7e\u5219\u4e0d\u662f\u3002Google\u4e00\u5982\u65e2\u5f80\u5730\u5f3a\u5927\uff0c\u5bf9\u8fd9\u4e24\u79cd\u5f62\u5f0f\u90fd\u5b8c\u7f8e\u652f\u6301\uff1b<\/p>\n<p>10\uff09\u5efa\u8bae\u7ad9\u70b9\u6839\u76ee\u5f55\u4e0b\u8981\u653e\u4e00\u4e2arobots.txt\u6587\u4ef6\uff0c\u54ea\u6015\u662f\u7a7a\u767d\u6587\u4ef6\u4e5f\u597d\u3002\u867d\u8bf4\u4e0d\u653erobots.txt\u6587\u4ef6\u4e5f\u6ca1\u4ec0\u4e48\u5f71\u54cd\uff0c\u4f46\u662f\u8003\u8651\u5230Robot\u8bbf\u95ee\u7ad9\u70b9\u65f6\uff0c\u7b2c\u4e00\u76ee\u6807\u5c31\u662frobots.txt\u6587\u4ef6\uff0c\u8fd8\u662f\u987a\u7740\u5b83\u7684\u610f\u601d\u529e\u5427\uff0c\u8c01\u8ba9\u6e38\u620f\u89c4\u5219\u662f\u5b83\u5b9a\u7684\u5462\uff1f\uff01<\/p>\n","protected":false},"excerpt":{"rendered":"<p>robots.txt\u662f\u4e00\u4e2a\u7b80\u5355\u7684\u4ee5.txt\u7ed3\u5c3e\u7684\u6587\u672c\u6587\u4ef6\uff0c\u662f\u641c\u7d22\u5f15\u64ceRobot\uff08\u4e5f\u53eb\u641c\u7d22\u5f15\u64ce\u673a\u5668\u4eba\uff09\u7a0b\u5e8f\u6293\u53d6\u7f51 &#8230; <a title=\"robots.txt\u6587\u4ef6\u662f\u4ec0\u4e48\uff1f\" class=\"read-more\" href=\"https:\/\/cn.hostease.com\/xueyuan\/jishu\/seo\/robots-txt%e6%96%87%e4%bb%b6%e6%98%af%e4%bb%80%e4%b9%88%ef%bc%9f\/\" aria-label=\"\u9605\u8bfb robots.txt\u6587\u4ef6\u662f\u4ec0\u4e48\uff1f\">\u9605\u8bfb\u66f4\u591a<\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_jetpack_memberships_contains_paid_content":false,"footnotes":""},"categories":[18],"tags":[564],"class_list":["post-1839","post","type-post","status-publish","format-standard","hentry","category-seo","tag-robots-txt"],"aioseo_notices":[],"jetpack_featured_media_url":"","jetpack-related-posts":[],"jetpack_sharing_enabled":true,"_links":{"self":[{"href":"https:\/\/cn.hostease.com\/xueyuan\/wp-json\/wp\/v2\/posts\/1839","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/cn.hostease.com\/xueyuan\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/cn.hostease.com\/xueyuan\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/cn.hostease.com\/xueyuan\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/cn.hostease.com\/xueyuan\/wp-json\/wp\/v2\/comments?post=1839"}],"version-history":[{"count":1,"href":"https:\/\/cn.hostease.com\/xueyuan\/wp-json\/wp\/v2\/posts\/1839\/revisions"}],"predecessor-version":[{"id":1840,"href":"https:\/\/cn.hostease.com\/xueyuan\/wp-json\/wp\/v2\/posts\/1839\/revisions\/1840"}],"wp:attachment":[{"href":"https:\/\/cn.hostease.com\/xueyuan\/wp-json\/wp\/v2\/media?parent=1839"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/cn.hostease.com\/xueyuan\/wp-json\/wp\/v2\/categories?post=1839"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/cn.hostease.com\/xueyuan\/wp-json\/wp\/v2\/tags?post=1839"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}