Commit 69a32cc3435ccd2d5b83ef5b83832745abe29af5
Add config for Disallow: / in robots.txt
cel committed on 4/27/2018, 5:12:22 AMParent: 86801890069c72884af76b73462e38b8b85697db
Files changed
| index.js | changed |
| index.js | ||
|---|---|---|
| @@ -92,9 +92,9 @@ | ||
| 92 | 92 | switch (m[2]) { |
| 93 | 93 | case '%': return serveId(req, res, m[1], m[3], m[5]) |
| 94 | 94 | case '@': return serveFeed(req, res, m[1], m[3], m[5]) |
| 95 | 95 | case '&': return serveBlob(req, res, sbot, m[1]) |
| 96 | - default: return servePath(req, res, m[4]) | |
| 96 | + default: return servePath(req, res, m[4], conf) | |
| 97 | 97 | } |
| 98 | 98 | } |
| 99 | 99 | |
| 100 | 100 | function serveFeed(req, res, feedId, ext) { |
| @@ -452,11 +452,11 @@ | ||
| 452 | 452 | case 'rss': return 'text/xml' |
| 453 | 453 | } |
| 454 | 454 | } |
| 455 | 455 | |
| 456 | -function servePath(req, res, url) { | |
| 456 | +function servePath(req, res, url, conf) { | |
| 457 | 457 | switch (url) { |
| 458 | - case '/robots.txt': return res.end('User-agent: *') | |
| 458 | + case '/robots.txt': return serveRobots(req, res, conf) | |
| 459 | 459 | } |
| 460 | 460 | var m = /^(\/?[^\/]*)(\/.*)?$/.exec(url) |
| 461 | 461 | switch (m[1]) { |
| 462 | 462 | case '/static': return serveStatic(req, res, m[2]) |
| @@ -490,8 +490,13 @@ | ||
| 490 | 490 | fs.createReadStream(file).pipe(res) |
| 491 | 491 | }) |
| 492 | 492 | } |
| 493 | 493 | |
| 494 | +function serveRobots(req, res, conf) { | |
| 495 | + res.end('User-agent: *' | |
| 496 | + + (conf.disallowRobots ? '\nDisallow: /' : '')) | |
| 497 | +} | |
| 498 | + | |
| 494 | 499 | function prepend(fn, arg) { |
| 495 | 500 | return function (read) { |
| 496 | 501 | return function (abort, cb) { |
| 497 | 502 | if (fn && !abort) { |
Built with git-ssb-web