User-Agent: *

## Prevent bots from walking the many multicast Beacon graphs
Disallow: /beacon?*src*

## Don't want crawlers to crawl the TWiki boilerplate/documentation
Disallow: /Main
Disallow: /cgi-bin/twiki/view/Main
Disallow: /Sandbox
Disallow: /cgi-bin/twiki/view/Sandbox
Disallow: /TWiki
Disallow: /cgi-bin/twiki/view/TWiki
Disallow: /Trash
Disallow: /cgi-bin/twiki/view/Trash

## Disallow most TWiki actions except for "view".
## Actually "view" is bad enough because the crawler can be lead down
## all kinds of paths with different skins, different revisions, etc.
## Although maybe it's not bad to have all revisions archived in the
## Search Engines' caches
##
Disallow: /cgi-bin/twiki/actionnotify
Disallow: /cgi-bin/twiki/ambarsave
Disallow: /cgi-bin/twiki/attach
Disallow: /cgi-bin/twiki/changes
Disallow: /cgi-bin/twiki/compare
Disallow: /cgi-bin/twiki/configure
Disallow: /cgi-bin/twiki/edit
Disallow: /cgi-bin/twiki/geturl
Disallow: /cgi-bin/twiki/installpasswd
Disallow: /cgi-bin/twiki/login
Disallow: /cgi-bin/twiki/logon
Disallow: /cgi-bin/twiki/mailnotify
Disallow: /cgi-bin/twiki/manage
Disallow: /cgi-bin/twiki/oops
Disallow: /cgi-bin/twiki/passwd
Disallow: /cgi-bin/twiki/preview
Disallow: /cgi-bin/twiki/publish
Disallow: /cgi-bin/twiki/rdiff
Disallow: /cgi-bin/twiki/rdiffauth
Disallow: /cgi-bin/twiki/register
Disallow: /cgi-bin/twiki/rename
Disallow: /cgi-bin/twiki/resetpasswd
Disallow: /cgi-bin/twiki/rest
Disallow: /cgi-bin/twiki/save
Disallow: /cgi-bin/twiki/savemulti
Disallow: /cgi-bin/twiki/search
Disallow: /cgi-bin/twiki/setlib.cfg
Disallow: /cgi-bin/twiki/statistics
Disallow: /cgi-bin/twiki/testenv
Disallow: /cgi-bin/twiki/twiki
Disallow: /cgi-bin/twiki/upload
Disallow: /cgi-bin/twiki/viewauth
Disallow: /cgi-bin/twiki/viewfile
Disallow: /*?raw
Disallow: /*?rev
Disallow: /*?t=
Disallow: /*?skin
Disallow: /*?sortcol

Sitemap: http://kb.pert.geant2.net/sitemap.xml