"*", , /robots.txt

Disallow

(record) (line) User-Agent - Disallow

/robots.txt , , , .

1: 

# robots.txt for http://www.site.com

        User-Agent: *

        Disallow: /cyberworld/map/ # this is an infinite virtual URL space
        Disallow: /tmp/ # these will soon disappear

1 /cyberworld/map/ /tmp/.

2: 

# robots.txt for http://www.site.com

        User-Agent: *

        Disallow: /cyberworld/map/ # this is an infinite virtual URL space

# Cybermapper knows where to go

        User-Agent: cybermapper

        Disallow:

2 /cyberworld/map/, cybermapper .

3:

# robots.txt for http://www.site.com

        User-Agent: *

        Disallow: /

3 .

, , User-Agent , .

Martijn Koster, m.koster@webcrawler.com

: , info@citmgu.ru