User-agent: ia_archiver
Disallow: /

User-agent: duggmirror
Disallow: /

User-agent: *
disallow: /categories/
disallow: /topic/
disallow: /user/
disallow: /cgi-bin/

User-agent: Googlebot
disallow: /*.php$
disallow: /*.js$
disallow: /*.cgi$
disallow: /*.xhtml$
disallow: /*.php*
disallow: /*?*
disallow: /*.inc$
disallow: /*.css$
disallow: /*.txt$