18 lines
417 B
Racket
18 lines
417 B
Racket
#lang racket
|
|
(define (robots-field name (body '()))
|
|
(define robots-field-base (string-append name ":"))
|
|
(if (nil? body) robots-field-base
|
|
(string-append robots-field-base " " body)))
|
|
|
|
(define (robots-ua (name "*"))
|
|
(string-append "User-agent: " name))
|
|
|
|
(define (robots-disallow list)
|
|
(if (empty? list)
|
|
"Disallow:"
|
|
|
|
(define (robots-config (#:crawl-delay 10))
|
|
(string-append
|
|
(robots-ua)
|
|
(
|
|
|