Standard library Β· The web
web/http
Imported as import "web/http" as http, its names are then http.β¦. Every signature below is the one the checker infers.
A small HTTP/1.1 server, written in Rill on top of the socket builtins. One strand per connection: while a connection has nothing to say its strand is parked, so idle clients cost nothing.
port = 18901 server = listen_at("127.0.0.1", port)? spawn http.accept_n(server, \req -> http.text_response(200, "hello " + http.param(http.query_of(req.path), "name")), 1) raw = http.get("127.0.0.1", port, "/greet?name=ada") http.status_of(raw) # => 200 http.response_body(raw) # => hello ada
Types
Request
Request(method: Str, path: Str, headers: List(Header), body: Str)
Header
A name and what it was given, as they arrived. The name is folded to lower case on the way in, because HTTP says the two are the same header and a program that compares them itself will one day meet a proxy that disagrees about the capital letters.
Header(name: Str, value: Str)
Response
A reply carries what it is as well as what it says: a browser shown text/plain renders the tags rather than the page, so a server that can only say one type can only serve one kind of thing. Anything else it has to say β how long this may be cached, what it is a version of β goes in headers, which the three helpers below leave empty.
Response(status: Int, content_type: Str, headers: List(Header), body: Str)
Functions
fn text_response(status: Int, body: Str) -> Response
r = http.text_response(200, "ok") r.status # => 200 r.content_type # => text/plain
fn html_response(status: Int, body: Str) -> Response
http.html_response(404, "<h1>gone</h1>").content_type # => text/html; charset=utf-8
fn json_response(status: Int, body: Str) -> Response
http.json_response(200, "{}").content_type # => application/json
fn with_header(res: Response, name: Str, value: Str) -> Response
One more line on the way out.
r = http.with_header(http.text_response(200, ""), "Cache-Control", "no-store") r.headers # => Cons(http.Header(Cache-Control, no-store), Nil)
fn header(req: Request, name: Str) -> Str
What a request said under that name, or "" β the lookup a handler wants, since a header that is not there and one that is empty mean the same thing to almost every caller.
req = http.parse_request("GET / HTTP/1.1\r\nHost: h\r\nX-Thing: yes\r\n\r\n") http.header(req, "x-thing") # => yes http.header(req, "X-Thing") # => yes http.header(req, "other") # =>
fn header_in(headers: List(Header), name: Str) -> Str
http.header_in(Cons(http.Header("host", "h"), Nil), "host") # => h
fn cookie(req: Request, name: Str) -> Str
What the browser sent under that name, or "". Cookies arrive as one header β a=1; b=2 β so this is a walk over that line rather than another lookup.
A browser's localStorage and sessionStorage are not here and cannot be: they never leave the page. What a server sees is what the browser sends, and that is a cookie (which it sends by itself) or whatever the page decides to put in a header or a body.
req = http.parse_request("GET / HTTP/1.1\r\nCookie: a=1; who=ada\r\n\r\n") http.cookie(req, "who") # => ada http.cookie(req, "nope") # =>
fn cookie_in(pairs: List(Str), name: Str) -> Str
http.cookie_in(Cons("a=1", Cons(" b=2", Nil)), "b") # => 2
fn set_cookie(res: Response, name: Str, value: Str, rest: Str) -> Response
One Set-Cookie line; a response may carry several, since its headers are a list rather than a table. What goes in rest is the browser's own language:
# not run set_cookie(res, "who", id, "; Path=/; HttpOnly; SameSite=Lax; Max-Age=86400") r = http.set_cookie(http.text_response(200, ""), "who", "ada", "; Path=/; HttpOnly") r.headers # => Cons(http.Header(Set-Cookie, who=ada; Path=/; HttpOnly), Nil)
fn parse_request(raw: Str) -> Request
req = http.parse_request("POST /a?b=1 HTTP/1.1\r\nHost: h\r\nContent-Length: 2\r\n\r\nhi") req.method # => POST req.path # => /a?b=1 req.body # => hi
fn path_of(target: Str) -> Str
The path with the query taken off.
http.path_of("/a/b?x=1") # => /a/b http.path_of("/a/b") # => /a/b
fn query_of(target: Str) -> Str
The query string, without the ?.
http.query_of("/a/b?x=1&y=2") # => x=1&y=2 http.query_of("/a/b") # =>
fn param(qs: Str, name: Str) -> Str
What the query said under that name, decoded, or "".
http.param("q=two+words&n=1", "q") # => two words http.param("q=1", "z") # =>
fn params(qs: Str) -> List(Header)
Every parameter, in the order they were written, decoded.
http.params("a=1&b=%41") # => Cons(http.Header(a, 1), Cons(http.Header(b, A), Nil))
fn decoded(s: Str) -> Str
%41 is an A and + is a space β the second only inside a query, which is where this is used. A % that is not followed by two digits is a %: a decoder that refuses would turn a badly written link into a 400 rather than into what the reader meant.
http.decoded("a%20b+c") # => a b c http.decoded("50%") # => 50%
fn encoded(s: Str) -> Str
The other way, for a link a server writes itself: everything but the unreserved characters becomes %XX, so a value with a space or an ampersand in it survives being put in a query string.
http.encoded("a b&c") # => a%20b%26c
fn status_text(code: Int) -> Str
http.status_text(404) # => Not Found http.status_text(503) # => Service Unavailable
fn response_bytes(res: Response, keep: Bool) -> Str
Named for what it makes rather than for the act of making it: render is the trait method lib/web/tmpl.rill owns, and a program that serves pages built from templates wants both files at once.
starts_with(http.response_bytes(http.text_response(204, ""), false), "HTTP/1.1 204 No Content\r\nContent-Type: text/plain\r\nContent-Length: 0\r\nConnection: close\r\nDate: ") # => true
fn http_date(millis: Int) -> Str
Sun, 06 Nov 1994 08:49:37 GMT β the one date format every HTTP client can read. The calendar is worked out rather than looked up: days since 1970 become a civil date by counting four-hundred-year eras, which repeat exactly, so there is no table to be wrong.
http.http_date(784111777000) # => Sun, 06 Nov 1994 08:49:37 GMT
fn handle_conn(conn: Int, handler: (Request) -> Response) -> Int
A connection serves request after request until one side asks to stop, so a client that makes many requests pays for a single accept.
# not run conn = tcp_accept(server) http.handle_conn(conn, \req -> http.text_response(200, "hi")) # every request on it, until a side closes
fn serve_taking(server: Int, handler: (Request) -> Response, taken: (Int, Str) -> Bool) -> 'a
The same loop with a way out of it: taken is shown every request before it is answered and says whether it has taken the connection for itself. That is what an upgrade is β a WebSocket, a stream, anything that stops being request-and-answer β and once it says yes this loop lets go: the socket belongs to whoever took it, closing included.
# not run h.serve_taking(server, \req -> route(req), \conn, raw -> if ws.ws_wanted(raw) then live(conn, raw) else false)
Every connection's strand is handed the same handler and taken, and what a closure holds is not in its type, so a --parallel build asks for the claim apart makes: that what they hold is read, or written apart. A handler that writes storage it closed over from every connection β a counter, a table β is a race on several workers, and that is the program's to keep out of a handler or to keep on one worker.
# not run server = listen(8080)? http.serve_taking(server, \req -> route(req), \conn, raw -> if ws.ws_wanted(raw) then live(conn, raw) else false)
fn serve_n(port: Int, handler: (Request) -> Response, count: Int) -> Unit
Serves count requests and then stops, which is what makes a server testable. serve below never returns.
port = 18902 spawn http.serve_n(port, \req -> http.text_response(200, req.path), 2) sleep_ms(50) # the strand has to get to its listen http.response_body(http.get("127.0.0.1", port, "/one")) # => /one http.response_body(http.get("127.0.0.1", port, "/two")) # => /two
fn accept_n(server: Int, handler: (Request) -> Response, count: Int) -> Unit
port = 18903 server = listen_at("127.0.0.1", port)? spawn http.accept_n(server, \req -> http.text_response(200, "x"), 1) http.status_of(http.get("127.0.0.1", port, "/")) # => 200
fn serve(port: Int, handler: (Request) -> Response) -> Unit
# not run http.serve(8080, \req -> http.text_response(200, "hello")) # never returns
fn serve_at(host: Str, port: Int, handler: (Request) -> Response) -> Unit
The same on one address β serve_at("127.0.0.1", port, handler) for a server that is for this machine alone.
# not run http.serve_at("127.0.0.1", 8080, \req -> http.text_response(200, "hello")) # this machine alone
fn accept_forever(server: Int, handler: (Request) -> Response) -> 'a
# not run server = listen(8080)? http.accept_forever(server, http.wrap(Cons(http.rate_limit(20), Nil), \req -> route(req)))
fn wrap(layers: List(('a, ('a) -> 'b) -> 'b), handler: ('a) -> 'b) -> ('a) -> 'b
-- middleware ----------------------------------------------------------------
A handler answers a request: Request -> Response. A middleware is one layer around that β it is handed the request and the handler underneath it, and decides what to do with both:
# not run fn timed(req, next) = t0 = now_nanos() res = next(req) println(req.path + " " + int_to_str((now_nanos() - t0) / 1000) + " us") res
It may answer without calling next at all, which is what a rate limiter and an authenticator are for; it may change the request it passes down; and it may look at the answer on the way back out.
wrap stacks them. The first in the list is the outermost β it sees the request first and the answer last:
# not run answer = wrap(Cons(rate_limit(20), Cons(timed, Nil)), \req -> route(db, req)) accept_forever(server, answer)
There is nothing else to it: what comes out of wrap is a handler like any other, so a middleware stack can be built once and handed to serve, serve_n or accept_forever unchanged.
timed = \req, next -> http.with_header(next(req), "X-Timed", "yes") answer = http.wrap(Cons(timed, Nil), \req -> http.text_response(200, "inner")) res = answer(http.parse_request("GET / HTTP/1.1\r\n\r\n")) res.body # => inner res.headers # => Cons(http.Header(X-Timed, yes), Nil)
fn rate_limit(per_second: Int) -> (Request, (Request) -> Response) -> Response
How many requests a second one caller may have, and what the rest are told.
Three things decide whether a limiter is worth having, and all three are easy to get wrong:
Who is counted. Not the route. A limit of twenty a second on POST /doc/<id> is no limit at all when there are a million ids: the caller simply moves along them and never meets the count. The bucket is the caller, so what is bounded is what a caller can make this server do, which is the thing worth bounding.
Whether the caller can be believed. A caller names itself in X-Forwarded-For, and a caller that writes its own name has no limit β a fresh name is a fresh bucket, one header away. So the header is ignored unless this server is told how many proxies stand in front of it, and then only their entries are read; see client_of.
Reads as well as writes. A read that groups a million rows costs more than a write of one cell. Reads are counted too, in a bucket of their own so that browsing and editing do not crowd each other out, and usually at a looser rate β a page fetches a handful of things at once, which is one person behaving normally.
rate_limit(n) is writes only and trusts nothing, which is what a server reached directly should do. rate_limits(writes, reads, hops) is the whole of it.
The window is a whole second that everyone shares rather than one per caller, so a burst that straddles the boundary can be twice the limit. It is a guard against a caller that will not stop, not a token bucket.
limited = http.wrap(Cons(http.rate_limit(1), Nil), \req -> http.text_response(200, "ok")) req = http.parse_request("POST / HTTP/1.1\r\n\r\n") limited(req).status # => 200 limited(req).status # => 429
fn rate_limits(writes: Int, reads: Int, hops: Int) -> (Request, (Request) -> Response) -> Response
limited = http.wrap(Cons(http.rate_limits(1, 100, 0), Nil), \req -> http.text_response(200, "ok")) limited(http.parse_request("GET / HTTP/1.1\r\n\r\n")).status # => 200 limited(http.parse_request("POST / HTTP/1.1\r\n\r\n")).status # => 200 limited(http.parse_request("POST / HTTP/1.1\r\n\r\n")).status # => 429
fn client_of(req: Request, hops: Int) -> Str
Which name in X-Forwarded-For is the caller, given how many proxies stand in front of this server.
Every hop appends the address it saw to the end of the header, so the entries at the end are the ones written by the proxies and the entries at the front are whatever the caller sent β which is anything it likes. With hops proxies in front, the caller is hops from the right: the last entry is what the nearest proxy saw, and the one before it is what the proxy before that saw.
# not run X-Forwarded-For: 9.9.9.9, 203.0.113.7, 172.68.1.1 ^ sent ^ the caller ^ seen by the last proxy (hops = 2)
hops of 0 says there is no proxy, and then the header is not read at all: every caller counts as the same one, which is a limit on the server rather than on a client and is the honest thing to say about a server that can only see its own socket. A header with fewer entries than there are proxies has been stripped or the count is wrong β either way it cannot be read, and everyone shares a bucket again rather than being let through.
A proxy that overwrites the header instead of appending β proxy_set_header X-Forwarded-For $remote_addr in nginx, where $remote_addr is already the real caller β leaves exactly one entry, and hops is 1. That is the arrangement to prefer: nothing the caller sends survives it.
req = http.parse_request("GET / HTTP/1.1\r\nX-Forwarded-For: 9.9.9.9, 203.0.113.7, 172.68.1.1\r\n\r\n") http.client_of(req, 2) # => 203.0.113.7 http.client_of(req, 1) # => 172.68.1.1 http.client_of(req, 0) # => anyone
fn ask(host: Str, port: Int, method: Str, path: Str, headers: List(Header), body: Str) -> Str
One request, spelled out: a method, headers of your own, a body. get and get_all below are this with all three decided. The answer comes back raw, for status_of and response_body to take apart.
port = 18904 server = listen_at("127.0.0.1", port)? spawn http.accept_n(server, \req -> http.text_response(200, req.method + " " + http.header(req, "x-k") + " " + req.body), 1) raw = http.ask("127.0.0.1", port, "PUT", "/x", Cons(http.Header("X-K", "v"), Nil), "body") http.response_body(raw) # => PUT v body
fn get(host: Str, port: Int, path: Str) -> Str
port = 18905 server = listen_at("127.0.0.1", port)? spawn http.accept_n(server, \req -> http.text_response(200, "got " + req.path), 1) http.response_body(http.get("127.0.0.1", port, "/it")) # => got /it
fn get_all(host: Str, port: Int, paths: List(Str)) -> List(Str)
Every path down one connection, which is what keep-alive buys: one accept, one handshake, one strand on the server.
port = 18906 server = listen_at("127.0.0.1", port)? spawn http.accept_n(server, \req -> http.text_response(200, req.path), 1) map(http.get_all("127.0.0.1", port, Cons("/a", Cons("/b", Nil))), http.response_body) # => Cons(/a, Cons(/b, Nil))
fn read_response(fd: Int, acc: Str) -> Str
A keep-alive server never closes, so the client stops at Content-Length rather than at end of stream.
# not run raw = http.read_response(fd, "") # a whole answer off a socket the caller opened
fn content_length(raw: Str) -> Int
18 = the length of "-Length: ".
http.content_length("HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello") # => 5
fn response_body(raw: Str) -> Str
http.response_body("HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello") # => hello
fn status_of(raw: Str) -> Int
http.status_of("HTTP/1.1 404 Not Found\r\n\r\n") # => 404