import gleam/http/response.{Response} import gleam/http/request.{Request} import gleam/http.{Get} import gleam/bit_builder.{BitBuilder} import gleam/erlang/process import gleam/erlang/atom import gleam/io import gleam/int import gleam/string import gleam/bit_string import gleam/list import gleam/map.{Map} import gleam/uri import gleam/hackney import mist external type Index external type Field external fn index_new(atom.Atom) -> Index = "Elixir.Haystack.Index" "new" external fn index_ref(Index, Field) -> Index = "Elixir.Haystack.Index" "ref" external fn index_field(Index, Field) -> Index = "Elixir.Haystack.Index" "field" external fn field_term(String) -> Field = "Elixir.Haystack.Index.Field" "term" external fn field_new(String) -> Field = "Elixir.Haystack.Index.Field" "new" external fn index_add(Index, List(a)) -> Index = "Elixir.Haystack.Index" "add" external fn index_search(Index, String) -> List(Map(atom.Atom, String)) = "Elixir.Haystack.Index" "search" pub external fn inspect(a) -> a = "Elixir.IO" "inspect" external type Query external type Clause external type Expression external fn query_new() -> Query = "Elixir.Haystack.Query" "new" external fn query_clause(Query, Clause) -> Query = "Elixir.Haystack.Query" "clause" external fn query_run(Query, Index) -> List(Map(atom.Atom, String)) = "Elixir.Haystack.Query" "run" external fn clause_new(atom.Atom) -> Clause = "Elixir.Haystack.Query.Clause" "new" external fn query_expressions(Clause, List(Expression)) -> Clause = "Elixir.Haystack.Query.Clause" "expressions" external fn query_expression_new(atom.Atom, List(#(atom.Atom, String))) -> Expression = "Elixir.Haystack.Query.Expression" "new" external fn tokenize(String) -> List(Map(atom.Atom, String)) = "Elixir.Haystack.Tokenizer" "tokenize" fn unpercent(encoded: String) -> String { let #([head], chunks) = encoded |> string.split(on: "%") |> list.split(at: 1) let assert Ok(res) = chunks |> list.map(fn(str) { case string.length(str) < 2 { True -> bit_string.from_string(str) False -> { let assert Ok(codepoint) = str |> string.slice(at_index: 0, length: 2) |> int.base_parse(16) <> } } }) |> list.prepend(bit_string.from_string(head)) |> bit_string.concat |> bit_string.to_string res } fn the_lookup( query: String, stations: Map(String, String), ds100s: Map(String, String), fuzzy: fn(String) -> List(String) ) -> #(Int, String) { case map.get(ds100s, query) { Ok(name) -> #(200, name) _ -> { io.println(query) case map.get(stations, query) { Ok(ds100) -> #(200, ds100) _ -> { let results = fuzzy(query) |> list.filter_map(fn (res) { map.get(ds100s, string.uppercase(res)) }) case results { // results -> { // let names = results // |> list.map (fn (res) { // map.get(ds100s, string.uppercase(res)) // |> result.map(fn(a) { "/" <> a }) // |> result.unwrap("/")}) // #(200, string.join(names, "\n")) // } [res] -> #(302, res) [res, ..] -> #(302, res) _ -> #(404, "??") } } } } } } fn lookup_station( request: Request(t), stations: Map(String, String), ds100s: Map(String, String), baseurl: String, fuzzy: fn (String) -> List(String) ) -> Response(BitBuilder) { let #(code, text) = case request { Request(method: Get, path: "/help", ..) | Request(method: Get, path: "/", ..) -> #( 200, "ds100 → Name: " <> baseurl <> "/NN\n" <> "Name → ds100: " <> baseurl <> "/Nürnberg Hbf", ) Request(method: Get, path: "/" <> path, ..) -> the_lookup(unpercent(path), stations, ds100s, fuzzy) _ -> #(404, "intended usage is e.g. curl " <> baseurl <> "/FF") } let body = bit_builder.from_string(text) response.new(code) |> response.prepend_header( "x-data-source", "https://data.deutschebahn.com/dataset/data-betriebsstellen.html", ) |> response.prepend_header( "x-sources-at", "https://stuebinm.eu/git/bahnhof.name", ) |> response.prepend_header("content-type", "text/plain; charset=utf8") |> fn (a) { case code == 302 { True -> response.prepend_header(a, "location", text) _ -> a } } |> response.set_body(body) } fn fetch_data() -> Result(String, hackney.Error) { let assert Ok(uri) = uri.parse( "https://download-data.deutschebahn.com/static/datasets/betriebsstellen/DBNetz-Betriebsstellenverzeichnis-Stand2021-10.csv", ) let assert Ok(request) = request.from_uri(uri) let assert Ok(response) = hackney.send(request) // some ü are corrupted for some reason Ok(string.replace(response.body, "�", "ü")) } fn read_csv() -> List(#(String, String)) { // let assert Ok(contents) = file.read(path) let assert Ok(contents) = fetch_data() contents // the file doesn't use quotes, so this is fine |> string.split(on: "\n") |> list.map(fn(a) { string.split(a, on: ";") }) |> list.filter_map(fn(fields) { case fields { [_, ds100, name, ..] -> Ok(#(name, ds100)) _ -> Error(fields) } }) } pub fn main() { let baseurl = "https://bahnhof.name" let stations = read_csv() let stationmap = stations |> map.from_list let ds100map = stations |> list.map(fn(a) { #(a.1, a.0) }) |> map.from_list let ref = atom.create_from_string("ref") let index = index_new(atom.create_from_string("stations")) |> index_ref(field_term("id")) |> index_field(field_new("name")) |> index_add(stations |> list.map(fn(tuple) {case tuple { #(name, ds100) -> map.from_list([#("id", ds100), #("name", name)] )}})) let fuzzy = fn(searchterm: String) -> List(String) { let query = query_new() let match = atom.create_from_string("match") let field = atom.create_from_string("field") let term = atom.create_from_string("term") let expressions = tokenize(inspect(searchterm)) |> list.filter_map(fn (a) { map.get(a, atom.create_from_string("v")) }) |> list.map(fn (token) { query_expression_new(match, [#(field, "name"), #(term, token)]) }) let clause = query_expressions(clause_new(atom.create_from_string("all")), expressions) let query = query_clause(query, clause) let matches = query_run(query, index) |> list.filter_map(fn (a) { map.get(a, ref) }) inspect(matches) case list.length(matches) > 5 { True -> { let query = query_new() let clause = query_expressions( clause_new(atom.create_from_string("all")), [query_expression_new(match, [#(field, "name"), #(term, "hbf")]) , ..expressions] ) let query = query_clause(query, clause) let narrow = query_run(query, index) |> list.filter_map(fn (a) { map.get(a, ref) }) case narrow { [] -> matches _ -> narrow } } _ -> matches } } io.println("compiled index, starting server …") let _ = mist.run_service( 2345, fn(req) { lookup_station(req, stationmap, ds100map, baseurl, fuzzy) }, max_body_limit: 100, ) process.sleep_forever() }