; Word statistics over a paragraph: a frequency table, the longest words,
; and a grade for how varied the vocabulary is.

enum Grade
  poor = 1
  fair = 2
  rich = 3

struct Count
  word: [const u8]
  n: i32

fn is-letter(c: u8) -> bool
  (c >= \a and c <= \z)
    or (c >= \A and c <= \Z)
    or c == \'

; The words of text, lowercased, in order.
fn words(text: [const u8]) -> Vec([const u8])
  let out = vec-new([const u8])
      lowered = to-lower(text)
      lower = bytes-view(lowered)
      i = 0
      n = length(lower)
  while :scan i < n
    until i >= n or is-letter(lower[i])
      i += 1
    if i >= n
      break :scan
    let start = i
    while i < n and is-letter(lower[i])
      i += 1
    push(out, slice(lower, start, i))
  out

fn tally(ws: [[const u8]]) -> Vec(Count)
  let seen = map-new(str, i32)
  defer free(seen)
  let counts = vec-new(Count)
  for i in range(length(ws))
    match get(seen, str(ws[i]))
      Some(k) -> counts[k].n += 1
      None ->
        put(seen, str(ws[i]), i32(length(counts)))
        push(counts, Count{.word ws[i], .n 1})
  counts

fn grade(distinct: i32, total: i32) -> Grade
  let ratio = distinct * 10 / max(total, 1)
  if ratio >= 7 then :rich else if ratio >= 4 then :fair else :poor

fn describe(g: Grade) -> str
  match i32(g)
    1 -> "repetitive"
    2 -> "ordinary"
    _ -> "varied"

fn main() -> i32
  let text = "The cat saw the dog. The dog didn't see the cat, but the bird saw both!"
  let ws = words(bytes-view(text))
  let counts = tally(slice(ws))
  let by-count = fn(a: Count, b: Count) -> bool =>
    if a.n != b.n
      return a.n > b.n
    is-bytes-less(a.word, b.word)
  sort-by(slice(counts), by-count)
  for i in range(3)
    let {.word .n} = counts[i]
    println(str(word), n)
  let top: [3 i32] = [counts[0].n counts[1].n counts[2].n]
      [most & others] = top
  println("most frequent seen", most, "times, then", others)
  let total = i32(length(ws))
      distinct = i32(length(counts))
      g = grade(distinct, total)
  println(distinct, "of", total, "distinct:", describe(g))
  let longest = reduce(slice(ws), slice(ws[0], 0, 0), fn(a, b) =>
    if length(b) > length(a) then b else a)
  println("longest", str(longest))
  let short = 0 < length(longest) < 5
  match short
    true -> println("short words only")
    false -> println("some long words")
  println("three distinct counts?", !=(counts[0].n, counts[1].n, counts[2].n))
  0