85 lines
2.3 KiB
Plaintext
85 lines
2.3 KiB
Plaintext
; Word statistics over a paragraph: a frequency table, the longest words,
|
|
; and a grade for how varied the vocabulary is.
|
|
|
|
enum Grade
|
|
poor = 1
|
|
fair = 2
|
|
rich = 3
|
|
|
|
struct Count
|
|
word: [const u8]
|
|
n: i32
|
|
|
|
fn letter?(c: u8) -> bool
|
|
(c >= \a and c <= \z)
|
|
or (c >= \A and c <= \Z)
|
|
or c == \'
|
|
|
|
; The words of text, lowercased, in order.
|
|
fn words(text: [const u8]) -> Vec([const u8])
|
|
let out = vec-new([const u8])
|
|
lower = to-lower(text)
|
|
i = 0
|
|
n = length(lower)
|
|
while :scan i < n
|
|
until i >= n or letter?(lower[i])
|
|
i += 1
|
|
if i >= n
|
|
break :scan
|
|
let start = i
|
|
while i < n and letter?(lower[i])
|
|
i += 1
|
|
push(out, slice(lower, start, i))
|
|
out
|
|
|
|
fn tally(ws: [[const u8]]) -> Vec(Count)
|
|
let seen = map-new(str, i32)
|
|
defer free(seen)
|
|
let counts = vec-new(Count)
|
|
for i in range(length(ws))
|
|
match get(seen, str(ws[i]))
|
|
Some(k) -> counts[k].n += 1
|
|
None ->
|
|
put(seen, str(ws[i]), i32(length(counts)))
|
|
push(counts, Count{.word ws[i], .n 1})
|
|
counts
|
|
|
|
fn grade(distinct: i32, total: i32) -> Grade
|
|
let ratio = distinct * 10 / max(total, 1)
|
|
if ratio >= 7 then :rich else if ratio >= 4 then :fair else :poor
|
|
|
|
fn describe(g: Grade) -> str
|
|
match i32(g)
|
|
1 -> "repetitive"
|
|
2 -> "ordinary"
|
|
_ -> "varied"
|
|
|
|
fn main() -> i32
|
|
let text = "The cat saw the dog. The dog didn't see the cat, but the bird saw both!"
|
|
let ws = words(bytes-view(text))
|
|
let counts = tally(slice(ws))
|
|
let by-count = fn(a: Count, b: Count) -> bool =>
|
|
if a.n != b.n
|
|
return a.n > b.n
|
|
bytes<?(a.word, b.word)
|
|
sort-by(slice(counts), by-count)
|
|
for i in range(3)
|
|
let {.word .n} = counts[i]
|
|
println(str(word), n)
|
|
let top: [3 i32] = [counts[0].n counts[1].n counts[2].n]
|
|
[most & others] = top
|
|
println("most frequent seen", most, "times, then", others)
|
|
let total = i32(length(ws))
|
|
distinct = i32(length(counts))
|
|
g = grade(distinct, total)
|
|
println(distinct, "of", total, "distinct:", describe(g))
|
|
let longest = reduce(slice(ws), slice(ws[0], 0, 0), fn(a, b) =>
|
|
if length(b) > length(a) then b else a)
|
|
println("longest", str(longest))
|
|
let short = 0 < length(longest) < 5
|
|
match short
|
|
true -> println("short words only")
|
|
false -> println("some long words")
|
|
println("three distinct counts?", !=(counts[0].n, counts[1].n, counts[2].n))
|
|
0
|