From fa0382d93e73b66e7ec769cec27b9fbb21020641 Mon Sep 17 00:00:00 2001
From: TJ DeVries <devries.timothyj@gmail.com>
Date: Fri, 31 Jul 2020 00:05:22 -0400
Subject: [PATCH] Streamed some refactoring. More work to do

---
 lua/telescope/finders.lua    |   3 +-
 lua/telescope/mappings.lua   |  61 ++------
 lua/telescope/pickers.lua    | 191 ++++++++++++++++++-------
 lua/telescope/previewers.lua |   5 +-
 lua/telescope/sorters.lua    |   2 +
 lua/tests/telescope_spec.lua |  15 ++
 scratch/bk_tree.lua          | 270 +++++++++++++++++++++++++++++++++++
 scratch/file_finder.lua      |  13 +-
 scratch/fuzzy_tester.lua     |   8 ++
 scratch/ngrams.lua           |  87 +++++++++++
 scratch/picker_locations.lua |   4 +
 scratch/slow_proc.sh         |   8 +-
 12 files changed, 555 insertions(+), 112 deletions(-)
 create mode 100644 lua/tests/telescope_spec.lua
 create mode 100644 scratch/bk_tree.lua
 create mode 100644 scratch/fuzzy_tester.lua
 create mode 100644 scratch/ngrams.lua
 create mode 100644 scratch/picker_locations.lua

diff --git a/lua/telescope/finders.lua b/lua/telescope/finders.lua
index 0b7e185..518dfc1 100644
--- a/lua/telescope/finders.lua
+++ b/lua/telescope/finders.lua
@@ -10,7 +10,8 @@ Finder.__call = function(t, ... ) return t:_find(...) end
 
 --- Create a new finder command
 ---
---@param fn_command function The function to call
+---@param opts table Keys:
+--     fn_command function The function to call
 function Finder:new(opts)
   opts = opts or {}
 
diff --git a/lua/telescope/mappings.lua b/lua/telescope/mappings.lua
index 3c6de18..a3704a6 100644
--- a/lua/telescope/mappings.lua
+++ b/lua/telescope/mappings.lua
@@ -1,12 +1,10 @@
 -- TODO: Customize keymap
 local a = vim.api
-local ns_telescope_selection = a.nvim_create_namespace('telescope_selection')
 
 local state = require('telescope.state')
 
 local mappings = {}
 local keymap = {}
-local telescope_selections = {}
 
 mappings.set_keymap = function(prompt_bufnr, results_bufnr)
   local function default_mapper(map_key, table_key)
@@ -31,31 +29,8 @@ mappings.set_keymap = function(prompt_bufnr, results_bufnr)
   default_mapper('<CR>', 'enter')
 end
 
-local function update_current_selection(prompt_bufnr, results_bufnr, row)
-  local status = state.get_status(prompt_bufnr)
-
-  a.nvim_buf_clear_namespace(results_bufnr, ns_telescope_selection, 0, -1)
-  a.nvim_buf_add_highlight(
-    results_bufnr,
-    ns_telescope_selection,
-    'Error',
-    row,
-    0,
-    -1
-  )
-
-  telescope_selections[prompt_bufnr] = row
-
-  if status.previewer then
-    vim.g.got_here = true
-
-    status.previewer:preview(
-      status.preview_win,
-      status.preview_bufnr,
-      status.results_bufnr,
-      row
-    )
-  end
+local function update_current_selection(prompt_bufnr, change)
+  state.get_status(prompt_bufnr).picker:move_selection(change)
 end
 
 
@@ -68,34 +43,16 @@ end
 -- TODO: Refactor this to use shared code.
 -- TODO: Move from top to bottom, etc.
 -- TODO: It seems like doing this brings us back to the beginning of the prompt, which is not great.
-keymap["control-n"] = function(prompt_bufnr, results_bufnr)
-  if telescope_selections[prompt_bufnr] == nil then
-    telescope_selections[prompt_bufnr] = 0
-  end
-
-  local row = telescope_selections[prompt_bufnr] + 1
-  update_current_selection(prompt_bufnr, results_bufnr, row)
+keymap["control-n"] = function(prompt_bufnr, _)
+  update_current_selection(prompt_bufnr, 1)
 end
 
-keymap["control-p"] = function(prompt_bufnr, results_bufnr)
-  if telescope_selections[prompt_bufnr] == nil then
-    telescope_selections[prompt_bufnr] = 0
-  end
-
-  local row = telescope_selections[prompt_bufnr] - 1
-  update_current_selection(prompt_bufnr, results_bufnr, row)
+keymap["control-p"] = function(prompt_bufnr, _)
+  update_current_selection(prompt_bufnr, -1)
 end
 
 keymap["enter"] = function(prompt_bufnr, results_bufnr)
-  local extmark = a.nvim_buf_get_extmarks(
-    results_bufnr,
-    ns_telescope_selection,
-    0,
-    -1,
-    {}
-  )
-
-  local row = extmark[1][2]
+  local row = state.get_status(prompt_bufnr).picker:get_selection()
   if row == nil then
     print("Could not do anything...")
     return
@@ -117,7 +74,9 @@ keymap["enter"] = function(prompt_bufnr, results_bufnr)
     local bufnr = vim.fn.bufnr(filename, true)
     a.nvim_set_current_buf(bufnr)
     a.nvim_buf_set_option(bufnr, 'buflisted', true)
-    a.nvim_win_set_cursor(0, {row, col})
+    if row and col then
+      a.nvim_win_set_cursor(0, {row, col})
+    end
   end
 end
 
diff --git a/lua/telescope/pickers.lua b/lua/telescope/pickers.lua
index f9f64df..5b4485d 100644
--- a/lua/telescope/pickers.lua
+++ b/lua/telescope/pickers.lua
@@ -10,8 +10,21 @@ local pickers = {}
 local Picker = {}
 Picker.__index = Picker
 
+
+local Sorter = require('telescope.sorters').Sorter
+local Previewer = require('telescope.previewers').Previewer
+
+assert(Sorter)
+assert(Previewer)
+
+---@class PickOpts
+---@field filter Sorter
+---@field maps table
+---@field unseen string
+
+--- Create new picker
+--- @param opts PickOpts
 function Picker:new(opts)
-  opts = opts or {}
   return setmetatable({
     filter = opts.filter,
     previewer = opts.previewer,
@@ -19,6 +32,56 @@ function Picker:new(opts)
   }, Picker)
 end
 
+function Picker._get_window_options(max_columns, max_lines, prompt_title)
+  local preview = {
+    border = {},
+    enter = false,
+    highlight = false
+  }
+  local results = {
+    border = {},
+    enter = false,
+  }
+  local prompt = {
+    title = prompt_title,
+    border = {},
+    enter = true
+  }
+
+  local width_padding = 10
+  if max_columns < 200 then
+    preview.width = 80
+  else
+    preview.width = 120
+  end
+
+  local other_width = max_columns - preview.width - (2 * width_padding)
+  results.width = other_width
+  prompt.width = other_width
+
+  results.height = 25
+  results.minheight = results.height
+  prompt.height = 1
+  prompt.minheight = prompt.height
+
+  preview.height = results.height + prompt.height + 2
+  preview.minheight = preview.height
+
+  results.col = width_padding
+  prompt.col = width_padding
+  preview.col = results.col + results.width + 2
+
+  local height_padding = math.floor(0.95 * max_lines)
+  results.line = max_lines - height_padding
+  prompt.line = results.line + results.height + 2
+  preview.line = results.line
+
+  return {
+    preview = preview,
+    results = results,
+    prompt = prompt,
+  }
+end
 
 function Picker:find(opts)
   opts = opts or {}
@@ -29,64 +92,41 @@ function Picker:find(opts)
   local sorter = opts.sorter
 
   local prompt_string = opts.prompt
+
   -- Create three windows:
   -- 1. Prompt window
   -- 2. Options window
   -- 3. Preview window
-
-  local width = 100
-  local col = 10
-  local prompt_line = 50
-
-  local result_height = 25
-  local prompt_height = 1
+  local popup_opts = Picker._get_window_options(vim.o.columns, vim.o.lines, prompt_string)
 
   -- TODO: Add back the borders after fixing some stuff in popup.nvim
-  local results_win, results_opts = popup.create('', {
-    height = result_height,
-    minheight = result_height,
-    width = width,
-    line = prompt_line - 2 - result_height,
-    col = col,
-    border = {},
-    enter = false,
-  })
+  local results_win, results_opts = popup.create('', popup_opts.results)
   local results_bufnr = a.nvim_win_get_buf(results_win)
 
-  local preview_win, preview_opts = popup.create('', {
-    height = result_height + prompt_height + 2,
-    minheight = result_height + prompt_height + 2,
-    width = 100,
-    line = prompt_line - 2 - result_height,
-    col = col + width + 2,
-    border = {},
-    enter = false,
-    highlight = false,
-  })
+  local preview_win, preview_opts = popup.create('', popup_opts.preview)
   local preview_bufnr = a.nvim_win_get_buf(preview_win)
 
   -- TODO: For some reason, highlighting is kind of weird on these windows.
   --        It may actually be my colorscheme tho...
   a.nvim_win_set_option(preview_win, 'winhl', 'Normal:Normal')
+  a.nvim_win_set_option(preview_win, 'winblend', 1)
 
   -- TODO: We need to center this and make it prettier...
-  local prompt_win, prompt_opts = popup.create('', {
-    height = prompt_height,
-    width = width,
-    line = prompt_line,
-    col = col,
-    border = {},
-    title = prompt_string
-  })
+  local prompt_win, prompt_opts = popup.create('', popup_opts.prompt)
   local prompt_bufnr = a.nvim_win_get_buf(prompt_win)
 
   -- a.nvim_buf_set_option(prompt_bufnr, 'buftype', 'prompt')
   -- vim.fn.prompt_setprompt(prompt_bufnr, prompt_string)
 
+  -- First thing we want to do is set all the lines to blank.
+  local max_results = popup_opts.results.height
+  local initial_lines = {}
+  for _ = 1, max_results do table.insert(initial_lines, "") end
+  vim.api.nvim_buf_set_lines(results_bufnr, 0, max_results, false, initial_lines)
+
   local on_lines = function(_, _, _, first_line, last_line)
     local prompt = vim.api.nvim_buf_get_lines(prompt_bufnr, first_line, last_line, false)[1]
 
-    vim.api.nvim_buf_set_lines(results_bufnr, 0, -1, false, {})
 
     -- Create a closure that has all the data we need
     -- We pass a function called "newResult" to get_results
@@ -94,13 +134,21 @@ function Picker:find(opts)
     --    picker then (if available) calls sorter
     --    and then appropriately places new result in the buffer.
 
-    local line_scores = {}
+
+    -- Sorted table by scores.
+    --  Lowest score gets lowest index.
+    self.line_scores = {}
 
     -- TODO: We need to fix the sorting
     -- TODO: We should provide a simple fuzzy matcher in Lua for people
     -- TODO: We should get all the stuff on the bottom line directly, not floating around
     -- TODO: We need to handle huge lists in a good way, cause currently we'll just put too much stuff in the buffer
     -- TODO: Stop having things crash if we have an error.
+
+    local replace_line = function(row, line)
+      vim.api.nvim_buf_set_lines(results_bufnr, row, row + 1, false, {line})
+    end
+
     finder(prompt, function(line)
       if sorter then
         local sort_score = sorter:score(prompt, line)
@@ -110,24 +158,30 @@ function Picker:find(opts)
 
         -- { 7, 3, 1, 1 }
         -- 2
-        for row, row_score in utils.reversed_ipairs(line_scores) do
+        for row, row_score in utils.reversed_ipairs(self.line_scores) do
           if row_score > sort_score then
             -- Insert line at row
-            vim.api.nvim_buf_set_lines(results_bufnr, row, row, false, {
-              string.format("%s // %s %s", line, sort_score, row)
-            })
+            replace_line(max_results - row, line)
 
             -- Insert current score in the table
-            table.insert(line_scores, row + 1, sort_score)
+            table.insert(self.line_scores, row + 1, sort_score)
 
             -- All done :)
             return
           end
+
+          -- Don't keep inserting stuff
+          if row > max_results then
+            return
+          end
         end
 
         -- Worst score so far, so add to end
-        vim.api.nvim_buf_set_lines(results_bufnr, -1, -1, false, {line})
-        table.insert(line_scores, sort_score)
+
+        -- example: 5 max results, 8
+        local worst_line = max_results - #self.line_scores
+        replace_line(worst_line, line)
+        table.insert(self.line_scores, sort_score)
       else
         -- Just always append to the end of the buffer if this is all you got.
         vim.api.nvim_buf_set_lines(results_bufnr, -1, -1, false, {line})
@@ -159,6 +213,8 @@ function Picker:find(opts)
   vim.cmd(    on_buf_leave)
   vim.cmd([[augroup END]])
 
+  self.prompt_bufnr = prompt_bufnr
+
   state.set_status(prompt_bufnr, {
     prompt_bufnr = prompt_bufnr,
     prompt_win = prompt_win,
@@ -196,12 +252,12 @@ function Picker:close_windows(status)
   local preview_border_win = status.preview_border_win
 
   local function del_win(name, win_id, force)
-    local file = io.open("/home/tj/test.txt", "a")
-    file:write(string.format("Closing.... %s %s\n", name, win_id))
+    -- local file = io.open("/home/tj/test.txt", "a")
+    -- file:write(string.format("Closing.... %s %s\n", name, win_id))
     local ok = pcall(vim.api.nvim_win_close, win_id, force)
-    file:write(string.format("OK: %s\n", ok))
-    file:write("...Done\n\n")
-    file:close()
+    -- file:write(string.format("OK: %s\n", ok))
+    -- file:write("...Done\n\n")
+    -- file:close()
   end
 
   del_win("prompt_win", prompt_win, true)
@@ -224,7 +280,44 @@ function Picker:close_windows(status)
   state.clear_status(status.prompt_bufnr)
 end
 
+local ns_telescope_selection = a.nvim_create_namespace('telescope_selection')
 
+function Picker:get_selection()
+  return self.selection or #(self.line_scores or {})
+end
+
+function Picker:move_selection(change)
+  self:set_selection(self:get_selection() + change)
+end
+
+function Picker:set_selection(row)
+  local status = state.get_status(self.prompt_bufnr)
+
+  a.nvim_buf_clear_namespace(status.results_bufnr, ns_telescope_selection, 0, -1)
+  a.nvim_buf_add_highlight(
+    status.results_bufnr,
+    ns_telescope_selection,
+    'Error',
+    row,
+    0,
+    -1
+  )
+
+  -- TODO: Don't let you go over / under the buffer limits
+  -- TODO: Make sure you start exactly at the bottom selected
+
+  -- TODO: Get row & text in the same obj
+  self.selection = row
+
+  if self.previewer then
+    self.previewer:preview(
+      status.preview_win,
+      status.preview_bufnr,
+      status.results_bufnr,
+      row
+    )
+  end
+end
 
 pickers.new = function(...)
   return Picker:new(...)
diff --git a/lua/telescope/previewers.lua b/lua/telescope/previewers.lua
index ccbd7db..5b686b7 100644
--- a/lua/telescope/previewers.lua
+++ b/lua/telescope/previewers.lua
@@ -38,10 +38,13 @@ previewers.vim_buffer = previewers.new {
     vim.api.nvim_win_set_buf(preview_win, bufnr)
     vim.api.nvim_win_set_option(preview_win, 'wrap', false)
     vim.api.nvim_win_set_option(preview_win, 'winhl', 'Normal:Normal')
-    vim.api.nvim_win_set_option(preview_win, 'winblend', 20)
+    -- vim.api.nvim_win_set_option(preview_win, 'winblend', 20)
     vim.api.nvim_win_set_option(preview_win, 'signcolumn', 'no')
     vim.api.nvim_win_set_option(preview_win, 'foldlevel', 100)
   end,
 }
 
+
+previewers.Previewer = Previewer
+
 return previewers
diff --git a/lua/telescope/sorters.lua b/lua/telescope/sorters.lua
index d5fb6f9..a648007 100644
--- a/lua/telescope/sorters.lua
+++ b/lua/telescope/sorters.lua
@@ -29,4 +29,6 @@ function sorters.new(...)
   return Sorter:new(...)
 end
 
+sorters.Sorter = Sorter
+
 return sorters
diff --git a/lua/tests/telescope_spec.lua b/lua/tests/telescope_spec.lua
new file mode 100644
index 0000000..b6e01a6
--- /dev/null
+++ b/lua/tests/telescope_spec.lua
@@ -0,0 +1,15 @@
+-- require('plenary.test_harness'):setup_busted()
+
+--[[
+require("plenary.test_harness"):test_directory("busted", "./tests/telescope_spec.lua")
+--]]
+
+if false then
+  describe('Picker', function()
+    describe('window_dimensions', function()
+      it('', function()
+        assert(true)
+      end)
+    end)
+  end)
+end
diff --git a/scratch/bk_tree.lua b/scratch/bk_tree.lua
new file mode 100644
index 0000000..3957224
--- /dev/null
+++ b/scratch/bk_tree.lua
@@ -0,0 +1,270 @@
+
+---------------------------
+-- bk-tree datastructure
+--
+-- http://en.wikipedia.org/wiki/BK-tree
+-- @module bk-tree
+-- @author Robin Hübner
+-- robinhubner@gmail.com
+-- @release version 1.0.2
+-- @license MIT
+
+local bk_tree = {}
+
+
+local function lazy_copy(t1)
+
+	local cp = {}
+
+	for k, v in pairs(t1) do
+		cp[k] = v
+	end
+
+	return cp
+
+end
+
+local function min(a, b, c)
+
+	local min_val = a
+
+	if b < min_val then min_val = b end
+	if c < min_val then min_val = c end
+
+	return min_val
+
+end
+
+----------------------------------
+--- Levenshtein distance function.
+-- @tparam string s1
+-- @tparam string s2
+-- @treturn number the levenshtein distance
+-- @within Metrics
+function bk_tree.levenshtein_dist(s1, s2)
+	
+	if s1 == s2 then return 0 end
+	if s1:len() == 0 then return s2:len() end
+	if s2:len() == 0 then return s1:len() end
+	if s1:len() < s2:len() then s1, s2 = s2, s1 end
+
+	t = {}
+	for i=1, #s1+1 do
+		t[i] = {i-1}
+	end
+
+	for i=1, #s2+1 do
+		t[1][i] = i-1
+	end
+
+	local cost
+	for i=2, #s1+1 do
+	
+		for j=2, #s2+1 do
+			cost = (s1:sub(i-1,i-1) == s2:sub(j-1,j-1) and 0) or 1
+			t[i][j] = min(
+				t[i-1][j] + 1,
+				t[i][j-1] + 1,
+				t[i-1][j-1] + cost)
+		end
+
+	end
+
+	return t[#s1+1][#s2+1]
+	
+end
+
+function bk_tree.hook(param)
+
+	local name, callee = debug.getlocal(2, 1)
+	local f = debug.getinfo(2, "f").func
+	local p = debug.getinfo(3, "f").func
+	--[[ previous function in the callstack, if called from the same place,
+			don't add to the insert/remove counters. ]]--
+
+	if f == bk_tree.insert and p ~= bk_tree.insert then
+		callee.stats.nodes = callee.stats.nodes + 1
+	elseif f == bk_tree.remove and p ~= bk_tree.remove then
+		callee.stats.nodes = callee.stats.nodes - 1
+	elseif f == bk_tree.query and p == bk_tree.query then
+		callee.stats.queries = callee.stats.queries + 1
+	end
+
+end
+
+--- Hooks debugging into tree execution.
+-- Keeps track of number of nodes created, queries made,
+-- note that this must be run directly after tree is created
+-- in order to get correct information.
+-- @within Debug
+--- @usage
+-- local bktree = require "bk-tree"
+-- local tree = bktree:new("word")
+-- tree:debug()
+-- tree:insert("perceive")
+-- tree:insert("beautiful")
+-- tree:insert("definitely")
+-- local result = tree:query("definately", 3)
+-- tree:print_stats()
+--
+-- -- output
+-- Nodes: 4
+-- Queries: 3
+-- Nodes Queried: 75%
+function bk_tree:debug()
+
+	local nc = 0
+	if self.root then nc = 1 end
+	self.stats = { nodes = nc, queries = 0 }
+	debug.sethook(self.hook, "c")
+
+end
+
+--- Print execution stats.
+-- Prints nodes queried and total nodes, as well as a fraction of 
+-- nodes visited to satisfy the query, resets the counter of nodes queried when called.
+-- @within Debug
+-- @see debug
+function bk_tree:print_stats()
+
+	print("\nNodes: " .. self.stats.nodes)
+	print("Queries: " .. self.stats.queries)
+	print("Nodes Queried: " .. self.stats.queries/self.stats.nodes*100 .. "%\n")
+	self.stats.queries = 0
+
+end
+
+--- Fetch execution stats.
+-- Returns a copy of the execution stats that @{print_stats} would print, requires debug to have been enabled
+-- to not just return defaults. Useful if you want to profile things.
+-- @within Debug
+-- @return {key = value,...}
+function bk_tree:get_stats()
+
+	return lazy_copy(self.stats)
+
+end
+
+---------------------------
+--- Creates a new bk-tree.
+-- @constructor
+-- @string[opt] root_word the root of the new tree
+-- @tparam[opt=levenshtein_dist] function dist_func the distance function used
+-- @see levenshtein_dist
+-- @return the new bk-tree instance
+--- @usage
+-- local bktree = require "bk-tree"
+-- local tree = bktree:new("word")
+function bk_tree:new(root_word, dist_func)
+
+	local n_obj = {}
+	if root_word then n_obj.root = { str = root_word, children = {} } end
+	n_obj.dist_func = dist_func or self.levenshtein_dist
+
+	setmetatable(n_obj, self)
+	self.__index = self
+
+	return n_obj
+
+end
+
+--------------------------------
+--- Inserts word into the tree.
+-- @string word
+-- @treturn bool true if inserted, false if word already exists in tree
+--- @usage
+-- local bktree = require "bk-tree"
+-- local tree = bktree:new("root")
+-- local success = tree:insert("other_word")
+function bk_tree:insert(word, node)
+
+	node = node or self.root
+
+	if not node then
+		self.root = { str = word, children = {} }
+		return true
+	end	
+
+	local dist = self.dist_func(word, node.str)
+	if dist == 0 then return false end
+
+	local some_node = node.children[dist]
+
+	if not some_node then
+		node.children[dist] = { str = word, children = {} }
+		return true
+	end	
+
+	return self:insert(word, some_node)
+
+end
+
+--------------------------------
+--- Query the tree for matches.
+-- @string word
+-- @tparam number n max edit distance to use when querying
+-- @treturn {{str=string,distance=number},....} table of tables with matching words, empty table if no matches
+--- @usage
+-- local bktree = require "bk-tree"
+-- local tree = bktree:new("word")
+-- tree:insert("hello")
+-- tree:insert("goodbye")
+-- tree:insert("woop")
+-- local result = tree:query("woop", 1)
+function bk_tree:query(word, n, node, matches)
+
+	node = node or self.root
+	matches = matches or {}
+
+	if not node then return matches end
+
+	local dist = self.dist_func(word, node.str)
+	if dist <= n then matches[#matches+1] = {str = node.str, distance = dist} end
+	
+	for k, child in pairs(node.children) do
+		if child ~= nil then
+			if k >= dist-n and k <= dist+n then
+				self:query(word, n, child, matches)
+			end
+		end
+	end
+
+	return matches
+
+end
+
+---------------------------------------------------------
+--- Queries the the tree for a match, sorts the results.
+-- Calls @{query} and returns the results sorted.
+-- @string word
+-- @tparam number n max edit distance to use when querying
+-- @treturn {{str=string,distance=number},....} table of tables with matching words sorted by distance, empty table if no matches
+--- @usage
+-- local bktree = require "bk-tree"
+-- local tree = bktree:new("word")
+-- tree:insert("woop")
+-- tree:insert("worp")
+-- tree:insert("warp")
+-- local result = tree:query_sorted("woop", 3)
+function bk_tree:query_sorted(word, n)
+
+	local result = self:query(word, n)
+
+	table.sort(result, function(a,b) return a.distance < b.distance end)
+
+	return result
+
+end
+
+
+local tree = bk_tree:new("word")
+tree:insert("hello")
+tree:insert("welp")
+tree:insert("function")
+tree:insert("this long line what")
+tree:insert("what this long line what")
+print(vim.inspect(tree))
+print(vim.inspect(tree:query_sorted("what", 3)))
+
+
+return bk_tree
diff --git a/scratch/file_finder.lua b/scratch/file_finder.lua
index 8b5bb7b..ae84e2f 100644
--- a/scratch/file_finder.lua
+++ b/scratch/file_finder.lua
@@ -1,3 +1,4 @@
+package.loaded['telescope.pickers'] = nil
 local telescope = require('telescope')
 
 -- Goals:
@@ -61,15 +62,15 @@ local file_sorter = telescope.sorters.new {
     if prompt == '' then return 0 end
     if not line then return -1 end
 
-    local dist = string_distance(prompt, line)
-    -- if dist > (0.75 * #line) and #prompt > 3 then
-    --   return -1
-    -- end
-
-    return dist
+    return tonumber(vim.fn.systemlist(string.format(
+      "echo '%s' | ~/tmp/fuzzy_test/target/debug/fuzzy_test '%s'",
+      line,
+      prompt
+    ))[1])
   end
 }
 
+
 local file_previewer = telescope.previewers.vim_buffer
 
 local file_picker = telescope.pickers.new {
diff --git a/scratch/fuzzy_tester.lua b/scratch/fuzzy_tester.lua
new file mode 100644
index 0000000..0971381
--- /dev/null
+++ b/scratch/fuzzy_tester.lua
@@ -0,0 +1,8 @@
+
+line = "hello"
+prompt = "h"
+print(vim.inspect(vim.fn.systemlist(string.format(
+  "echo '%s' | ~/tmp/fuzzy_test/target/debug/fuzzy_test '%s'",
+  line,
+  prompt
+))))
diff --git a/scratch/ngrams.lua b/scratch/ngrams.lua
new file mode 100644
index 0000000..8b763a8
--- /dev/null
+++ b/scratch/ngrams.lua
@@ -0,0 +1,87 @@
+
+local function ngrams(counts, doc)
+  local DEPTH = 5
+  local docLen = #doc
+  local min, concat = math.min, table.concat
+  for i = 1, docLen - 1 do
+    for j = i, min(i + DEPTH - 1, docLen) do
+      if not doc[j] then break end
+      local k = concat(doc, " ", i, j)
+      counts[k] = (counts[k] or 0) + 1
+    end
+  end
+end
+
+
+local bz = io.popen('bzcat /home/tj/Downloads/pages.xml.bz2')
+local title, content = "", ""
+local inText = false
+
+local numDocs = 0
+local globalCounts = {}
+
+local function set(t) 
+  local s = {}
+  for _, v in pairs(t) do s[v] = true end
+  return s
+end
+
+local bad = set({
+                  'after', 'also', 'article', 'date', 'defaultsort', 'external', 'first', 'from',
+                  'have', 'html', 'http', 'image', 'infobox', 'links', 'name', 'other', 'preserve',
+                  'references', 'reflist', 'space', 'that', 'this', 'title', 'which', 'with',
+                  'quot', 'ref', 'name', 'http', 'amp', 'ndash', 'www', 'cite', 'nbsp',
+                  'style', 'text', 'align', 'center', 'background'
+                })
+
+local function isnumber(w)
+  s, e = w:find("[0-9]+")
+  return s
+end
+
+for line in bz:lines() do
+  local _, _, mTitle = line:find("<title>(.*)</title>")
+  local _, _, bText = line:find("<text[^>]*>([^<]*)")
+  local eText, _ = line:find("</text>")
+
+  if mTitle then
+    title = mTitle
+  elseif bText then
+    content = bText
+    inText = true
+  elseif inText then
+    content = content .. line
+  end
+  
+  if eText then
+    words = {}
+    for v in content:gmatch("%w+") do
+      v = v:lower()
+      if #v >= 3 and #v < 12 and not bad[v] and not isnumber(v) then
+        table.insert(words, v)
+      else
+        table.insert(words, nil)
+      end
+    end
+
+    ngrams(globalCounts, words)
+    inText = false
+
+    numDocs = numDocs + 1
+    if numDocs % 10 == 0 then
+      io.write(string.format("Working... %d documents processed.\r", numDocs))
+      io.flush()
+    end
+    
+    if numDocs == 500 then
+      local f = io.open('/tmp/freqs.lua.txt', 'w')
+      for k, v in pairs(globalCounts) do
+        f:write(k, '\t', v, '\n')
+      end
+      f:close()
+
+      globalCounts = {}
+      os.exit(0)
+    end
+  end  
+end
diff --git a/scratch/picker_locations.lua b/scratch/picker_locations.lua
new file mode 100644
index 0000000..82fe7f7
--- /dev/null
+++ b/scratch/picker_locations.lua
@@ -0,0 +1,4 @@
+package.loaded['telescope'] = nil
+package.loaded['telescope.init'] = nil
+package.loaded['telescope.picker'] = nil
+package.loaded['telescope.finder'] = nil
diff --git a/scratch/slow_proc.sh b/scratch/slow_proc.sh
index bbcf8bf..580e617 100755
--- a/scratch/slow_proc.sh
+++ b/scratch/slow_proc.sh
@@ -1,10 +1,10 @@
 
 echo "hello"
 sleep 1
-echo "cool"
+echo "help"
 sleep 1
-echo "world"
+echo "hi"
 sleep 1
-echo "x"
+echo "husband"
 sleep 1
-echo "y"
+echo "helper"