fix(pickers): improve CRLF line splitting support for windows (#3127)

* fix(help_tags): show help tags on windows (#3126)

On Windows, `builtin.help_tags` picker does not show any help tags.

To fix this, the following changes are needed:

1. `util.path_tail` checks unix separator `/` on Windows and leave the
original implementation intact on unix systems.

2. Line endings should be taken carefully on Windows. `vim.split` with
   only newline `\n` character as separator may result in unexpected
   crash when parsing large files. When splits on lines are needed, call
   it with `\r?\n`, or even set up a wrapper function in utils is more
   prefered.

Fixes #3126

* fix: handle cross platform line splits

lua/telescope/builtin/__internal.lua

@@ -531,7 +531,7 @@ internal.oldfiles = function(opts)
   local results = {}
 
   if opts.include_current_session then
-    for _, buffer in ipairs(vim.split(vim.fn.execute ":buffers! t", "\n")) do
+    for _, buffer in ipairs(utils.split_lines(vim.fn.execute ":buffers! t")) do
       local match = tonumber(string.match(buffer, "%s*(%d+)"))
       local open_by_lsp = string.match(buffer, "line 0$")
       if match and not open_by_lsp then
@@ -574,7 +574,7 @@ end
 
 internal.command_history = function(opts)
   local history_string = vim.fn.execute "history cmd"
-  local history_list = vim.split(history_string, "\n")
+  local history_list = utils.split_lines(history_string)
 
   local results = {}
   local filter_fn = opts.filter_fn
@@ -614,7 +614,7 @@ end
 
 internal.search_history = function(opts)
   local search_string = vim.fn.execute "history search"
-  local search_list = vim.split(search_string, "\n")
+  local search_list = utils.split_lines(search_string)
 
   local results = {}
   for i = #search_list, 3, -1 do
@@ -690,7 +690,7 @@ internal.help_tags = function(opts)
   opts.fallback = vim.F.if_nil(opts.fallback, true)
   opts.file_ignore_patterns = {}
 
-  local langs = vim.split(opts.lang, ",", true)
+  local langs = vim.split(opts.lang, ",", { trimempty = true })
   if opts.fallback and not vim.tbl_contains(langs, "en") then
     table.insert(langs, "en")
   end
@@ -729,11 +729,11 @@ internal.help_tags = function(opts)
   local delimiter = string.char(9)
   for _, lang in ipairs(langs) do
     for _, file in ipairs(tag_files[lang] or {}) do
-      local lines = vim.split(Path:new(file):read(), "\n", true)
+      local lines = utils.split_lines(Path:new(file):read(), { trimempty = true })
       for _, line in ipairs(lines) do
         -- TODO: also ignore tagComment starting with ';'
         if not line:match "^!_TAG_" then
-          local fields = vim.split(line, delimiter, true)
+          local fields = vim.split(line, delimiter, { trimempty = true })
           if #fields == 3 and not tags_map[fields[1]] then
             if fields[1] ~= "help-tags" or fields[2] ~= "tags" then
               table.insert(tags, {

lua/telescope/previewers/buffer_previewer.lua

@@ -1090,7 +1090,7 @@ previewers.highlights = defaulter(function(_)
 
     define_preview = function(self, entry)
       if not self.state.bufname then
-        local output = vim.split(vim.fn.execute "highlight", "\n")
+        local output = utils.split_lines(vim.fn.execute "highlight")
         local hl_groups = {}
         for _, v in ipairs(output) do
           if v ~= "" then

lua/telescope/previewers/utils.lua

@@ -5,12 +5,14 @@ local conf = require("telescope.config").values
 local Job = require "plenary.job"
 local Path = require "plenary.path"
 
+local telescope_utils = require "telescope.utils"
+
 local utils = {}
 
 local detect_from_shebang = function(p)
   local s = p:readbyterange(0, 256)
   if s then
-    local lines = vim.split(s, "\n")
+    local lines = telescope_utils.split_lines(s)
     return vim.filetype.match { contents = lines }
   end
 end
@@ -24,7 +26,7 @@ end
 local detect_from_modeline = function(p)
   local s = p:readbyterange(-256, 256)
   if s then
-    local lines = vim.split(s, "\n")
+    local lines = telescope_utils.split_lines(s)
     local idx = lines[#lines] ~= "" and #lines or #lines - 1
     if idx >= 1 then
       return parse_modeline(lines[idx])

lua/telescope/utils.lua

@@ -216,13 +216,25 @@ end)()
 utils.path_tail = (function()
   local os_sep = utils.get_separator()
 
-  return function(path)
+  if os_sep == "/" then
-    for i = #path, 1, -1 do
+    return function(path)
-      if path:sub(i, i) == os_sep then
+      for i = #path, 1, -1 do
-        return path:sub(i + 1, -1)
+        if path:sub(i, i) == os_sep then
+          return path:sub(i + 1, -1)
+        end
       end
+      return path
+    end
+  else
+    return function(path)
+      for i = #path, 1, -1 do
+        local c = path:sub(i, i)
+        if c == os_sep or c == "/" then
+          return path:sub(i + 1, -1)
+        end
+      end
+      return path
     end
-    return path
   end
 end)()
 
@@ -727,4 +739,16 @@ utils.reverse_table = function(input_table)
   return temp_table
 end
 
+utils.split_lines = (function()
+  if utils.iswin then
+    return function(s, opts)
+      return vim.split(s, "\r?\n", opts)
+    end
+  else
+    return function(s, opts)
+      return vim.split(s, "\n", opts)
+    end
+  end
+end)()
+
 return utils

lua/tests/automated/utils_spec.lua

@@ -307,3 +307,87 @@ describe("transform_path", function()
     end, new_relpath "doc/mydoc.md", new_relpath "d/mydoc.md")
   end)
 end)
+
+describe("path_tail", function()
+  local function assert_tails(paths)
+    for _, path in ipairs(paths) do
+      it("gets the tail of " .. path, function()
+        local tail = vim.fn.fnamemodify(path, ":p:t")
+        eq(tail, utils.path_tail(path))
+      end)
+    end
+  end
+
+  if jit and jit.os:lower() == "windows" then
+    describe("handles windows paths", function()
+      local paths = {
+        [[C:\Users\username\AppData\Local\nvim-data\log]],
+        [[D:\Projects\project_folder\source_code.py]],
+        [[E:\Music\song.mp3]],
+        [[/home/usuario/documents/archivo.txt]],
+        [[/var/www/html/index.html]],
+        [[/mnt/backup/backup_file.tar.gz]],
+      }
+
+      assert_tails(paths)
+    end)
+  elseif jit and jit.os:lower() == "linux" then
+    describe("handles linux paths", function()
+      local paths = {
+        [[/home/usuario/documents/archivo.txt]],
+        [[/var/www/html/index.html]],
+        [[/mnt/backup/backup_file.tar.gz]],
+      }
+
+      assert_tails(paths)
+    end)
+  elseif jit and jit.os:lower() == "osx" then
+    describe("handles macos paths", function()
+      local paths = {
+        [[/Users/Usuario/Documents/archivo.txt]],
+        [[/Applications/App.app/Contents/MacOS/app_executable]],
+        [[/Volumes/ExternalDrive/Data/file.xlsx]],
+      }
+
+      assert_tails(paths)
+    end)
+  end
+end)
+
+describe("split_lines", function()
+  local expect = {
+    "",
+    "",
+    "line3 of the file",
+    "",
+    "line5 of the file",
+    "",
+    "",
+    "line8 of the file, last line of file",
+    "",
+  }
+
+  local function get_fake_file(line_ending)
+    return table.concat(expect, line_ending)
+  end
+
+  local newline_file = get_fake_file "\n"
+  local carriage_newline_file = get_fake_file "\r\n"
+
+  if utils.iswin then
+    describe("handles files on Windows", function()
+      it("reads file with newline only", function()
+        assert.are.same(expect, utils.split_lines(newline_file))
+      end)
+      it("reads file with carriage return and newline", function()
+        assert.are.same(expect, utils.split_lines(carriage_newline_file))
+      end)
+    end)
+  else
+    describe("handles files on non Windows environment", function()
+      it("reads file with newline only", function()
+        assert.are.same(expect, utils.split_lines(newline_file))
+      end)
+    end)
+  end
+end)