#!/usr/bin/env lua
--
-- Index each paragraph of a text file as a Xapian document.
--
-- Copyright (C) 2011 Xiaona Han
--
-- This program is free software; you can redistribute it and/or
-- modify it under the terms of the GNU General Public License as
-- published by the Free Software Foundation; either version 2 of the
-- License, or (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-- GNU General Public License for more details.
--
-- You should have received a copy of the GNU General Public License
-- along with this program; if not, see
-- <https://www.gnu.org/licenses/>.

require("xapian")

-- Remove leading and trailing whitespace from a string
function trim(s)
  return (s:gsub("^%s*(.-)%s*$", "%1"))
end

-- Require one command line argument to mean the database directory
if #arg ~= 1 then
  io.stderr:write("Usage: " .. arg[0] .. " PATH_TO_DATABASE\n")
  os.exit()
end

-- Open the database for update, creating a new database if necessary.
database = xapian.WritableDatabase(arg[1], xapian.DB_CREATE_OR_OPEN)

indexer = xapian.TermGenerator()
stemmer = xapian.Stem("english")
indexer:set_stemmer(stemmer)

-- Read document data from standard input line.
-- An empty line means the end of a doc and the start of another doc
local para = ''
for line in io.lines() do
  if line == nil then
    break
  end

  line = trim(line)
  if line == '' then
    if para ~= '' then
      -- We've reached the end of a paragraph, so index it.
      doc = xapian.Document()
      doc:set_data(para)

      indexer:set_document(doc)
      indexer:index_text(para)

      -- Add the document to the database.
      database:add_document(doc)
      para = ''
    end
  else
    if para ~= '' then
      para = para .. ' '
    end
    para = para .. line
  end
end
