view src/luan/modules/lucene/Lucene.luan @ 1567:349eef23a13c

lucene named backup
author Franklin Schmidt <fschmidt@gmail.com>
date Sat, 14 Nov 2020 20:05:07 -0700
parents 44e444a8da64
children c922446f53aa
line wrap: on
line source
require "java"
local Luan = require "luan:Luan.luan"
local error = Luan.error
local ipairs = Luan.ipairs or error()
local pairs = Luan.pairs or error()
local type = Luan.type or error()
local set_metatable = Luan.set_metatable or error()
local Boot = require "luan:Boot.luan"
local Html = require "luan:Html.luan"
local Number = require "luan:Number.luan"
local integer = Number.integer or error()
local Time = require "luan:Time.luan"
local Io = require "luan:Io.luan"
local uri = Io.uri or error()
local String = require "luan:String.luan"
local matches = String.matches or error()
local Rpc = require "luan:Rpc.luan"
local LuceneIndex = require "java:luan.modules.lucene.LuceneIndex"
local NumberFieldParser = require "java:goodjava.lucene.queryparser.NumberFieldParser"
local GoodQueryParser = require "java:goodjava.lucene.queryparser.GoodQueryParser"
local Logging = require "luan:logging/Logging.luan"
local logger = Logging.logger "Lucene"


local Lucene = {}

local indexes = {}

function Rpc.functions.lucene_backup(password,name)
	Io.password == password or error "wrong password"
	local index = indexes[name] or error("index not found: "..name)
	local zip_file = index.dir.parent().child("backup.zip")
	index.zip(zip_file)
	return zip_file
end

function Rpc.functions.lucene_restore(password,name,zip_file)
	Io.password == password or error "wrong password"
	local index = indexes[name] or error "index not found"
	local backup_zip = index.dir.parent().child("backup.zip")
	backup_zip.write(zip_file)
	index.restore(backup_zip)
end


Lucene.type = {
	english = LuceneIndex.ENGLISH_FIELD_PARSER
	string = LuceneIndex.STRING_FIELD_PARSER
	lowercase = LuceneIndex.LOWERCASE_FIELD_PARSER
	integer = NumberFieldParser.INT
	long = NumberFieldParser.LONG
	double = NumberFieldParser.DOUBLE
}

Lucene.quote = GoodQueryParser.quote

local function get_file(f,name)
	type(f)=="table" or error(name.." must be table")
	f.to_uri_string and matches(f.to_uri_string(),"^file:") or error(name.." must be file")
	return f.java.file or error()
end

function Lucene.index(index_dir,options)
	local index = {}
	if options.name ~= nil then
		indexes[options.name] = index
		options.name = nil
	end
	index.dir = index_dir
	index_dir = get_file(index_dir)
	options = options or {}
	options.log_dir = options.log_dir and get_file(options.log_dir)
	options.log_time = options.log_time or Time.period{days=30}
	local java_index = LuceneIndex.getLuceneIndex(index_dir,options)
	index.java = java_index

	index.indexed_fields = {}
	local mt = {}
	set_metatable(index.indexed_fields,mt)
	function mt.__index(_,key)
		return java_index.getIndexedFieldParser(key)
	end
	function mt.__new_index(_,key,value)
		return java_index.setIndexedFieldParser(key,value)
	end

	index.to_string = java_index.to_string
	index.snapshot = java_index.snapshot
	index.advanced_search = java_index.advanced_search
	index.search_in_transaction = java_index.search_in_transaction
	index.delete_all = java_index.delete_all
	index.delete = java_index.delete
	index.reindex = java_index.reindex
	--index.save = java_index.save
	index.is_in_transaction = java_index.is_in_transaction
	index.run_in_transaction = java_index.run_in_transaction
	index.ensure_open = java_index.ensure_open
	index.highlighter = java_index.highlighter
	index.count_tokens = java_index.count_tokens
	index.tag = java_index.tag

	index.relog = java_index.relog
	index.restore_from_log = java_index.restore_from_log
	index.force_restore_from_log = java_index.force_restore_from_log

	index.has_postgres_backup = java_index.hasPostgresBackup()
	index.rebuild_postgres_backup = java_index.rebuild_postgres_backup
	index.restore_from_postgres = java_index.restore_from_postgres
	index.force_restore_from_postgres = java_index.force_restore_from_postgres
	index.check = java_index.check

	function index.not_in_transaction() end

	function index.check_in_transaction()
		if not index.is_in_transaction() then
			index.not_in_transaction()
		end
	end

	function index.save(doc)
		index.check_in_transaction()
		java_index.save(doc)
	end

	function index.search( query, from, to, options )
		from or error "missing 'from' parameter"
		to or error "missing 'to' parameter"
		options = options or {}
		local explain_fld = options.explain
		local score_fld = options.score
		local results = {}
		local function fn(i,doc_fn,score)
			if i >= from then
				local doc
				if explain_fld == nil then
					doc = doc_fn()
				else
					local explanation
					doc, explanation = doc_fn("explain")
					doc[explain_fld] = explanation.toString()
				end
				if score_fld ~= nil then
					doc[score_fld] = score
				end
				results[#results+1] = doc
			end
		end
		local total_hits = index.advanced_search(query,fn,to,options.sort)
		return results, total_hits
	end

	function index.get_document(query)
		local doc
		local function fn(_,doc_fn,_)
			doc = doc_fn()
		end
		local total_hits = index.advanced_search(query,fn,1)
		total_hits <= 1 or error("found "..total_hits.." documents for query: "..query)
		return doc
	end

	function index.count(query)
		return index.advanced_search(query)
	end

	function index.html_highlighter(query,formatter,container_tags)
		local highlighter = index.highlighter(query,formatter)
		return function(html)
			local list = Html.parse(html,container_tags)
			local result = {}
			for _, obj in ipairs(list) do
				if type(obj) == "string" then
					obj = highlighter(obj)
				end
				result[#result+1] = obj
			end
			return Html.to_string(result)
		end
	end

	function index.zip(zip_file)
		index.snapshot( function(dir_path,file_names)
			zip_file.delete()
			local zip_path = zip_file.canonical().to_string()
			local dir = uri("file:"..dir_path)
			local dir_name = dir.name()
			local options = {dir=dir.parent()}
			for _, file_name in ipairs(file_names) do
				local cmd = "zip "..zip_path.." "..dir_name.."/"..file_name
				uri("os:"..cmd,options).read_text()
			end
		end )
	end
	index.zip = Boot.no_security(index.zip)

	function index.restore(zip_file)
		java_index.run_in_lock( function()
			local lucene_dir = index.dir
			local before_restore = lucene_dir.parent().child("before_restore.zip")
			index.zip(before_restore)
			java_index.close()
			lucene_dir.delete()
			uri("os:unzip "..zip_file.canonical().to_string(),{dir=lucene_dir.parent()}).read_text()
			java_index.reopen()
		end )
	end
	index.restore = Boot.no_security(index.restore)

	function index.update(steps)
		local version = 0
		for i in pairs(steps) do
			if version < i then
				version = i
			end
		end
		local doc = index.get_document"type:version" or { type="version", version=integer(0) }
		while doc.version < version do
			doc.version = integer(doc.version + 1)
			logger.error("step "..doc.version)
			index.run_in_transaction( function()
				local step = steps[doc.version] or error("missing step "..doc.version)
				step()
				index.save(doc)
			end )
		end
	end

	return index
end

return Lucene