Mercurial Hosting > luan

annotate src/luan/modules/lucene/Lucene.luan @ 1392:002152af497a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
hosted postgres
author Franklin Schmidt <fschmidt@gmail.com>
date Fri, 06 Sep 2019 00:19:47 -0600
parents 94f48cc76de8
children 9dfff82dfc59
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1353
8d95711f6615 replace java() with require "java"
Franklin Schmidt <fschmidt@gmail.com>
parents: 1348
diff changeset
1 require "java"
693
ca169567ce07 module URIs must now include ".luan"
Franklin Schmidt <fschmidt@gmail.com>
parents: 625
diff changeset
2 local Luan = require "luan:Luan.luan"
503
92c3d22745b8 make _ENV optional
Franklin Schmidt <fschmidt@gmail.com>
parents: 435
diff changeset
3 local error = Luan.error
625
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
4 local ipairs = Luan.ipairs or error()
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
5 local type = Luan.type or error()
796
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
6 local set_metatable = Luan.set_metatable or error()
1365
6617763dfd76 fix logging and lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1353
diff changeset
7 local Boot = require "luan:Boot.luan"
693
ca169567ce07 module URIs must now include ".luan"
Franklin Schmidt <fschmidt@gmail.com>
parents: 625
diff changeset
8 local Html = require "luan:Html.luan"
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
9 local Io = require "luan:Io.luan"
746
293c397e8dee improve zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
10 local uri = Io.uri or error()
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
11 local String = require "luan:String.luan"
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
12 local matches = String.matches or error()
1368
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
13 local Thread = require "luan:Thread.luan"
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
14 local schedule = Thread.schedule or error()
1377
Franklin Schmidt <fschmidt@gmail.com>
parents: 1375
diff changeset
15 local run_in_lock = Thread.run_in_lock or error()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1375
diff changeset
16 local backup_read_lock = Thread.backup_read_lock or error()
1368
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
17 local Time = require "luan:Time.luan"
749
85f5444fb7d4 add rcp lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 748
diff changeset
18 local Rpc = require "luan:Rpc.luan"
321
7f7708e8fdd4 remove import statement
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 320
diff changeset
19 local LuceneIndex = require "java:luan.modules.lucene.LuceneIndex"
1344
dc2af9d5463b move queryparser to lib
Franklin Schmidt <fschmidt@gmail.com>
parents: 1343
diff changeset
20 local NumberFieldParser = require "java:luan.lib.queryparser.NumberFieldParser"
dc2af9d5463b move queryparser to lib
Franklin Schmidt <fschmidt@gmail.com>
parents: 1343
diff changeset
21 local SaneQueryParser = require "java:luan.lib.queryparser.SaneQueryParser"
1368
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
22 local Logging = require "luan:logging/Logging.luan"
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
23 local logger = Logging.logger "Lucene"
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 542
diff changeset
24
320
fed1893821bf remove global namespace
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 312
diff changeset
25
1088
bae2d0c2576c change module naming convention
Franklin Schmidt <fschmidt@gmail.com>
parents: 796
diff changeset
26 local Lucene = {}
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
27
1088
bae2d0c2576c change module naming convention
Franklin Schmidt <fschmidt@gmail.com>
parents: 796
diff changeset
28 Lucene.type = {
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
29 english = LuceneIndex.ENGLISH_FIELD_PARSER
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
30 string = LuceneIndex.STRING_FIELD_PARSER
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
31 integer = NumberFieldParser.INT
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
32 long = NumberFieldParser.LONG
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
33 double = NumberFieldParser.DOUBLE
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 542
diff changeset
34 }
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 542
diff changeset
35
1088
bae2d0c2576c change module naming convention
Franklin Schmidt <fschmidt@gmail.com>
parents: 796
diff changeset
36 Lucene.literal = SaneQueryParser.literal
547
0be287ab0309 add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents: 546
diff changeset
37
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
38 function Lucene.index(index_dir,default_type,default_fields,postgres_spec)
1369
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1368
diff changeset
39 type(index_dir)=="table" or error "index_dir must be table"
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1368
diff changeset
40 index_dir.to_uri_string and matches(index_dir.to_uri_string(),"^file:") or error "must be file"
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
41 postgres_spec==nil or type(postgres_spec)=="table" or error "postgres_spec must be table"
303
fdb4bd391c28 add lucene close();
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 300
diff changeset
42 local index = {}
591
790d5de23042 add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents: 547
diff changeset
43 index.dir = index_dir
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
44 local java_index, closer = LuceneIndex.getLuceneIndex(index_dir.java.file,default_type,default_fields,postgres_spec)
1129
3234a14bb1f8 minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents: 1088
diff changeset
45 index.java = java_index
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
46 index.closer = closer or error()
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
47 index.completer = postgres_spec and postgres_spec.completer
796
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
48
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
49 index.indexed_fields = {}
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
50 local mt = {}
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
51 set_metatable(index.indexed_fields,mt)
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
52 function mt.__index(_,key)
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
53 return java_index.getIndexedFieldParser(key)
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
54 end
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
55 function mt.__new_index(_,key,value)
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
56 return java_index.setIndexedFieldParser(key,value)
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
57 end
618
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
58
303
fdb4bd391c28 add lucene close();
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 300
diff changeset
59 index.to_string = java_index.to_string
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
60 index.snapshot = java_index.snapshot
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
61 index.advanced_search = java_index.advanced_search
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
62 index.search_in_transaction = java_index.search_in_transaction
303
fdb4bd391c28 add lucene close();
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 300
diff changeset
63 index.delete_all = java_index.delete_all
547
0be287ab0309 add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents: 546
diff changeset
64 index.delete = java_index.delete
0be287ab0309 add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents: 546
diff changeset
65 index.save = java_index.save
1379
87a3738d7cc5 run_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1377
diff changeset
66 index.run_in_transaction = java_index.run_in_transaction
591
790d5de23042 add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents: 547
diff changeset
67 index.ensure_open = java_index.ensure_open
617
e54c1646eed0 add Lucene.next_id()
Franklin Schmidt <fschmidt@gmail.com>
parents: 599
diff changeset
68 index.next_id = java_index.nextId
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
69 index.highlighter = java_index.highlighter
1343
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
70 index.indexed_only_fields = java_index.indexed_only_fields
1341
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1330
diff changeset
71 index.count_tokens = java_index.count_tokens
1375
5c3702f60200 cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1369
diff changeset
72 index.close = closer.close
709
96a280ca32a2 add Lucene.instances
Franklin Schmidt <fschmidt@gmail.com>
parents: 707
diff changeset
73
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
74 index.has_postgres_backup = java_index.hasPostgresBackup()
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1388
diff changeset
75 function index.rebuild_postgres_backup()
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1388
diff changeset
76 java_index.rebuild_postgres_backup(index.completer)
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1388
diff changeset
77 end
1388
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
78 index.restore_from_postgres = java_index.restore_from_postgres
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1388
diff changeset
79 function index.check()
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1388
diff changeset
80 java_index.check(index.completer)
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1388
diff changeset
81 end
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
82
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
83 function index.search( query, from, to, options )
770
Franklin Schmidt <fschmidt@gmail.com>
parents: 761
diff changeset
84 from or error "missing 'from' parameter"
Franklin Schmidt <fschmidt@gmail.com>
parents: 761
diff changeset
85 to or error "missing 'to' parameter"
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
86 options = options or {}
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
87 local explain_fld = options.explain
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
88 local score_fld = options.score
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
89 local results = {}
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
90 local function fn(i,doc_fn,score)
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
91 if i >= from then
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
92 local doc
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
93 if explain_fld == nil then
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
94 doc = doc_fn()
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
95 else
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
96 local explanation
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
97 doc, explanation = doc_fn("explain")
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
98 doc[explain_fld] = explanation.toString()
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
99 end
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
100 if score_fld ~= nil then
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
101 doc[score_fld] = score
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
102 end
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
103 results[#results+1] = doc
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
104 end
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
105 end
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
106 local total_hits = index.advanced_search(query,fn,to,options.sort)
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
107 return results, total_hits
257
c5c60eca33dd allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
108 end
c5c60eca33dd allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
109
c5c60eca33dd allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
110 function index.get_document(query)
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
111 local doc
1348
af478a1bd23d bug fix
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
112 local function fn(_,doc_fn,_)
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
113 doc = doc_fn()
257
c5c60eca33dd allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
114 end
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
115 local total_hits = index.advanced_search(query,fn,1)
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
116 total_hits <= 1 or error( "found " .. total_hits .. " documents" )
257
c5c60eca33dd allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
117 return doc
c5c60eca33dd allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
118 end
c5c60eca33dd allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
119
c5c60eca33dd allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
120 function index.count(query)
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
121 return index.advanced_search(query)
232
9ce18106f95a more lucene work
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 230
diff changeset
122 end
9ce18106f95a more lucene work
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 230
diff changeset
123
625
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
124 function index.html_highlighter(query,formatter,container_tags)
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
125 local highlighter = index.highlighter(query,formatter)
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
126 return function(html)
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
127 local list = Html.parse(html,container_tags)
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
128 local result = {}
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
129 for _, obj in ipairs(list) do
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
130 if type(obj) == "string" then
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
131 obj = highlighter(obj)
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
132 end
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
133 result[#result+1] = obj
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
134 end
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
135 return Html.to_string(result)
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
136 end
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
137 end
a3c1e11fb6aa rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
138
1367
836e00bf7ce2 add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1365
diff changeset
139 function index.backup_to(backup_dir)
836e00bf7ce2 add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1365
diff changeset
140 backup_dir.delete()
836e00bf7ce2 add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1365
diff changeset
141 backup_dir.mkdir()
836e00bf7ce2 add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1365
diff changeset
142 index.snapshot( function(dir_path,file_names)
836e00bf7ce2 add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1365
diff changeset
143 local lucene_dir = uri("file:"..dir_path)
836e00bf7ce2 add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1365
diff changeset
144 for _, file_name in ipairs(file_names) do
836e00bf7ce2 add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1365
diff changeset
145 local lucene_file = lucene_dir.child(file_name)
836e00bf7ce2 add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1365
diff changeset
146 local backup_file = backup_dir.child(file_name)
836e00bf7ce2 add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1365
diff changeset
147 backup_file.link_to(lucene_file)
836e00bf7ce2 add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1365
diff changeset
148 end
836e00bf7ce2 add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1365
diff changeset
149 end )
836e00bf7ce2 add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1365
diff changeset
150 end
836e00bf7ce2 add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1365
diff changeset
151
1368
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
152 function index.schedule_backups_to(backup_dir,repeating_delay)
1369
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1368
diff changeset
153 local lucene_dir = index.dir
1368
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
154 local function backup()
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
155 if backup_dir.last_modified() < lucene_dir.last_modified() then
1377
Franklin Schmidt <fschmidt@gmail.com>
parents: 1375
diff changeset
156 run_in_lock(backup_read_lock, index.backup_to,backup_dir)
Franklin Schmidt <fschmidt@gmail.com>
parents: 1375
diff changeset
157 logger.info "backup"
1368
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
158 end
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
159 end
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
160 schedule( backup, {
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
161 repeating_delay = repeating_delay or Time.period{minutes=10}
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
162 run_on_close = true
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
163 daemon = true
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
164 } )
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
165 end
5225cd6ed478 add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents: 1367
diff changeset
166
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
167 function index.zip(zip_file)
761
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
168 index.snapshot( function(dir_path,file_names)
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
169 zip_file.delete()
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
170 local zip_path = zip_file.canonical().to_string()
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
171 local dir = uri("file:"..dir_path)
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
172 local dir_name = dir.name()
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
173 local options = {dir=dir.parent()}
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
174 for _, file_name in ipairs(file_names) do
761
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
175 local cmd = "zip "..zip_path.." "..dir_name.."/"..file_name
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
176 Io.uri("os:"..cmd,options).read_text()
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
177 end
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
178 end )
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
179 end
1365
6617763dfd76 fix logging and lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1353
diff changeset
180 index.zip = Boot.no_security(index.zip)
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 693
diff changeset
181
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 753
diff changeset
182 function index.restore(zip_file)
756
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
183 java_index.run_in_lock( function()
1369
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1368
diff changeset
184 local lucene_dir = index.dir
756
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
185 local before_restore = lucene_dir.parent().child("before_restore.zip")
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
186 index.zip(before_restore)
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
187 java_index.doClose()
756
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
188 lucene_dir.delete()
761
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
189 Io.uri("os:unzip "..zip_file.canonical().to_string(),{dir=lucene_dir.parent()}).read_text()
756
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
190 java_index.reopen()
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
191 end )
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 753
diff changeset
192 end
1365
6617763dfd76 fix logging and lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1353
diff changeset
193 index.restore = Boot.no_security(index.restore)
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 753
diff changeset
194
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 753
diff changeset
195 local function multi_error()
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 753
diff changeset
196 error "multiple lucene instances"
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 753
diff changeset
197 end
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 753
diff changeset
198
1133
ba4daf107e07 fix security bug
Franklin Schmidt <fschmidt@gmail.com>
parents: 1129
diff changeset
199 if Rpc.functions.lucene_backup == nil then
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 753
diff changeset
200
749
85f5444fb7d4 add rcp lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 748
diff changeset
201 function Rpc.functions.lucene_backup(password)
757
Franklin Schmidt <fschmidt@gmail.com>
parents: 756
diff changeset
202 Io.password == password or error "wrong password"
1369
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1368
diff changeset
203 local zip_file = index.dir.parent().child("backup.zip")
749
85f5444fb7d4 add rcp lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 748
diff changeset
204 index.zip(zip_file)
85f5444fb7d4 add rcp lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 748
diff changeset
205 return zip_file
85f5444fb7d4 add rcp lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 748
diff changeset
206 end
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 753
diff changeset
207
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 753
diff changeset
208 function Rpc.functions.lucene_restore(password,zip_file)
757
Franklin Schmidt <fschmidt@gmail.com>
parents: 756
diff changeset
209 Io.password == password or error "wrong password"
1369
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1368
diff changeset
210 local backup_zip = index.dir.parent().child("backup.zip")
761
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
211 backup_zip.write(zip_file)
99356cfde2f0 remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 757
diff changeset
212 index.restore(backup_zip)
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 753
diff changeset
213 end
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 753
diff changeset
214
749
85f5444fb7d4 add rcp lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 748
diff changeset
215 else
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 753
diff changeset
216 Rpc.functions.lucene_backup = multi_error
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 753
diff changeset
217 Rpc.functions.lucene_restore = multi_error
749
85f5444fb7d4 add rcp lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 748
diff changeset
218 end
85f5444fb7d4 add rcp lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 748
diff changeset
219
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
220 return index
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
221 end
503
92c3d22745b8 make _ENV optional
Franklin Schmidt <fschmidt@gmail.com>
parents: 435
diff changeset
222
1369
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1368
diff changeset
223 function Lucene.recover(index_dir,restore_dir)
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1368
diff changeset
224 if not index_dir.exists() and restore_dir.exists() then
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1368
diff changeset
225 index_dir.mkdir()
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1368
diff changeset
226 for _, child in ipairs(restore_dir.children()) do
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1368
diff changeset
227 index_dir.child(child.name()).link_to(child)
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1368
diff changeset
228 end
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1368
diff changeset
229 logger.error "recovered"
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1368
diff changeset
230 end
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1368
diff changeset
231 end
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1368
diff changeset
232
1088
bae2d0c2576c change module naming convention
Franklin Schmidt <fschmidt@gmail.com>
parents: 796
diff changeset
233 return Lucene