Mercurial Hosting > luan
annotate src/luan/modules/lucene/Lucene.luan @ 1395:9dfff82dfc59
finish postgres work
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Mon, 09 Sep 2019 01:22:23 -0600 |
parents | 002152af497a |
children | 0dc9837c16be |
rev | line source |
---|---|
1353
8d95711f6615
replace java() with require "java"
Franklin Schmidt <fschmidt@gmail.com>
parents:
1348
diff
changeset
|
1 require "java" |
693
ca169567ce07
module URIs must now include ".luan"
Franklin Schmidt <fschmidt@gmail.com>
parents:
625
diff
changeset
|
2 local Luan = require "luan:Luan.luan" |
503 | 3 local error = Luan.error |
625
a3c1e11fb6aa
rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
4 local ipairs = Luan.ipairs or error() |
a3c1e11fb6aa
rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
5 local type = Luan.type or error() |
796 | 6 local set_metatable = Luan.set_metatable or error() |
1365
6617763dfd76
fix logging and lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1353
diff
changeset
|
7 local Boot = require "luan:Boot.luan" |
693
ca169567ce07
module URIs must now include ".luan"
Franklin Schmidt <fschmidt@gmail.com>
parents:
625
diff
changeset
|
8 local Html = require "luan:Html.luan" |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
693
diff
changeset
|
9 local Io = require "luan:Io.luan" |
746 | 10 local uri = Io.uri or error() |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
693
diff
changeset
|
11 local String = require "luan:String.luan" |
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
693
diff
changeset
|
12 local matches = String.matches or error() |
1368
5225cd6ed478
add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1367
diff
changeset
|
13 local Thread = require "luan:Thread.luan" |
5225cd6ed478
add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1367
diff
changeset
|
14 local schedule = Thread.schedule or error() |
1377 | 15 local run_in_lock = Thread.run_in_lock or error() |
16 local backup_read_lock = Thread.backup_read_lock or error() | |
1368
5225cd6ed478
add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1367
diff
changeset
|
17 local Time = require "luan:Time.luan" |
749
85f5444fb7d4
add rcp lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
748
diff
changeset
|
18 local Rpc = require "luan:Rpc.luan" |
321
7f7708e8fdd4
remove import statement
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
320
diff
changeset
|
19 local LuceneIndex = require "java:luan.modules.lucene.LuceneIndex" |
1344
dc2af9d5463b
move queryparser to lib
Franklin Schmidt <fschmidt@gmail.com>
parents:
1343
diff
changeset
|
20 local NumberFieldParser = require "java:luan.lib.queryparser.NumberFieldParser" |
dc2af9d5463b
move queryparser to lib
Franklin Schmidt <fschmidt@gmail.com>
parents:
1343
diff
changeset
|
21 local SaneQueryParser = require "java:luan.lib.queryparser.SaneQueryParser" |
1368
5225cd6ed478
add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1367
diff
changeset
|
22 local Logging = require "luan:logging/Logging.luan" |
5225cd6ed478
add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1367
diff
changeset
|
23 local logger = Logging.logger "Lucene" |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
542
diff
changeset
|
24 |
320
fed1893821bf
remove global namespace
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
312
diff
changeset
|
25 |
1088
bae2d0c2576c
change module naming convention
Franklin Schmidt <fschmidt@gmail.com>
parents:
796
diff
changeset
|
26 local Lucene = {} |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
27 |
1088
bae2d0c2576c
change module naming convention
Franklin Schmidt <fschmidt@gmail.com>
parents:
796
diff
changeset
|
28 Lucene.type = { |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
29 english = LuceneIndex.ENGLISH_FIELD_PARSER |
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
30 string = LuceneIndex.STRING_FIELD_PARSER |
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
31 integer = NumberFieldParser.INT |
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
32 long = NumberFieldParser.LONG |
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
33 double = NumberFieldParser.DOUBLE |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
542
diff
changeset
|
34 } |
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
542
diff
changeset
|
35 |
1088
bae2d0c2576c
change module naming convention
Franklin Schmidt <fschmidt@gmail.com>
parents:
796
diff
changeset
|
36 Lucene.literal = SaneQueryParser.literal |
547
0be287ab0309
add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents:
546
diff
changeset
|
37 |
1395
9dfff82dfc59
finish postgres work
Franklin Schmidt <fschmidt@gmail.com>
parents:
1392
diff
changeset
|
38 -- function Lucene.index(index_dir,default_type,default_fields) |
9dfff82dfc59
finish postgres work
Franklin Schmidt <fschmidt@gmail.com>
parents:
1392
diff
changeset
|
39 |
9dfff82dfc59
finish postgres work
Franklin Schmidt <fschmidt@gmail.com>
parents:
1392
diff
changeset
|
40 function Lucene.index(index_dir,options) |
1369
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1368
diff
changeset
|
41 type(index_dir)=="table" or error "index_dir must be table" |
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1368
diff
changeset
|
42 index_dir.to_uri_string and matches(index_dir.to_uri_string(),"^file:") or error "must be file" |
1395
9dfff82dfc59
finish postgres work
Franklin Schmidt <fschmidt@gmail.com>
parents:
1392
diff
changeset
|
43 options = options or {} |
9dfff82dfc59
finish postgres work
Franklin Schmidt <fschmidt@gmail.com>
parents:
1392
diff
changeset
|
44 options.postgres_spec==nil or type(options.postgres_spec)=="table" or error "postgres_spec must be table" |
9dfff82dfc59
finish postgres work
Franklin Schmidt <fschmidt@gmail.com>
parents:
1392
diff
changeset
|
45 options.completer==nil or type(options.completer)=="function" or error "completer must be table" |
303
fdb4bd391c28
add lucene close();
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
300
diff
changeset
|
46 local index = {} |
591
790d5de23042
add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents:
547
diff
changeset
|
47 index.dir = index_dir |
1395
9dfff82dfc59
finish postgres work
Franklin Schmidt <fschmidt@gmail.com>
parents:
1392
diff
changeset
|
48 local java_index, closer = LuceneIndex.getLuceneIndex(index_dir.java.file,options.default_type,options.default_fields,options.completer,options.postgres_spec) |
1129
3234a14bb1f8
minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents:
1088
diff
changeset
|
49 index.java = java_index |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
50 index.closer = closer or error() |
1395
9dfff82dfc59
finish postgres work
Franklin Schmidt <fschmidt@gmail.com>
parents:
1392
diff
changeset
|
51 index.completer = options.completer |
796 | 52 |
53 index.indexed_fields = {} | |
54 local mt = {} | |
55 set_metatable(index.indexed_fields,mt) | |
56 function mt.__index(_,key) | |
57 return java_index.getIndexedFieldParser(key) | |
58 end | |
59 function mt.__new_index(_,key,value) | |
60 return java_index.setIndexedFieldParser(key,value) | |
61 end | |
618
5e495e4e560b
add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents:
617
diff
changeset
|
62 |
303
fdb4bd391c28
add lucene close();
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
300
diff
changeset
|
63 index.to_string = java_index.to_string |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
693
diff
changeset
|
64 index.snapshot = java_index.snapshot |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
65 index.advanced_search = java_index.advanced_search |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
66 index.search_in_transaction = java_index.search_in_transaction |
303
fdb4bd391c28
add lucene close();
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
300
diff
changeset
|
67 index.delete_all = java_index.delete_all |
547
0be287ab0309
add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents:
546
diff
changeset
|
68 index.delete = java_index.delete |
0be287ab0309
add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents:
546
diff
changeset
|
69 index.save = java_index.save |
1379 | 70 index.run_in_transaction = java_index.run_in_transaction |
591
790d5de23042
add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents:
547
diff
changeset
|
71 index.ensure_open = java_index.ensure_open |
617 | 72 index.next_id = java_index.nextId |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
73 index.highlighter = java_index.highlighter |
1343
7d9a1f8894b0
lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1342
diff
changeset
|
74 index.indexed_only_fields = java_index.indexed_only_fields |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1330
diff
changeset
|
75 index.count_tokens = java_index.count_tokens |
1375 | 76 index.close = closer.close |
709 | 77 |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
78 index.has_postgres_backup = java_index.hasPostgresBackup() |
1391 | 79 function index.rebuild_postgres_backup() |
80 java_index.rebuild_postgres_backup(index.completer) | |
81 end | |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
82 index.restore_from_postgres = java_index.restore_from_postgres |
1391 | 83 function index.check() |
84 java_index.check(index.completer) | |
85 end | |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
86 |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
87 function index.search( query, from, to, options ) |
770 | 88 from or error "missing 'from' parameter" |
89 to or error "missing 'to' parameter" | |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
90 options = options or {} |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
91 local explain_fld = options.explain |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
92 local score_fld = options.score |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
93 local results = {} |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
94 local function fn(i,doc_fn,score) |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
95 if i >= from then |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
96 local doc |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
97 if explain_fld == nil then |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
98 doc = doc_fn() |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
99 else |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
100 local explanation |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
101 doc, explanation = doc_fn("explain") |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
102 doc[explain_fld] = explanation.toString() |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
103 end |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
104 if score_fld ~= nil then |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
105 doc[score_fld] = score |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
106 end |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
107 results[#results+1] = doc |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
108 end |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
109 end |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
110 local total_hits = index.advanced_search(query,fn,to,options.sort) |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
111 return results, total_hits |
257
c5c60eca33dd
allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
112 end |
c5c60eca33dd
allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
113 |
c5c60eca33dd
allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
114 function index.get_document(query) |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
115 local doc |
1348 | 116 local function fn(_,doc_fn,_) |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
117 doc = doc_fn() |
257
c5c60eca33dd
allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
118 end |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
119 local total_hits = index.advanced_search(query,fn,1) |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
120 total_hits <= 1 or error( "found " .. total_hits .. " documents" ) |
257
c5c60eca33dd
allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
121 return doc |
c5c60eca33dd
allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
122 end |
c5c60eca33dd
allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
123 |
c5c60eca33dd
allow Lucene search for 0 rows
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
124 function index.count(query) |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
125 return index.advanced_search(query) |
232
9ce18106f95a
more lucene work
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
230
diff
changeset
|
126 end |
9ce18106f95a
more lucene work
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
230
diff
changeset
|
127 |
625
a3c1e11fb6aa
rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
128 function index.html_highlighter(query,formatter,container_tags) |
a3c1e11fb6aa
rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
129 local highlighter = index.highlighter(query,formatter) |
a3c1e11fb6aa
rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
130 return function(html) |
a3c1e11fb6aa
rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
131 local list = Html.parse(html,container_tags) |
a3c1e11fb6aa
rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
132 local result = {} |
a3c1e11fb6aa
rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
133 for _, obj in ipairs(list) do |
a3c1e11fb6aa
rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
134 if type(obj) == "string" then |
a3c1e11fb6aa
rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
135 obj = highlighter(obj) |
a3c1e11fb6aa
rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
136 end |
a3c1e11fb6aa
rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
137 result[#result+1] = obj |
a3c1e11fb6aa
rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
138 end |
a3c1e11fb6aa
rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
139 return Html.to_string(result) |
a3c1e11fb6aa
rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
140 end |
a3c1e11fb6aa
rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
141 end |
a3c1e11fb6aa
rewrite much of Html to be more understandable;
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
142 |
1367
836e00bf7ce2
add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1365
diff
changeset
|
143 function index.backup_to(backup_dir) |
836e00bf7ce2
add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1365
diff
changeset
|
144 backup_dir.delete() |
836e00bf7ce2
add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1365
diff
changeset
|
145 backup_dir.mkdir() |
836e00bf7ce2
add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1365
diff
changeset
|
146 index.snapshot( function(dir_path,file_names) |
836e00bf7ce2
add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1365
diff
changeset
|
147 local lucene_dir = uri("file:"..dir_path) |
836e00bf7ce2
add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1365
diff
changeset
|
148 for _, file_name in ipairs(file_names) do |
836e00bf7ce2
add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1365
diff
changeset
|
149 local lucene_file = lucene_dir.child(file_name) |
836e00bf7ce2
add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1365
diff
changeset
|
150 local backup_file = backup_dir.child(file_name) |
836e00bf7ce2
add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1365
diff
changeset
|
151 backup_file.link_to(lucene_file) |
836e00bf7ce2
add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1365
diff
changeset
|
152 end |
836e00bf7ce2
add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1365
diff
changeset
|
153 end ) |
836e00bf7ce2
add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1365
diff
changeset
|
154 end |
836e00bf7ce2
add Lucene backup_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1365
diff
changeset
|
155 |
1368
5225cd6ed478
add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1367
diff
changeset
|
156 function index.schedule_backups_to(backup_dir,repeating_delay) |
1369
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1368
diff
changeset
|
157 local lucene_dir = index.dir |
1368
5225cd6ed478
add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1367
diff
changeset
|
158 local function backup() |
5225cd6ed478
add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1367
diff
changeset
|
159 if backup_dir.last_modified() < lucene_dir.last_modified() then |
1377 | 160 run_in_lock(backup_read_lock, index.backup_to,backup_dir) |
161 logger.info "backup" | |
1368
5225cd6ed478
add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1367
diff
changeset
|
162 end |
5225cd6ed478
add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1367
diff
changeset
|
163 end |
5225cd6ed478
add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1367
diff
changeset
|
164 schedule( backup, { |
5225cd6ed478
add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1367
diff
changeset
|
165 repeating_delay = repeating_delay or Time.period{minutes=10} |
5225cd6ed478
add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1367
diff
changeset
|
166 run_on_close = true |
5225cd6ed478
add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1367
diff
changeset
|
167 daemon = true |
5225cd6ed478
add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1367
diff
changeset
|
168 } ) |
5225cd6ed478
add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1367
diff
changeset
|
169 end |
5225cd6ed478
add Lucene schedule_backups_to
Franklin Schmidt <fschmidt@gmail.com>
parents:
1367
diff
changeset
|
170 |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
693
diff
changeset
|
171 function index.zip(zip_file) |
761
99356cfde2f0
remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents:
757
diff
changeset
|
172 index.snapshot( function(dir_path,file_names) |
99356cfde2f0
remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents:
757
diff
changeset
|
173 zip_file.delete() |
99356cfde2f0
remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents:
757
diff
changeset
|
174 local zip_path = zip_file.canonical().to_string() |
99356cfde2f0
remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents:
757
diff
changeset
|
175 local dir = uri("file:"..dir_path) |
99356cfde2f0
remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents:
757
diff
changeset
|
176 local dir_name = dir.name() |
99356cfde2f0
remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents:
757
diff
changeset
|
177 local options = {dir=dir.parent()} |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
693
diff
changeset
|
178 for _, file_name in ipairs(file_names) do |
761
99356cfde2f0
remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents:
757
diff
changeset
|
179 local cmd = "zip "..zip_path.." "..dir_name.."/"..file_name |
99356cfde2f0
remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents:
757
diff
changeset
|
180 Io.uri("os:"..cmd,options).read_text() |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
693
diff
changeset
|
181 end |
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
693
diff
changeset
|
182 end ) |
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
693
diff
changeset
|
183 end |
1365
6617763dfd76
fix logging and lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1353
diff
changeset
|
184 index.zip = Boot.no_security(index.zip) |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
693
diff
changeset
|
185 |
754 | 186 function index.restore(zip_file) |
756
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
187 java_index.run_in_lock( function() |
1369
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1368
diff
changeset
|
188 local lucene_dir = index.dir |
756
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
189 local before_restore = lucene_dir.parent().child("before_restore.zip") |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
190 index.zip(before_restore) |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
191 java_index.doClose() |
756
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
192 lucene_dir.delete() |
761
99356cfde2f0
remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents:
757
diff
changeset
|
193 Io.uri("os:unzip "..zip_file.canonical().to_string(),{dir=lucene_dir.parent()}).read_text() |
756
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
194 java_index.reopen() |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
195 end ) |
754 | 196 end |
1365
6617763dfd76
fix logging and lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1353
diff
changeset
|
197 index.restore = Boot.no_security(index.restore) |
754 | 198 |
199 local function multi_error() | |
200 error "multiple lucene instances" | |
201 end | |
202 | |
1133 | 203 if Rpc.functions.lucene_backup == nil then |
754 | 204 |
749
85f5444fb7d4
add rcp lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
748
diff
changeset
|
205 function Rpc.functions.lucene_backup(password) |
757 | 206 Io.password == password or error "wrong password" |
1369
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1368
diff
changeset
|
207 local zip_file = index.dir.parent().child("backup.zip") |
749
85f5444fb7d4
add rcp lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
748
diff
changeset
|
208 index.zip(zip_file) |
85f5444fb7d4
add rcp lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
748
diff
changeset
|
209 return zip_file |
85f5444fb7d4
add rcp lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
748
diff
changeset
|
210 end |
754 | 211 |
212 function Rpc.functions.lucene_restore(password,zip_file) | |
757 | 213 Io.password == password or error "wrong password" |
1369
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1368
diff
changeset
|
214 local backup_zip = index.dir.parent().child("backup.zip") |
761
99356cfde2f0
remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents:
757
diff
changeset
|
215 backup_zip.write(zip_file) |
99356cfde2f0
remove horrible java zip
Franklin Schmidt <fschmidt@gmail.com>
parents:
757
diff
changeset
|
216 index.restore(backup_zip) |
754 | 217 end |
218 | |
749
85f5444fb7d4
add rcp lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
748
diff
changeset
|
219 else |
754 | 220 Rpc.functions.lucene_backup = multi_error |
221 Rpc.functions.lucene_restore = multi_error | |
749
85f5444fb7d4
add rcp lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
748
diff
changeset
|
222 end |
85f5444fb7d4
add rcp lucene backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
748
diff
changeset
|
223 |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
224 return index |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
225 end |
503 | 226 |
1369
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1368
diff
changeset
|
227 function Lucene.recover(index_dir,restore_dir) |
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1368
diff
changeset
|
228 if not index_dir.exists() and restore_dir.exists() then |
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1368
diff
changeset
|
229 index_dir.mkdir() |
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1368
diff
changeset
|
230 for _, child in ipairs(restore_dir.children()) do |
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1368
diff
changeset
|
231 index_dir.child(child.name()).link_to(child) |
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1368
diff
changeset
|
232 end |
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1368
diff
changeset
|
233 logger.error "recovered" |
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1368
diff
changeset
|
234 end |
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1368
diff
changeset
|
235 end |
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1368
diff
changeset
|
236 |
1088
bae2d0c2576c
change module naming convention
Franklin Schmidt <fschmidt@gmail.com>
parents:
796
diff
changeset
|
237 return Lucene |