Mercurial Hosting > luan
annotate src/luan/modules/lucene/LuceneIndex.java @ 1342:60599adc27b8
add lucene search options
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Wed, 20 Feb 2019 12:09:51 -0700 |
parents | a015a0b5c388 |
children | 7d9a1f8894b0 |
rev | line source |
---|---|
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
1 package luan.modules.lucene; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
2 |
527 | 3 import java.io.Closeable; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
4 import java.io.File; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
5 import java.io.FileOutputStream; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
6 import java.io.FileInputStream; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
7 import java.io.IOException; |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
8 import java.util.Iterator; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
9 import java.util.Map; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
10 import java.util.List; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
11 import java.util.ArrayList; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
12 import java.util.Set; |
618
5e495e4e560b
add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents:
617
diff
changeset
|
13 import java.util.HashSet; |
5e495e4e560b
add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents:
617
diff
changeset
|
14 import java.util.Collections; |
704 | 15 import java.util.concurrent.ConcurrentMap; |
16 import java.util.concurrent.ConcurrentHashMap; | |
17 import java.util.concurrent.atomic.AtomicInteger; | |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
18 import java.util.concurrent.locks.Lock; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
19 import java.util.concurrent.locks.ReentrantLock; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
20 import java.util.zip.ZipOutputStream; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
21 import java.util.zip.ZipEntry; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
22 import org.apache.lucene.analysis.Analyzer; |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
23 import org.apache.lucene.analysis.TokenStream; |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
24 import org.apache.lucene.analysis.core.KeywordAnalyzer; |
233
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
25 import org.apache.lucene.document.Document; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
26 import org.apache.lucene.document.Field; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
27 import org.apache.lucene.document.StoredField; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
28 import org.apache.lucene.document.StringField; |
599
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
29 import org.apache.lucene.document.TextField; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
30 import org.apache.lucene.document.IntField; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
31 import org.apache.lucene.document.LongField; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
32 import org.apache.lucene.document.DoubleField; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
33 import org.apache.lucene.index.IndexableField; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
34 import org.apache.lucene.index.IndexWriter; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
35 import org.apache.lucene.index.IndexWriterConfig; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
36 import org.apache.lucene.index.DirectoryReader; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
37 import org.apache.lucene.index.Term; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
38 import org.apache.lucene.index.SnapshotDeletionPolicy; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
39 import org.apache.lucene.index.IndexCommit; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
40 import org.apache.lucene.index.AtomicReaderContext; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
41 import org.apache.lucene.store.Directory; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
42 import org.apache.lucene.store.FSDirectory; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
43 import org.apache.lucene.util.Version; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
44 import org.apache.lucene.util.BytesRef; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
45 import org.apache.lucene.util.NumericUtils; |
312
d34be4588556
add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
303
diff
changeset
|
46 import org.apache.lucene.search.Query; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
47 import org.apache.lucene.search.TermQuery; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
48 import org.apache.lucene.search.TopDocs; |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
49 import org.apache.lucene.search.Sort; |
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
50 import org.apache.lucene.search.SortField; |
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
51 import org.apache.lucene.search.IndexSearcher; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
52 import org.apache.lucene.search.TotalHitCountCollector; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
53 import org.apache.lucene.search.ScoreDoc; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
54 import org.apache.lucene.search.Collector; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
55 import org.apache.lucene.search.Scorer; |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
56 import org.apache.lucene.search.Explanation; |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
57 import org.apache.lucene.search.highlight.Formatter; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
58 import org.apache.lucene.search.highlight.Highlighter; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
59 import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; |
1248
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
60 import org.apache.lucene.search.highlight.Fragmenter; |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
61 import org.apache.lucene.search.highlight.NullFragmenter; |
1248
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
62 import org.apache.lucene.search.highlight.SimpleSpanFragmenter; |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
63 import org.apache.lucene.search.highlight.QueryScorer; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
64 import org.apache.lucene.search.highlight.TokenGroup; |
730
01e68da6983b
add sane-lucene-queryparser source to luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
707
diff
changeset
|
65 import luan.modules.lucene.queryparser.SaneQueryParser; |
01e68da6983b
add sane-lucene-queryparser source to luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
707
diff
changeset
|
66 import luan.modules.lucene.queryparser.FieldParser; |
01e68da6983b
add sane-lucene-queryparser source to luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
707
diff
changeset
|
67 import luan.modules.lucene.queryparser.MultiFieldParser; |
01e68da6983b
add sane-lucene-queryparser source to luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
707
diff
changeset
|
68 import luan.modules.lucene.queryparser.StringFieldParser; |
01e68da6983b
add sane-lucene-queryparser source to luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
707
diff
changeset
|
69 import luan.modules.lucene.queryparser.NumberFieldParser; |
1111
88b5b81cad4a
move Parser to luan.lib.parser
Franklin Schmidt <fschmidt@gmail.com>
parents:
1110
diff
changeset
|
70 import luan.lib.parser.ParseException; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
71 import luan.modules.Utils; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
72 import luan.Luan; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
73 import luan.LuanTable; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
74 import luan.LuanFunction; |
1271
48f302bdc187
fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents:
1267
diff
changeset
|
75 import luan.LuanCloner; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
76 import luan.LuanException; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
77 import luan.LuanRuntimeException; |
1337 | 78 import luan.lib.logging.Logger; |
79 import luan.lib.logging.LoggerFactory; | |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
80 |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
81 |
527 | 82 public final class LuceneIndex implements Closeable { |
521
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
83 private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class); |
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
84 |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
85 private static final String FLD_NEXT_ID = "nextId"; |
599
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
86 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer()); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
87 |
754 | 88 private static final Version version = Version.LUCENE_4_9; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
89 private final ReentrantLock writeLock = new ReentrantLock(); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
90 private final File indexDir; |
754 | 91 private SnapshotDeletionPolicy snapshotDeletionPolicy; |
92 private IndexWriter writer; | |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
93 private DirectoryReader reader; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
94 private IndexSearcher searcher; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
95 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>(); |
754 | 96 private boolean isClosed = true; |
599
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
97 private final MultiFieldParser mfp; |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
98 private final Analyzer analyzer; |
1227 | 99 private final Exception created = new Exception("created"); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
100 |
704 | 101 private static ConcurrentMap<File,AtomicInteger> globalWriteCounters = new ConcurrentHashMap<File,AtomicInteger>(); |
754 | 102 private File fileDir; |
704 | 103 private int writeCount; |
104 | |
1271
48f302bdc187
fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents:
1267
diff
changeset
|
105 private final ConcurrentMap<String,Map<String,LuanFunction>> indexedOnlyFields = new ConcurrentHashMap<String,Map<String,LuanFunction>>(); |
1333
25746915a241
merge Luan and LuanState
Franklin Schmidt <fschmidt@gmail.com>
parents:
1271
diff
changeset
|
106 private final Luan luanMine = new Luan(); |
1271
48f302bdc187
fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents:
1267
diff
changeset
|
107 private final LuanCloner cloner = new LuanCloner(LuanCloner.Type.COMPLETE); |
48f302bdc187
fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents:
1267
diff
changeset
|
108 |
1333
25746915a241
merge Luan and LuanState
Franklin Schmidt <fschmidt@gmail.com>
parents:
1271
diff
changeset
|
109 public LuceneIndex(Luan luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields) |
1267 | 110 throws LuanException, IOException |
111 { | |
599
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
112 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields); |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
113 mfp.fields.put( "type", STRING_FIELD_PARSER ); |
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
114 mfp.fields.put( "id", NumberFieldParser.LONG ); |
233
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
115 File indexDir = new File(indexDirStr); |
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
116 this.indexDir = indexDir; |
599
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
117 Analyzer analyzer = STRING_FIELD_PARSER.analyzer; |
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
118 if( defaultFieldParser instanceof StringFieldParser ) { |
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
119 StringFieldParser sfp = (StringFieldParser)defaultFieldParser; |
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
120 analyzer = sfp.analyzer; |
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
121 } |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
122 this.analyzer = analyzer; |
754 | 123 luan.onClose(this); |
124 reopen(); | |
125 } | |
126 | |
127 public void reopen() throws LuanException, IOException { | |
128 if( !isClosed ) throw new RuntimeException(); | |
129 isClosed = false; | |
233
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
130 IndexWriterConfig conf = new IndexWriterConfig(version,analyzer); |
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
131 snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()); |
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
132 conf.setIndexDeletionPolicy(snapshotDeletionPolicy); |
754 | 133 FSDirectory dir = FSDirectory.open(indexDir); |
134 fileDir = dir.getDirectory(); | |
135 globalWriteCounters.putIfAbsent(fileDir,new AtomicInteger()); | |
233
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
136 writer = new IndexWriter(dir,conf); |
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
137 writer.commit(); // commit index creation |
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
138 reader = DirectoryReader.open(dir); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
139 searcher = new IndexSearcher(reader); |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
140 initId(); |
233
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
141 } |
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
142 |
704 | 143 private int globalWriteCount() { |
144 return globalWriteCounters.get(fileDir).get(); | |
145 } | |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
146 |
704 | 147 private void wrote() { |
148 globalWriteCounters.get(fileDir).incrementAndGet(); | |
149 } | |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
150 |
252
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
151 public void delete_all() throws IOException { |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
152 boolean commit = !writeLock.isHeldByCurrentThread(); |
252
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
153 writeLock.lock(); |
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
154 try { |
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
155 writer.deleteAll(); |
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
156 id = idLim = 0; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
157 if(commit) writer.commit(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
158 } finally { |
704 | 159 wrote(); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
160 writeLock.unlock(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
161 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
162 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
163 |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
164 private static Term term(String key,long value) { |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
165 BytesRef br = new BytesRef(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
166 NumericUtils.longToPrefixCoded(value,0,br); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
167 return new Term(key,br); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
168 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
169 |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
170 public void delete(Luan luan,String queryStr) |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
171 throws LuanException, IOException, ParseException |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
172 { |
622
1a53333eb4d5
remove Lucene all_search_terms_must_match() since now query suffix "~a" handles this
Franklin Schmidt <fschmidt@gmail.com>
parents:
621
diff
changeset
|
173 Query query = SaneQueryParser.parseQuery(mfp,queryStr); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
174 |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
175 boolean commit = !writeLock.isHeldByCurrentThread(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
176 writeLock.lock(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
177 try { |
547
0be287ab0309
add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents:
546
diff
changeset
|
178 writer.deleteDocuments(query); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
179 if(commit) writer.commit(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
180 } finally { |
704 | 181 wrote(); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
182 writeLock.unlock(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
183 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
184 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
185 |
1271
48f302bdc187
fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents:
1267
diff
changeset
|
186 public void indexed_only_field(String type,String field,LuanFunction fn) { |
48f302bdc187
fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents:
1267
diff
changeset
|
187 fn = (LuanFunction)cloner.get(fn); |
48f302bdc187
fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents:
1267
diff
changeset
|
188 indexedOnlyFields.putIfAbsent(type,new ConcurrentHashMap<String,LuanFunction>()); |
48f302bdc187
fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents:
1267
diff
changeset
|
189 Map<String,LuanFunction> map = indexedOnlyFields.get(type); |
48f302bdc187
fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents:
1267
diff
changeset
|
190 map.put(field,fn); |
48f302bdc187
fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents:
1267
diff
changeset
|
191 } |
48f302bdc187
fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents:
1267
diff
changeset
|
192 |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
193 public void save(Luan luan,LuanTable doc,LuanTable boosts) |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
194 throws LuanException, IOException |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
195 { |
618
5e495e4e560b
add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents:
617
diff
changeset
|
196 Set indexedOnlySet = new HashSet(); |
1267 | 197 Object typeObj = doc.get("type"); |
618
5e495e4e560b
add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents:
617
diff
changeset
|
198 if( typeObj==null ) |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
199 throw new LuanException("missing 'type' field"); |
618
5e495e4e560b
add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents:
617
diff
changeset
|
200 if( !(typeObj instanceof String) ) |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
201 throw new LuanException("type must be string"); |
618
5e495e4e560b
add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents:
617
diff
changeset
|
202 String type = (String)typeObj; |
1271
48f302bdc187
fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents:
1267
diff
changeset
|
203 Map<String,LuanFunction> map = indexedOnlyFields.get(type); |
48f302bdc187
fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents:
1267
diff
changeset
|
204 if( map != null ) { |
48f302bdc187
fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents:
1267
diff
changeset
|
205 for( Map.Entry<String,LuanFunction> entry : map.entrySet() ) { |
48f302bdc187
fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents:
1267
diff
changeset
|
206 String name = entry.getKey(); |
48f302bdc187
fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents:
1267
diff
changeset
|
207 LuanFunction fn = entry.getValue(); |
1335 | 208 Object value = Luan.first(fn.call(doc)); |
1267 | 209 doc.put( name, value ); |
618
5e495e4e560b
add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents:
617
diff
changeset
|
210 indexedOnlySet.add(name); |
5e495e4e560b
add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents:
617
diff
changeset
|
211 } |
5e495e4e560b
add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents:
617
diff
changeset
|
212 } |
1267 | 213 Object obj = doc.get("id"); |
601 | 214 Long id; |
215 try { | |
216 id = (Long)obj; | |
217 } catch(ClassCastException e) { | |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
218 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName()); |
601 | 219 } |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
220 |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
221 boolean commit = !writeLock.isHeldByCurrentThread(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
222 writeLock.lock(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
223 try { |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
224 if( id == null ) { |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
225 id = nextId(luan); |
1267 | 226 doc.put("id",id); |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
227 writer.addDocument(toLucene(doc,indexedOnlySet,boosts)); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
228 } else { |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
229 writer.updateDocument( term("id",id), toLucene(doc,indexedOnlySet,boosts) ); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
230 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
231 if(commit) writer.commit(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
232 } finally { |
704 | 233 wrote(); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
234 writeLock.unlock(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
235 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
236 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
237 |
1335 | 238 public void update_in_transaction(LuanFunction fn) throws IOException, LuanException { |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
239 boolean commit = !writeLock.isHeldByCurrentThread(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
240 writeLock.lock(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
241 try { |
1335 | 242 fn.call(); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
243 if(commit) writer.commit(); |
252
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
244 } finally { |
704 | 245 wrote(); |
252
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
246 writeLock.unlock(); |
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
247 } |
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
248 } |
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
249 |
1335 | 250 public void run_in_lock(LuanFunction fn) throws IOException, LuanException { |
756
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
251 if( writeLock.isHeldByCurrentThread() ) |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
252 throw new RuntimeException(); |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
253 writeLock.lock(); |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
254 try { |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
255 synchronized(this) { |
1335 | 256 fn.call(); |
756
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
257 } |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
258 } finally { |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
259 wrote(); |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
260 writeLock.unlock(); |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
261 } |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
262 } |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
263 |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
264 |
754 | 265 private long id; |
266 private long idLim; | |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
267 private final int idBatch = 10; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
268 |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
269 private void initId() throws LuanException, IOException { |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
270 TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
271 switch(td.totalHits) { |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
272 case 0: |
754 | 273 id = 0; |
274 idLim = 0; | |
275 break; | |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
276 case 1: |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
277 idLim = (Long)searcher.doc(td.scoreDocs[0].doc).getField(FLD_NEXT_ID).numericValue(); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
278 id = idLim; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
279 break; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
280 default: |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
281 throw new RuntimeException(); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
282 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
283 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
284 |
1333
25746915a241
merge Luan and LuanState
Franklin Schmidt <fschmidt@gmail.com>
parents:
1271
diff
changeset
|
285 public synchronized long nextId(Luan luan) throws LuanException, IOException { |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
286 if( ++id > idLim ) { |
233
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
287 idLim += idBatch; |
1267 | 288 LuanTable doc = new LuanTable(luan); |
427
dae264ad6a7b
fix LuanTable.put() to use metatables
Franklin Schmidt <fschmidt@gmail.com>
parents:
426
diff
changeset
|
289 doc.rawPut( "type", "next_id" ); |
dae264ad6a7b
fix LuanTable.put() to use metatables
Franklin Schmidt <fschmidt@gmail.com>
parents:
426
diff
changeset
|
290 doc.rawPut( FLD_NEXT_ID, idLim ); |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
291 writer.updateDocument(new Term("type","next_id"),toLucene(doc,Collections.EMPTY_SET,null)); |
704 | 292 wrote(); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
293 } |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
294 return id; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
295 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
296 |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
297 /* |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
298 public void backup(String zipFile) throws LuanException, IOException { |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
299 if( !zipFile.endsWith(".zip") ) |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
300 throw new LuanException("file "+zipFile+" doesn't end with '.zip'"); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
301 IndexCommit ic = snapshotDeletionPolicy.snapshot(); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
302 try { |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
303 ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile)); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
304 for( String fileName : ic.getFileNames() ) { |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
305 out.putNextEntry(new ZipEntry(fileName)); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
306 FileInputStream in = new FileInputStream(new File(indexDir,fileName)); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
307 Utils.copyAll(in,out); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
308 in.close(); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
309 out.closeEntry(); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
310 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
311 out.close(); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
312 } finally { |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
313 snapshotDeletionPolicy.release(ic); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
314 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
315 } |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
316 */ |
1129
3234a14bb1f8
minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents:
1111
diff
changeset
|
317 public SnapshotDeletionPolicy snapshotDeletionPolicy() { |
3234a14bb1f8
minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents:
1111
diff
changeset
|
318 return snapshotDeletionPolicy; |
3234a14bb1f8
minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents:
1111
diff
changeset
|
319 } |
3234a14bb1f8
minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents:
1111
diff
changeset
|
320 |
1333
25746915a241
merge Luan and LuanState
Franklin Schmidt <fschmidt@gmail.com>
parents:
1271
diff
changeset
|
321 public Object snapshot(Luan luan,LuanFunction fn) throws LuanException, IOException { |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
322 IndexCommit ic = snapshotDeletionPolicy.snapshot(); |
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
323 try { |
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
324 String dir = fileDir.toString(); |
1267 | 325 LuanTable fileNames = new LuanTable(luan,new ArrayList(ic.getFileNames())); |
1335 | 326 return fn.call(dir,fileNames); |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
327 } finally { |
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
328 snapshotDeletionPolicy.release(ic); |
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
329 } |
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
330 } |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
331 |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
332 |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
333 |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
334 public String to_string() { |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
335 return writer.getDirectory().toString(); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
336 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
337 |
303
fdb4bd391c28
add lucene close();
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
287
diff
changeset
|
338 public void close() throws IOException { |
521
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
339 if( !isClosed ) { |
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
340 writer.close(); |
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
341 reader.close(); |
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
342 isClosed = true; |
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
343 } |
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
344 } |
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
345 |
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
346 protected void finalize() throws Throwable { |
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
347 if( !isClosed ) { |
1227 | 348 logger.error("not closed",created); |
521
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
349 close(); |
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
350 } |
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
351 super.finalize(); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
352 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
353 |
312
d34be4588556
add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
303
diff
changeset
|
354 |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
355 |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
356 private static class DocFn extends LuanFunction { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
357 final IndexSearcher searcher; |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
358 final Query query; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
359 int docID; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
360 |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
361 DocFn(Luan luan,IndexSearcher searcher,Query query) { |
1335 | 362 super(luan); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
363 this.searcher = searcher; |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
364 this.query = query; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
365 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
366 |
1335 | 367 @Override public Object call(Object[] args) throws LuanException { |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
368 try { |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
369 LuanTable doc = toTable(luan(),searcher.doc(docID)); |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
370 if( args.length > 0 && "explain".equals(args[0]) ) { |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
371 Explanation explanation = searcher.explain(query,docID); |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
372 return new Object[]{doc,explanation}; |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
373 } else { |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
374 return doc; |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
375 } |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
376 } catch(IOException e) { |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
377 throw new LuanException(e); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
378 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
379 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
380 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
381 |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
382 private static abstract class MyCollector extends Collector { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
383 int docBase; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
384 int i = 0; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
385 |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
386 @Override public void setScorer(Scorer scorer) {} |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
387 @Override public void setNextReader(AtomicReaderContext context) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
388 this.docBase = context.docBase; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
389 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
390 @Override public boolean acceptsDocsOutOfOrder() { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
391 return true; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
392 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
393 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
394 |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
395 private synchronized IndexSearcher openSearcher() throws IOException { |
704 | 396 int gwc = globalWriteCount(); |
397 if( writeCount != gwc ) { | |
398 writeCount = gwc; | |
399 DirectoryReader newReader = DirectoryReader.openIfChanged(reader); | |
400 if( newReader != null ) { | |
401 reader.decRef(); | |
402 reader = newReader; | |
403 searcher = new IndexSearcher(reader); | |
404 } | |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
405 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
406 reader.incRef(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
407 return searcher; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
408 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
409 |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
410 // call in finally block |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
411 private static void close(IndexSearcher searcher) throws IOException { |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
412 searcher.getIndexReader().decRef(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
413 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
414 |
591
790d5de23042
add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents:
578
diff
changeset
|
415 public void ensure_open() throws IOException { |
790d5de23042
add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents:
578
diff
changeset
|
416 close(openSearcher()); |
790d5de23042
add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents:
578
diff
changeset
|
417 } |
790d5de23042
add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents:
578
diff
changeset
|
418 |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
419 public int advanced_search( final Luan luan, String queryStr, LuanFunction fn, Integer n, String sortStr ) |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
420 throws LuanException, IOException, ParseException |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
421 { |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
422 Utils.checkNotNull(queryStr); |
622
1a53333eb4d5
remove Lucene all_search_terms_must_match() since now query suffix "~a" handles this
Franklin Schmidt <fschmidt@gmail.com>
parents:
621
diff
changeset
|
423 Query query = SaneQueryParser.parseQuery(mfp,queryStr); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
424 IndexSearcher searcher = threadLocalSearcher.get(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
425 boolean inTransaction = searcher != null; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
426 if( !inTransaction ) |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
427 searcher = openSearcher(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
428 try { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
429 if( fn!=null && n==null ) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
430 if( sortStr != null ) |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
431 throw new LuanException("sort must be nil when n is nil"); |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
432 final DocFn docFn = new DocFn(luan,searcher,query); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
433 MyCollector col = new MyCollector() { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
434 @Override public void collect(int doc) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
435 try { |
547
0be287ab0309
add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents:
546
diff
changeset
|
436 docFn.docID = docBase + doc; |
1335 | 437 fn.call(++i,docFn); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
438 } catch(LuanException e) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
439 throw new LuanRuntimeException(e); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
440 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
441 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
442 }; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
443 try { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
444 searcher.search(query,col); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
445 } catch(LuanRuntimeException e) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
446 throw (LuanException)e.getCause(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
447 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
448 return col.i; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
449 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
450 if( fn==null || n==0 ) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
451 TotalHitCountCollector thcc = new TotalHitCountCollector(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
452 searcher.search(query,thcc); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
453 return thcc.getTotalHits(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
454 } |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
455 Sort sort = sortStr==null ? null : SaneQueryParser.parseSort(mfp,sortStr); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
456 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
457 final ScoreDoc[] scoreDocs = td.scoreDocs; |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
458 DocFn docFn = new DocFn(luan,searcher,query); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
459 for( int i=0; i<scoreDocs.length; i++ ) { |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
460 ScoreDoc scoreDoc = scoreDocs[i]; |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
461 docFn.docID = scoreDoc.doc; |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
462 fn.call(i+1,docFn,scoreDoc.score); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
463 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
464 return td.totalHits; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
465 } finally { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
466 if( !inTransaction ) |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
467 close(searcher); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
468 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
469 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
470 |
1335 | 471 public Object search_in_transaction(LuanFunction fn) throws LuanException, IOException { |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
472 if( threadLocalSearcher.get() != null ) |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
473 throw new LuanException("can't nest search_in_transaction calls"); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
474 IndexSearcher searcher = openSearcher(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
475 threadLocalSearcher.set(searcher); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
476 try { |
1335 | 477 return fn.call(); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
478 } finally { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
479 threadLocalSearcher.set(null); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
480 close(searcher); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
481 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
482 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
483 |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
484 |
796 | 485 public FieldParser getIndexedFieldParser(String field) { |
486 return mfp.fields.get(field); | |
487 } | |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
488 |
796 | 489 public void setIndexedFieldParser(String field,FieldParser fp) { |
490 if( fp==null ) { // delete | |
491 mfp.fields.remove(field); | |
492 return; | |
493 } | |
494 mfp.fields.put( field, fp ); | |
495 } | |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
496 |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
497 |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
498 private IndexableField newField(String name,Object value,Field.Store store,Set<String> indexed,Float boost) |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
499 throws LuanException |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
500 { |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
501 IndexableField fld = newField2(name,value,store,indexed); |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
502 if( boost != null ) |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
503 ((Field)fld).setBoost(boost); |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
504 return fld; |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
505 } |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
506 |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
507 private IndexableField newField2(String name,Object value,Field.Store store,Set<String> indexed) |
621
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
508 throws LuanException |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
509 { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
510 if( value instanceof String ) { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
511 String s = (String)value; |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
512 FieldParser fp = mfp.fields.get(name); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
513 if( fp != null ) { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
514 if( fp instanceof StringFieldParser && fp != STRING_FIELD_PARSER ) { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
515 return new TextField(name, s, store); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
516 } else { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
517 return new StringField(name, s, store); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
518 } |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
519 } else { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
520 return new StoredField(name, s); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
521 } |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
522 } else if( value instanceof Integer ) { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
523 int i = (Integer)value; |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
524 if( indexed.contains(name) ) { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
525 return new IntField(name, i, store); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
526 } else { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
527 return new StoredField(name, i); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
528 } |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
529 } else if( value instanceof Long ) { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
530 long i = (Long)value; |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
531 if( indexed.contains(name) ) { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
532 return new LongField(name, i, store); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
533 } else { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
534 return new StoredField(name, i); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
535 } |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
536 } else if( value instanceof Double ) { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
537 double i = (Double)value; |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
538 if( indexed.contains(name) ) { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
539 return new DoubleField(name, i, store); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
540 } else { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
541 return new StoredField(name, i); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
542 } |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
543 } else if( value instanceof byte[] ) { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
544 byte[] b = (byte[])value; |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
545 return new StoredField(name, b); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
546 } else |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
547 throw new LuanException("invalid value type "+value.getClass()+"' for '"+name+"'"); |
621
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
548 } |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
549 |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
550 private Document toLucene(LuanTable table,Set indexOnly,LuanTable boosts) throws LuanException { |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
551 Set<String> indexed = mfp.fields.keySet(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
552 Document doc = new Document(); |
1267 | 553 for( Map.Entry<Object,Object> entry : table.iterable() ) { |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
554 Object key = entry.getKey(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
555 if( !(key instanceof String) ) |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
556 throw new LuanException("key must be string"); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
557 String name = (String)key; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
558 Object value = entry.getValue(); |
618
5e495e4e560b
add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents:
617
diff
changeset
|
559 Field.Store store = indexOnly.contains(name) ? Field.Store.NO : Field.Store.YES; |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
560 Float boost = null; |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
561 if( boosts != null ) { |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
562 Object obj = boosts.get(name); |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
563 if( obj != null ) { |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
564 if( !(obj instanceof Number) ) |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
565 throw new LuanException("boost '"+name+"' must be number"); |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
566 boost = ((Number)obj).floatValue(); |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
567 } |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
568 } |
621
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
569 if( !(value instanceof LuanTable) ) { |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
570 doc.add(newField( name, value, store, indexed, boost )); |
621
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
571 } else { // list |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
572 LuanTable list = (LuanTable)value; |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
573 for( Object el : list.asList() ) { |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
574 doc.add(newField( name, el, store, indexed, boost )); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
575 } |
621
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
576 } |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
577 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
578 return doc; |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
579 } |
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
580 |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
581 private static Object getValue(IndexableField ifld) throws LuanException { |
621
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
582 BytesRef br = ifld.binaryValue(); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
583 if( br != null ) |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
584 return br.bytes; |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
585 Number n = ifld.numericValue(); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
586 if( n != null ) |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
587 return n; |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
588 String s = ifld.stringValue(); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
589 if( s != null ) |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
590 return s; |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
591 throw new LuanException("invalid field type for "+ifld); |
621
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
592 } |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
593 |
1333
25746915a241
merge Luan and LuanState
Franklin Schmidt <fschmidt@gmail.com>
parents:
1271
diff
changeset
|
594 private static LuanTable toTable(Luan luan,Document doc) throws LuanException { |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
595 if( doc==null ) |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
596 return null; |
1267 | 597 LuanTable table = new LuanTable(luan); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
598 for( IndexableField ifld : doc ) { |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
599 String name = ifld.name(); |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
600 Object value = getValue(ifld); |
621
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
601 Object old = table.rawGet(name); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
602 if( old == null ) { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
603 table.rawPut(name,value); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
604 } else { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
605 LuanTable list; |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
606 if( old instanceof LuanTable ) { |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
607 list = (LuanTable)old; |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
608 } else { |
1267 | 609 list = new LuanTable(luan); |
621
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
610 list.rawPut(1,old); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
611 table.rawPut(name,list); |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
612 } |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
613 list.rawPut(list.rawLength()+1,value); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
614 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
615 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
616 return table; |
312
d34be4588556
add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
303
diff
changeset
|
617 } |
d34be4588556
add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
303
diff
changeset
|
618 |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
619 |
1248
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
620 private static final Formatter nullFormatter = new Formatter() { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
621 public String highlightTerm(String originalText,TokenGroup tokenGroup) { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
622 return originalText; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
623 } |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
624 }; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
625 |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
626 public LuanFunction highlighter(String queryStr,final LuanFunction formatter,final Integer fragmentSize,String dotdotdot) |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
627 throws ParseException |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
628 { |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
629 Query query = SaneQueryParser.parseQuery(mfp,queryStr); |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
630 Formatter fmt = new Formatter() { |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
631 public String highlightTerm(String originalText,TokenGroup tokenGroup) { |
1248
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
632 if( tokenGroup.getTotalScore() <= 0 ) |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
633 return originalText; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
634 try { |
1335 | 635 return (String)Luan.first(formatter.call(originalText)); |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
636 } catch(LuanException e) { |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
637 throw new LuanRuntimeException(e); |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
638 } |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
639 } |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
640 }; |
1248
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
641 QueryScorer queryScorer = new QueryScorer(query); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
642 final Highlighter chooser = fragmentSize==null ? null : new Highlighter(nullFormatter,queryScorer); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
643 if( chooser != null ) |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
644 chooser.setTextFragmenter( new SimpleSpanFragmenter(queryScorer,fragmentSize) ); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
645 final Highlighter hl = new Highlighter(fmt,queryScorer); |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
646 hl.setTextFragmenter( new NullFragmenter() ); |
1335 | 647 return new LuanFunction(false) { // ??? |
648 @Override public String call(Object[] args) throws LuanException { | |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
649 String text = (String)args[0]; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
650 try { |
1248
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
651 if( chooser != null ) { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
652 String s = chooser.getBestFragment(analyzer,null,text); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
653 if( s != null ) { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
654 if( dotdotdot != null ) { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
655 boolean atStart = text.startsWith(s); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
656 boolean atEnd = text.endsWith(s); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
657 if( !atStart ) |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
658 s = dotdotdot + s; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
659 if( !atEnd ) |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
660 s = s + dotdotdot; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
661 } |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
662 text = s; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
663 } else if( text.length() > fragmentSize ) { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
664 text = text.substring(0,fragmentSize); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
665 if( dotdotdot != null ) |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
666 text += "..."; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
667 } |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
668 } |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
669 String s = hl.getBestFragment(analyzer,null,text); |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
670 return s!=null ? s : text; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
671 } catch(LuanRuntimeException e) { |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
672 throw (LuanException)e.getCause(); |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
673 } catch(IOException e) { |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
674 throw new RuntimeException(e); |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
675 } catch(InvalidTokenOffsetsException e) { |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
676 throw new RuntimeException(e); |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
677 } |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
678 } |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
679 }; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
680 } |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
681 |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
682 public int count_tokens(String text) |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
683 throws IOException |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
684 { |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
685 int n = 0; |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
686 TokenStream ts = analyzer.tokenStream(null,text); |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
687 ts.reset(); |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
688 while( ts.incrementToken() ) { |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
689 n++; |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
690 } |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
691 ts.close(); |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
692 return n; |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
693 } |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
694 |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
695 } |