Mercurial Hosting > luan

comparison lucene/src/luan/modules/lucene/LuceneIndex.java @ 545:ddcd4296107a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
clean up lucene search
author Franklin Schmidt <fschmidt@gmail.com>
date Sun, 14 Jun 2015 01:34:42 -0600
parents c5a93767cc5c
children eaef1005ab87
comparison
equal deleted inserted replaced
544:c5a93767cc5c 545:ddcd4296107a
16 import org.apache.lucene.index.IndexWriter; 16 import org.apache.lucene.index.IndexWriter;
17 import org.apache.lucene.index.IndexWriterConfig; 17 import org.apache.lucene.index.IndexWriterConfig;
18 import org.apache.lucene.index.DirectoryReader; 18 import org.apache.lucene.index.DirectoryReader;
19 import org.apache.lucene.index.Term; 19 import org.apache.lucene.index.Term;
20 import org.apache.lucene.index.SnapshotDeletionPolicy; 20 import org.apache.lucene.index.SnapshotDeletionPolicy;
21 import org.apache.lucene.index.AtomicReaderContext;
21 import org.apache.lucene.store.Directory; 22 import org.apache.lucene.store.Directory;
22 import org.apache.lucene.store.FSDirectory; 23 import org.apache.lucene.store.FSDirectory;
23 import org.apache.lucene.util.Version; 24 import org.apache.lucene.util.Version;
24 import org.apache.lucene.search.Query; 25 import org.apache.lucene.search.Query;
25 import org.apache.lucene.search.TermQuery; 26 import org.apache.lucene.search.TermQuery;
26 import org.apache.lucene.search.TopDocs; 27 import org.apache.lucene.search.TopDocs;
27 import org.apache.lucene.search.Sort; 28 import org.apache.lucene.search.Sort;
28 import org.apache.lucene.search.SortField; 29 import org.apache.lucene.search.SortField;
29 import org.apache.lucene.search.IndexSearcher; 30 import org.apache.lucene.search.IndexSearcher;
31 import org.apache.lucene.search.TotalHitCountCollector;
32 import org.apache.lucene.search.ScoreDoc;
33 import org.apache.lucene.search.Collector;
34 import org.apache.lucene.search.Scorer;
30 import sane.lucene.queryparser.SaneQueryParser; 35 import sane.lucene.queryparser.SaneQueryParser;
31 import sane.lucene.queryparser.FieldParser; 36 import sane.lucene.queryparser.FieldParser;
32 import sane.lucene.queryparser.MultiFieldParser; 37 import sane.lucene.queryparser.MultiFieldParser;
33 import sane.lucene.queryparser.StringFieldParser; 38 import sane.lucene.queryparser.StringFieldParser;
34 import sane.lucene.queryparser.NumberFieldParser; 39 import sane.lucene.queryparser.NumberFieldParser;
39 import luan.LuanTable; 44 import luan.LuanTable;
40 import luan.LuanFunction; 45 import luan.LuanFunction;
41 import luan.LuanJavaFunction; 46 import luan.LuanJavaFunction;
42 import luan.LuanException; 47 import luan.LuanException;
43 import luan.LuanMeta; 48 import luan.LuanMeta;
49 import luan.LuanRuntimeException;
44 import org.slf4j.Logger; 50 import org.slf4j.Logger;
45 import org.slf4j.LoggerFactory; 51 import org.slf4j.LoggerFactory;
46 52
47 53
48 public final class LuceneIndex implements Closeable { 54 public final class LuceneIndex implements Closeable {
50 56
51 private static final String FLD_NEXT_ID = "nextId"; 57 private static final String FLD_NEXT_ID = "nextId";
52 private static final Analyzer analyzer = new KeywordAnalyzer(); 58 private static final Analyzer analyzer = new KeywordAnalyzer();
53 public static final FieldParser STRING_FIELD_PARSER = new StringFieldParser(analyzer); 59 public static final FieldParser STRING_FIELD_PARSER = new StringFieldParser(analyzer);
54 60
61 final LuanTable myTable;
55 final Lock writeLock = new ReentrantLock(); 62 final Lock writeLock = new ReentrantLock();
56 private final File indexDir; 63 private final File indexDir;
57 final SnapshotDeletionPolicy snapshotDeletionPolicy; 64 final SnapshotDeletionPolicy snapshotDeletionPolicy;
58 final IndexWriter writer; 65 final IndexWriter writer;
59 private DirectoryReader reader; 66 private DirectoryReader reader;
60 private LuceneSearcher searcher; 67 private IndexSearcher searcher;
68 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>();
61 private boolean isClosed = false; 69 private boolean isClosed = false;
62 private final MultiFieldParser mfp = new MultiFieldParser(); 70 private final MultiFieldParser mfp = new MultiFieldParser();
63 71
64 public LuceneIndex(LuanState luan,String indexDirStr) throws LuanException, IOException { 72 public LuceneIndex(LuanState luan,String indexDirStr,LuanTable myTable) throws LuanException, IOException {
73 this.myTable = myTable;
65 mfp.fields.put( "type", STRING_FIELD_PARSER ); 74 mfp.fields.put( "type", STRING_FIELD_PARSER );
66 mfp.fields.put( "id", NumberFieldParser.LONG ); 75 mfp.fields.put( "id", NumberFieldParser.LONG );
67 File indexDir = new File(indexDirStr); 76 File indexDir = new File(indexDirStr);
68 this.indexDir = indexDir; 77 this.indexDir = indexDir;
69 Directory dir = FSDirectory.open(indexDir); 78 Directory dir = FSDirectory.open(indexDir);
73 conf.setIndexDeletionPolicy(snapshotDeletionPolicy); 82 conf.setIndexDeletionPolicy(snapshotDeletionPolicy);
74 writer = new IndexWriter(dir,conf); 83 writer = new IndexWriter(dir,conf);
75 writer.commit(); // commit index creation 84 writer.commit(); // commit index creation
76 reader = DirectoryReader.open(dir); 85 reader = DirectoryReader.open(dir);
77 luan.onClose(this); 86 luan.onClose(this);
78 searcher = new LuceneSearcher(this,reader); 87 searcher = new IndexSearcher(reader);
79 initId(luan); 88 initId(luan);
80 } 89 }
81 90
82 Document toLucene(LuanState luan,LuanTable table) throws LuanException { 91 Document toLucene(LuanState luan,LuanTable table) throws LuanException {
83 return LuceneDocument.toLucene(luan,table,mfp.fields.keySet()); 92 return LuceneDocument.toLucene(luan,table,mfp.fields.keySet());
85 94
86 public LuceneWriter openWriter() { 95 public LuceneWriter openWriter() {
87 return new LuceneWriter(this); 96 return new LuceneWriter(this);
88 } 97 }
89 98
90 synchronized LuceneSearcher openSearcher() throws IOException { 99 private synchronized IndexSearcher openSearcher() throws IOException {
91 DirectoryReader newReader = DirectoryReader.openIfChanged(reader); 100 DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
92 if( newReader != null ) { 101 if( newReader != null ) {
93 reader.decRef(); 102 reader.decRef();
94 reader = newReader; 103 reader = newReader;
95 searcher = new LuceneSearcher(this,reader); 104 searcher = new IndexSearcher(reader);
96 } 105 }
97 reader.incRef(); 106 reader.incRef();
98 return searcher; 107 return searcher;
108 }
109
110 // call in finally block
111 private static void close(IndexSearcher searcher) throws IOException {
112 searcher.getIndexReader().decRef();
99 } 113 }
100 114
101 LuceneSnapshot openSnapshot() throws IOException { 115 LuceneSnapshot openSnapshot() throws IOException {
102 return new LuceneSnapshot(this); 116 return new LuceneSnapshot(this);
103 } 117 }
117 private long id = 0; 131 private long id = 0;
118 private long idLim = 0; 132 private long idLim = 0;
119 private final int idBatch = 10; 133 private final int idBatch = 10;
120 134
121 private void initId(LuanState luan) throws LuanException, IOException { 135 private void initId(LuanState luan) throws LuanException, IOException {
122 IndexSearcher searcher = this.searcher.searcher;
123 TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1); 136 TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1);
124 switch(td.totalHits) { 137 switch(td.totalHits) {
125 case 0: 138 case 0:
126 break; // do nothing 139 break; // do nothing
127 case 1: 140 case 1:
170 183
171 public String to_string() { 184 public String to_string() {
172 return writer.getDirectory().toString(); 185 return writer.getDirectory().toString();
173 } 186 }
174 187
175 public void Writer(LuanState luan,LuanFunction fn) throws LuanException, IOException { 188 public void writer(LuanState luan,LuanFunction fn) throws LuanException, IOException {
176 LuceneWriter writer = openWriter(); 189 LuceneWriter writer = openWriter();
177 try { 190 try {
178 luan.call( fn, new Object[]{writer.table()} ); 191 luan.call( fn, new Object[]{writer.table()} );
179 writer.commit(); 192 writer.commit();
180 } finally { 193 } finally {
181 writer.close(); 194 writer.close();
182 }
183 }
184
185 public Object Searcher(LuanState luan,LuanFunction fn) throws LuanException, IOException {
186 LuceneSearcher searcher = openSearcher();
187 try {
188 return luan.call( fn, new Object[]{searcher.table()} );
189 } finally {
190 searcher.close();
191 } 195 }
192 } 196 }
193 197
194 public void close() throws IOException { 198 public void close() throws IOException {
195 if( !isClosed ) { 199 if( !isClosed ) {
203 if( !isClosed ) { 207 if( !isClosed ) {
204 logger.error("not closed"); 208 logger.error("not closed");
205 close(); 209 close();
206 } 210 }
207 super.finalize(); 211 super.finalize();
212 }
213
214
215
216 private static class DocFn extends LuanFunction {
217 final IndexSearcher searcher;
218 int docID;
219
220 DocFn(IndexSearcher searcher) {
221 this.searcher = searcher;
222 }
223
224 @Override public Object call(LuanState luan,Object[] args) throws LuanException {
225 try {
226 return LuceneDocument.toTable(luan,searcher.doc(docID));
227 } catch(IOException e) {
228 throw luan.exception(e);
229 }
230 }
231 }
232
233 private static abstract class MyCollector extends Collector {
234 int docBase;
235 int i = 0;
236
237 @Override public void setScorer(Scorer scorer) {}
238 @Override public void setNextReader(AtomicReaderContext context) {
239 this.docBase = context.docBase;
240 }
241 @Override public boolean acceptsDocsOutOfOrder() {
242 return true;
243 }
244 }
245
246 public int advanced_search( final LuanState luan, String queryStr, LuanFunction fn, Integer n, String sortStr ) throws LuanException, IOException, ParseException {
247 Utils.checkNotNull(luan,queryStr);
248 Query query = parseQuery(queryStr);
249 IndexSearcher searcher = threadLocalSearcher.get();
250 boolean inTransaction = searcher != null;
251 if( !inTransaction )
252 searcher = openSearcher();
253 try {
254 if( fn!=null && n==null ) {
255 if( sortStr != null )
256 throw luan.exception("sort must be nil when n is nil");
257 final DocFn docFn = new DocFn(searcher);
258 MyCollector col = new MyCollector() {
259 @Override public void collect(int doc) {
260 try {
261 docFn.docID = doc;
262 luan.call(fn,new Object[]{++i,docFn});
263 } catch(LuanException e) {
264 throw new LuanRuntimeException(e);
265 }
266 }
267 };
268 try {
269 searcher.search(query,col);
270 } catch(LuanRuntimeException e) {
271 throw (LuanException)e.getCause();
272 }
273 return col.i;
274 }
275 if( fn==null || n==0 ) {
276 TotalHitCountCollector thcc = new TotalHitCountCollector();
277 searcher.search(query,thcc);
278 return thcc.getTotalHits();
279 }
280 Sort sort = sortStr==null ? null : parseSort(sortStr);
281 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort);
282 final ScoreDoc[] scoreDocs = td.scoreDocs;
283 DocFn docFn = new DocFn(searcher);
284 for( int i=0; i<scoreDocs.length; i++ ) {
285 docFn.docID = scoreDocs[i].doc;
286 luan.call(fn,new Object[]{i+1,docFn});
287 }
288 return td.totalHits;
289 } finally {
290 if( !inTransaction )
291 close(searcher);
292 }
293 }
294
295 public Object search_in_transaction(LuanState luan,LuanFunction fn) throws LuanException, IOException {
296 if( threadLocalSearcher.get() != null )
297 throw luan.exception("can't nest search_in_transaction calls");
298 IndexSearcher searcher = openSearcher();
299 threadLocalSearcher.set(searcher);
300 try {
301 return luan.call(fn);
302 } finally {
303 threadLocalSearcher.set(null);
304 close(searcher);
305 }
208 } 306 }
209 307
210 308
211 309
212 public final LuanMeta indexedFieldsMeta = new LuanMeta() { 310 public final LuanMeta indexedFieldsMeta = new LuanMeta() {