repo/luan: src/luan/modules/lucene/LuceneIndex.java annotate

annotate src/luan/modules/lucene/LuceneIndex.java @ 1369:709f7498a363

change Lucene.index() and add Lucene.recover()

author	Franklin Schmidt <fschmidt@gmail.com>
date	Wed, 19 Jun 2019 00:26:10 -0600
parents	643cf1c37723
children	87a3738d7cc5

rev	line source
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	1 package luan.modules.lucene;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	2
527 ef0336efe33c minor Franklin Schmidt <fschmidt@gmail.com> parents: 524 diff changeset	3 import java.io.Closeable;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	4 import java.io.File;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	5 import java.io.FileOutputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	6 import java.io.FileInputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	7 import java.io.IOException;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	8 import java.util.Arrays;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	9 import java.util.Iterator;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	10 import java.util.Map;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	11 import java.util.HashMap;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	12 import java.util.List;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	13 import java.util.ArrayList;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	14 import java.util.Set;
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	15 import java.util.HashSet;
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	16 import java.util.Collections;
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	17 import java.util.concurrent.atomic.AtomicInteger;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	18 import java.util.concurrent.locks.Lock;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	19 import java.util.concurrent.locks.ReentrantLock;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	20 import java.util.zip.ZipOutputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	21 import java.util.zip.ZipEntry;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	22 import org.apache.lucene.analysis.Analyzer;
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	23 import org.apache.lucene.analysis.TokenStream;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	24 import org.apache.lucene.analysis.core.KeywordAnalyzer;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	25 import org.apache.lucene.analysis.en.EnglishAnalyzer;
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	26 import org.apache.lucene.document.Document;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	27 import org.apache.lucene.document.Field;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	28 import org.apache.lucene.document.StoredField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	29 import org.apache.lucene.document.StringField;
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	30 import org.apache.lucene.document.TextField;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	31 import org.apache.lucene.document.IntField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	32 import org.apache.lucene.document.LongField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	33 import org.apache.lucene.document.DoubleField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	34 import org.apache.lucene.index.IndexableField;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	35 import org.apache.lucene.index.IndexWriter;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	36 import org.apache.lucene.index.IndexWriterConfig;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	37 import org.apache.lucene.index.DirectoryReader;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	38 import org.apache.lucene.index.Term;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	39 import org.apache.lucene.index.SnapshotDeletionPolicy;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	40 import org.apache.lucene.index.IndexCommit;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	41 import org.apache.lucene.index.AtomicReaderContext;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	42 import org.apache.lucene.store.Directory;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	43 import org.apache.lucene.store.FSDirectory;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	44 import org.apache.lucene.util.Version;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	45 import org.apache.lucene.util.BytesRef;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	46 import org.apache.lucene.util.NumericUtils;
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	47 import org.apache.lucene.search.Query;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	48 import org.apache.lucene.search.TermQuery;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	49 import org.apache.lucene.search.TopDocs;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	50 import org.apache.lucene.search.Sort;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	51 import org.apache.lucene.search.SortField;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	52 import org.apache.lucene.search.IndexSearcher;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	53 import org.apache.lucene.search.TotalHitCountCollector;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	54 import org.apache.lucene.search.ScoreDoc;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	55 import org.apache.lucene.search.Collector;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	56 import org.apache.lucene.search.Scorer;
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	57 import org.apache.lucene.search.Explanation;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	58 import org.apache.lucene.search.highlight.Formatter;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	59 import org.apache.lucene.search.highlight.Highlighter;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	60 import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	61 import org.apache.lucene.search.highlight.Fragmenter;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	62 import org.apache.lucene.search.highlight.NullFragmenter;
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	63 import org.apache.lucene.search.highlight.SimpleSpanFragmenter;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	64 import org.apache.lucene.search.highlight.QueryScorer;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	65 import org.apache.lucene.search.highlight.TokenGroup;
1344 dc2af9d5463b move queryparser to lib Franklin Schmidt <fschmidt@gmail.com> parents: 1343 diff changeset	66 import luan.lib.queryparser.SaneQueryParser;
dc2af9d5463b move queryparser to lib Franklin Schmidt <fschmidt@gmail.com> parents: 1343 diff changeset	67 import luan.lib.queryparser.FieldParser;
dc2af9d5463b move queryparser to lib Franklin Schmidt <fschmidt@gmail.com> parents: 1343 diff changeset	68 import luan.lib.queryparser.MultiFieldParser;
dc2af9d5463b move queryparser to lib Franklin Schmidt <fschmidt@gmail.com> parents: 1343 diff changeset	69 import luan.lib.queryparser.StringFieldParser;
dc2af9d5463b move queryparser to lib Franklin Schmidt <fschmidt@gmail.com> parents: 1343 diff changeset	70 import luan.lib.queryparser.NumberFieldParser;
1111 88b5b81cad4a move Parser to luan.lib.parser Franklin Schmidt <fschmidt@gmail.com> parents: 1110 diff changeset	71 import luan.lib.parser.ParseException;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	72 import luan.modules.Utils;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	73 import luan.Luan;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	74 import luan.LuanTable;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	75 import luan.LuanFunction;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	76 import luan.LuanException;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	77 import luan.LuanRuntimeException;
1337 8b61c8c4e07a remove slf4j Franklin Schmidt <fschmidt@gmail.com> parents: 1335 diff changeset	78 import luan.lib.logging.Logger;
8b61c8c4e07a remove slf4j Franklin Schmidt <fschmidt@gmail.com> parents: 1335 diff changeset	79 import luan.lib.logging.LoggerFactory;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	80
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	81
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	82 public final class LuceneIndex {
521 8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	83 private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class);
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	84
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	85 private static final class Closer implements Closeable {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	86 final LuceneIndex li;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	87 boolean isClosed = false;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	88 private final Exception created = new Exception("created");
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	89
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	90 Closer(Luan luan,LuceneIndex li) {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	91 this.li = li;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	92 luan.onClose(this);
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	93 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	94
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	95 public void close() throws IOException {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	96 if( !isClosed ) {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	97 li.close();
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	98 isClosed = true;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	99 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	100 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	101
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	102 protected void finalize() throws Throwable {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	103 if( !isClosed ) {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	104 logger.error("not closed",created);
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	105 close();
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	106 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	107 super.finalize();
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	108 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	109 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	110
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	111 private static Map<String,LuceneIndex> indexes = new HashMap<String,LuceneIndex>();
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	112
1369 709f7498a363 change Lucene.index() and add Lucene.recover() Franklin Schmidt <fschmidt@gmail.com> parents: 1347 diff changeset	113 public static Object[] getLuceneIndex(Luan luan,File indexDir,FieldParser defaultFieldParser,String[] defaultFields)
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	114 throws LuanException, IOException
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	115 {
1369 709f7498a363 change Lucene.index() and add Lucene.recover() Franklin Schmidt <fschmidt@gmail.com> parents: 1347 diff changeset	116 String key = indexDir.getCanonicalPath();
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	117 synchronized(indexes) {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	118 LuceneIndex li = indexes.get(key);
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	119 if( li == null ) {
1369 709f7498a363 change Lucene.index() and add Lucene.recover() Franklin Schmidt <fschmidt@gmail.com> parents: 1347 diff changeset	120 li = new LuceneIndex(indexDir,defaultFieldParser,defaultFields,key);
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	121 li.openCount = 1;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	122 indexes.put(key,li);
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	123 } else {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	124 if( defaultFieldParser != li.defaultFieldParser )
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	125 throw new LuanException("default_type doesn't match previous use");
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	126 if( !Arrays.equals(defaultFields,li.defaultFields) )
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	127 throw new LuanException("default_fields don't match previous use");
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	128 li.openCount++;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	129 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	130 return new Object[]{li,new Closer(luan,li)};
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	131 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	132 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	133
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	134 private static final Version version = Version.LUCENE_4_9;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	135 private static final String FLD_NEXT_ID = "nextId";
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	136 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer());
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	137 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(version));
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	138
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	139 private final ReentrantLock writeLock = new ReentrantLock();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	140 private final File indexDir;
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	141 private SnapshotDeletionPolicy snapshotDeletionPolicy;
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	142 private IndexWriter writer;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	143 private DirectoryReader reader;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	144 private IndexSearcher searcher;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	145 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>();
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	146 private final MultiFieldParser mfp;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	147 private final Analyzer analyzer;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	148
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	149 private File fileDir;
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	150 private int writeCount;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	151 private AtomicInteger writeCounter = new AtomicInteger();
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	152
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	153 private Set<String> indexOnly = new HashSet<String>();
1271 48f302bdc187 fix indexed_only_field Franklin Schmidt <fschmidt@gmail.com> parents: 1267 diff changeset	154
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	155 private int openCount;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	156 private final String key;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	157 private final FieldParser defaultFieldParser;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	158 private final String[] defaultFields;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	159
1369 709f7498a363 change Lucene.index() and add Lucene.recover() Franklin Schmidt <fschmidt@gmail.com> parents: 1347 diff changeset	160 private LuceneIndex(File indexDir,FieldParser defaultFieldParser,String[] defaultFields,String key)
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	161 throws LuanException, IOException
9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	162 {
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	163 this.key = key;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	164 this.defaultFieldParser = defaultFieldParser;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	165 this.defaultFields = defaultFields;
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	166 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	167 mfp.fields.put( "type", STRING_FIELD_PARSER );
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	168 mfp.fields.put( "id", NumberFieldParser.LONG );
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	169 this.indexDir = indexDir;
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	170 Analyzer analyzer = STRING_FIELD_PARSER.analyzer;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	171 if( defaultFieldParser instanceof StringFieldParser ) {
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	172 StringFieldParser sfp = (StringFieldParser)defaultFieldParser;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	173 analyzer = sfp.analyzer;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	174 }
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	175 this.analyzer = analyzer;
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	176 reopen();
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	177 }
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	178
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	179 public void reopen() throws LuanException, IOException {
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	180 IndexWriterConfig conf = new IndexWriterConfig(version,analyzer);
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	181 snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	182 conf.setIndexDeletionPolicy(snapshotDeletionPolicy);
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	183 FSDirectory dir = FSDirectory.open(indexDir);
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	184 fileDir = dir.getDirectory();
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	185 writer = new IndexWriter(dir,conf);
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	186 writer.commit(); // commit index creation
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	187 reader = DirectoryReader.open(dir);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	188 searcher = new IndexSearcher(reader);
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	189 initId();
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	190 }
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	191
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	192 private void wrote() {
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	193 writeCounter.incrementAndGet();
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	194 }
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	195
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	196 public void delete_all() throws IOException {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	197 boolean commit = !writeLock.isHeldByCurrentThread();
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	198 writeLock.lock();
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	199 try {
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	200 writer.deleteAll();
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	201 id = idLim = 0;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	202 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	203 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	204 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	205 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	206 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	207 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	208
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	209 private static Term term(String key,long value) {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	210 BytesRef br = new BytesRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	211 NumericUtils.longToPrefixCoded(value,0,br);
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	212 return new Term(key,br);
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	213 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	214
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	215 public void delete(String queryStr)
efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	216 throws IOException, ParseException
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	217 {
622 1a53333eb4d5 remove Lucene all_search_terms_must_match() since now query suffix "~a" handles this Franklin Schmidt <fschmidt@gmail.com> parents: 621 diff changeset	218 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	219
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	220 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	221 writeLock.lock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	222 try {
547 0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	223 writer.deleteDocuments(query);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	224 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	225 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	226 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	227 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	228 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	229 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	230
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	231 public void indexed_only_fields(List<String> fields) {
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	232 indexOnly.addAll(fields);
1271 48f302bdc187 fix indexed_only_field Franklin Schmidt <fschmidt@gmail.com> parents: 1267 diff changeset	233 }
48f302bdc187 fix indexed_only_field Franklin Schmidt <fschmidt@gmail.com> parents: 1267 diff changeset	234
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	235 public void save(LuanTable doc,LuanTable boosts)
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	236 throws LuanException, IOException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	237 {
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	238 Object obj = doc.get("id");
601 d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	239 Long id;
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	240 try {
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	241 id = (Long)obj;
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	242 } catch(ClassCastException e) {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	243 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName());
601 d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	244 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	245
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	246 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	247 writeLock.lock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	248 try {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	249 if( id == null ) {
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	250 id = nextId();
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	251 doc.put("id",id);
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	252 writer.addDocument(toLucene(doc,boosts));
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	253 } else {
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	254 writer.updateDocument( term("id",id), toLucene(doc,boosts) );
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	255 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	256 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	257 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	258 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	259 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	260 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	261 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	262
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	263 public void update_in_transaction(LuanFunction fn) throws IOException, LuanException {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	264 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	265 writeLock.lock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	266 try {
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	267 fn.call();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	268 if(commit) writer.commit();
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	269 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	270 wrote();
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	271 writeLock.unlock();
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	272 }
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	273 }
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	274
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	275 public void run_in_lock(LuanFunction fn) throws IOException, LuanException {
756 9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	276 if( writeLock.isHeldByCurrentThread() )
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	277 throw new RuntimeException();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	278 writeLock.lock();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	279 try {
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	280 synchronized(this) {
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	281 fn.call();
756 9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	282 }
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	283 } finally {
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	284 wrote();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	285 writeLock.unlock();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	286 }
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	287 }
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	288
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	289
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	290 private long id;
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	291 private long idLim;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	292 private final int idBatch = 10;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	293
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	294 private void initId() throws LuanException, IOException {
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	295 TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1);
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	296 switch(td.totalHits) {
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	297 case 0:
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	298 id = 0;
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	299 idLim = 0;
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	300 break;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	301 case 1:
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	302 idLim = (Long)searcher.doc(td.scoreDocs[0].doc).getField(FLD_NEXT_ID).numericValue();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	303 id = idLim;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	304 break;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	305 default:
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	306 throw new RuntimeException();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	307 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	308 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	309
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	310 public synchronized long nextId() throws LuanException, IOException {
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	311 if( ++id > idLim ) {
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	312 idLim += idBatch;
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	313 Map doc = new HashMap();
efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	314 doc.put( "type", "next_id" );
efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	315 doc.put( FLD_NEXT_ID, idLim );
efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	316 writer.updateDocument(new Term("type","next_id"),toLucene(doc.entrySet(),null));
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	317 wrote();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	318 }
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	319 return id;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	320 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	321
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	322 /*
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	323 public void backup(String zipFile) throws LuanException, IOException {
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	324 if( !zipFile.endsWith(".zip") )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	325 throw new LuanException("file "+zipFile+" doesn't end with '.zip'");
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	326 IndexCommit ic = snapshotDeletionPolicy.snapshot();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	327 try {
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	328 ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile));
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	329 for( String fileName : ic.getFileNames() ) {
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	330 out.putNextEntry(new ZipEntry(fileName));
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	331 FileInputStream in = new FileInputStream(new File(indexDir,fileName));
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	332 Utils.copyAll(in,out);
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	333 in.close();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	334 out.closeEntry();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	335 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	336 out.close();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	337 } finally {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	338 snapshotDeletionPolicy.release(ic);
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	339 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	340 }
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	341 */
1129 3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	342 public SnapshotDeletionPolicy snapshotDeletionPolicy() {
3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	343 return snapshotDeletionPolicy;
3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	344 }
3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	345
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	346 public Object snapshot(LuanFunction fn) throws LuanException, IOException {
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	347 IndexCommit ic = snapshotDeletionPolicy.snapshot();
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	348 try {
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	349 String dir = fileDir.toString();
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	350 LuanTable fileNames = new LuanTable(fn.luan(),new ArrayList(ic.getFileNames()));
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	351 return fn.call(dir,fileNames);
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	352 } finally {
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	353 snapshotDeletionPolicy.release(ic);
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	354 }
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	355 }
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	356
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	357
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	358
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	359 public String to_string() {
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	360 return writer.getDirectory().toString();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	361 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	362
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	363 private synchronized void close() throws IOException {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	364 if( openCount > 0 ) {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	365 if( --openCount == 0 ) {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	366 doClose();
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	367 synchronized(indexes) {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	368 indexes.remove(key);
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	369 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	370 }
521 8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	371 }
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	372 }
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	373
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	374 public void doClose() throws IOException {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	375 writer.close();
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	376 reader.close();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	377 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	378
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	379
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	380 private static class DocFn extends LuanFunction {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	381 final IndexSearcher searcher;
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	382 final Query query;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	383 int docID;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	384
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	385 DocFn(Luan luan,IndexSearcher searcher,Query query) {
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	386 super(luan);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	387 this.searcher = searcher;
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	388 this.query = query;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	389 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	390
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	391 @Override public Object call(Object[] args) throws LuanException {
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	392 try {
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	393 LuanTable doc = toTable(luan(),searcher.doc(docID));
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	394 if( args.length > 0 && "explain".equals(args[0]) ) {
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	395 Explanation explanation = searcher.explain(query,docID);
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	396 return new Object[]{doc,explanation};
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	397 } else {
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	398 return doc;
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	399 }
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	400 } catch(IOException e) {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	401 throw new LuanException(e);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	402 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	403 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	404 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	405
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	406 private static abstract class MyCollector extends Collector {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	407 int docBase;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	408 int i = 0;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	409
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	410 @Override public void setScorer(Scorer scorer) {}
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	411 @Override public void setNextReader(AtomicReaderContext context) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	412 this.docBase = context.docBase;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	413 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	414 @Override public boolean acceptsDocsOutOfOrder() {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	415 return true;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	416 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	417 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	418
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	419 private synchronized IndexSearcher openSearcher() throws IOException {
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	420 int gwc = writeCounter.get();
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	421 if( writeCount != gwc ) {
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	422 writeCount = gwc;
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	423 DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	424 if( newReader != null ) {
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	425 reader.decRef();
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	426 reader = newReader;
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	427 searcher = new IndexSearcher(reader);
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	428 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	429 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	430 reader.incRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	431 return searcher;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	432 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	433
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	434 // call in finally block
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	435 private static void close(IndexSearcher searcher) throws IOException {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	436 searcher.getIndexReader().decRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	437 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	438
591 790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	439 public void ensure_open() throws IOException {
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	440 close(openSearcher());
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	441 }
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	442
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	443 public int advanced_search( String queryStr, LuanFunction fn, Integer n, String sortStr )
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	444 throws LuanException, IOException, ParseException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	445 {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	446 Utils.checkNotNull(queryStr);
622 1a53333eb4d5 remove Lucene all_search_terms_must_match() since now query suffix "~a" handles this Franklin Schmidt <fschmidt@gmail.com> parents: 621 diff changeset	447 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	448 IndexSearcher searcher = threadLocalSearcher.get();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	449 boolean inTransaction = searcher != null;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	450 if( !inTransaction )
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	451 searcher = openSearcher();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	452 try {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	453 if( fn!=null && n==null ) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	454 if( sortStr != null )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	455 throw new LuanException("sort must be nil when n is nil");
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	456 final DocFn docFn = new DocFn(fn.luan(),searcher,query);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	457 MyCollector col = new MyCollector() {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	458 @Override public void collect(int doc) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	459 try {
547 0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	460 docFn.docID = docBase + doc;
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	461 fn.call(++i,docFn);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	462 } catch(LuanException e) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	463 throw new LuanRuntimeException(e);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	464 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	465 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	466 };
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	467 try {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	468 searcher.search(query,col);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	469 } catch(LuanRuntimeException e) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	470 throw (LuanException)e.getCause();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	471 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	472 return col.i;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	473 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	474 if( fn==null \|\| n==0 ) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	475 TotalHitCountCollector thcc = new TotalHitCountCollector();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	476 searcher.search(query,thcc);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	477 return thcc.getTotalHits();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	478 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	479 Sort sort = sortStr==null ? null : SaneQueryParser.parseSort(mfp,sortStr);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	480 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	481 final ScoreDoc[] scoreDocs = td.scoreDocs;
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	482 DocFn docFn = new DocFn(fn.luan(),searcher,query);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	483 for( int i=0; i<scoreDocs.length; i++ ) {
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	484 ScoreDoc scoreDoc = scoreDocs[i];
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	485 docFn.docID = scoreDoc.doc;
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	486 fn.call(i+1,docFn,scoreDoc.score);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	487 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	488 return td.totalHits;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	489 } finally {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	490 if( !inTransaction )
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	491 close(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	492 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	493 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	494
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	495 public Object search_in_transaction(LuanFunction fn) throws LuanException, IOException {
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	496 if( threadLocalSearcher.get() != null )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	497 throw new LuanException("can't nest search_in_transaction calls");
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	498 IndexSearcher searcher = openSearcher();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	499 threadLocalSearcher.set(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	500 try {
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	501 return fn.call();
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	502 } finally {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	503 threadLocalSearcher.set(null);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	504 close(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	505 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	506 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	507
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	508
796 6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	509 public FieldParser getIndexedFieldParser(String field) {
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	510 return mfp.fields.get(field);
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	511 }
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	512
796 6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	513 public void setIndexedFieldParser(String field,FieldParser fp) {
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	514 if( fp==null ) { // delete
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	515 mfp.fields.remove(field);
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	516 return;
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	517 }
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	518 mfp.fields.put( field, fp );
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	519 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	520
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	521
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	522 private IndexableField newField(String name,Object value,Set<String> indexed,Float boost)
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	523 throws LuanException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	524 {
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	525 boolean hasBoost = boost!=null;
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	526 IndexableField fld = newField2(name,value,indexed,hasBoost);
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	527 if( hasBoost )
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	528 ((Field)fld).setBoost(boost);
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	529 return fld;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	530 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	531
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	532 private IndexableField newField2(String name,Object value,Set<String> indexed,boolean hasBoost)
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	533 throws LuanException
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	534 {
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	535 Field.Store store = indexOnly.contains(name) ? Field.Store.NO : Field.Store.YES;
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	536 if( value instanceof String ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	537 String s = (String)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	538 FieldParser fp = mfp.fields.get(name);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	539 if( fp != null ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	540 if( fp instanceof StringFieldParser && fp != STRING_FIELD_PARSER ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	541 return new TextField(name, s, store);
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	542 } else if (hasBoost) {
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	543 // fuck you modern lucene developers
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	544 return new Field(name, s, store, Field.Index.NOT_ANALYZED);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	545 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	546 return new StringField(name, s, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	547 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	548 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	549 return new StoredField(name, s);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	550 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	551 } else if( value instanceof Integer ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	552 int i = (Integer)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	553 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	554 return new IntField(name, i, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	555 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	556 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	557 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	558 } else if( value instanceof Long ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	559 long i = (Long)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	560 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	561 return new LongField(name, i, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	562 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	563 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	564 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	565 } else if( value instanceof Double ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	566 double i = (Double)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	567 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	568 return new DoubleField(name, i, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	569 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	570 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	571 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	572 } else if( value instanceof byte[] ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	573 byte[] b = (byte[])value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	574 return new StoredField(name, b);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	575 } else
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	576 throw new LuanException("invalid value type "+value.getClass()+"' for '"+name+"'");
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	577 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	578
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	579 private Document toLucene(LuanTable table,LuanTable boosts) throws LuanException {
1347 643cf1c37723 move webserver to lib and bug fixes Franklin Schmidt <fschmidt@gmail.com> parents: 1346 diff changeset	580 return toLucene(table.iterable(),boosts);
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	581 }
efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	582
efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	583 private Document toLucene(Iterable<Map.Entry> iterable,LuanTable boosts) throws LuanException {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	584 Set<String> indexed = mfp.fields.keySet();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	585 Document doc = new Document();
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	586 for( Map.Entry<Object,Object> entry : iterable ) {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	587 Object key = entry.getKey();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	588 if( !(key instanceof String) )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	589 throw new LuanException("key must be string");
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	590 String name = (String)key;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	591 Object value = entry.getValue();
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	592 Float boost = null;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	593 if( boosts != null ) {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	594 Object obj = boosts.get(name);
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	595 if( obj != null ) {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	596 if( !(obj instanceof Number) )
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	597 throw new LuanException("boost '"+name+"' must be number");
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	598 boost = ((Number)obj).floatValue();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	599 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	600 }
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	601 if( !(value instanceof LuanTable) ) {
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	602 doc.add(newField( name, value, indexed, boost ));
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	603 } else { // list
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	604 LuanTable list = (LuanTable)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	605 for( Object el : list.asList() ) {
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	606 doc.add(newField( name, el, indexed, boost ));
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	607 }
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	608 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	609 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	610 return doc;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	611 }
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	612
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	613 private static Object getValue(IndexableField ifld) throws LuanException {
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	614 BytesRef br = ifld.binaryValue();
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	615 if( br != null )
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	616 return br.bytes;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	617 Number n = ifld.numericValue();
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	618 if( n != null )
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	619 return n;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	620 String s = ifld.stringValue();
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	621 if( s != null )
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	622 return s;
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	623 throw new LuanException("invalid field type for "+ifld);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	624 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	625
1333 25746915a241 merge Luan and LuanState Franklin Schmidt <fschmidt@gmail.com> parents: 1271 diff changeset	626 private static LuanTable toTable(Luan luan,Document doc) throws LuanException {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	627 if( doc==null )
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	628 return null;
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	629 LuanTable table = new LuanTable(luan);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	630 for( IndexableField ifld : doc ) {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	631 String name = ifld.name();
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	632 Object value = getValue(ifld);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	633 Object old = table.rawGet(name);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	634 if( old == null ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	635 table.rawPut(name,value);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	636 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	637 LuanTable list;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	638 if( old instanceof LuanTable ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	639 list = (LuanTable)old;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	640 } else {
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	641 list = new LuanTable(luan);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	642 list.rawPut(1,old);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	643 table.rawPut(name,list);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	644 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	645 list.rawPut(list.rawLength()+1,value);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	646 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	647 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	648 return table;
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	649 }
d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	650
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	651
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	652 private static final Formatter nullFormatter = new Formatter() {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	653 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	654 return originalText;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	655 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	656 };
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	657
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	658 public LuanFunction highlighter(String queryStr,final LuanFunction formatter,final Integer fragmentSize,String dotdotdot)
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	659 throws ParseException
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	660 {
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	661 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	662 Formatter fmt = new Formatter() {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	663 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	664 if( tokenGroup.getTotalScore() <= 0 )
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	665 return originalText;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	666 try {
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	667 return (String)Luan.first(formatter.call(originalText));
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	668 } catch(LuanException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	669 throw new LuanRuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	670 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	671 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	672 };
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	673 QueryScorer queryScorer = new QueryScorer(query);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	674 final Highlighter chooser = fragmentSize==null ? null : new Highlighter(nullFormatter,queryScorer);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	675 if( chooser != null )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	676 chooser.setTextFragmenter( new SimpleSpanFragmenter(queryScorer,fragmentSize) );
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	677 final Highlighter hl = new Highlighter(fmt,queryScorer);
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	678 hl.setTextFragmenter( new NullFragmenter() );
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	679 return new LuanFunction(false) { // ???
e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	680 @Override public String call(Object[] args) throws LuanException {
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	681 String text = (String)args[0];
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	682 try {
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	683 if( chooser != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	684 String s = chooser.getBestFragment(analyzer,null,text);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	685 if( s != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	686 if( dotdotdot != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	687 boolean atStart = text.startsWith(s);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	688 boolean atEnd = text.endsWith(s);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	689 if( !atStart )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	690 s = dotdotdot + s;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	691 if( !atEnd )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	692 s = s + dotdotdot;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	693 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	694 text = s;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	695 } else if( text.length() > fragmentSize ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	696 text = text.substring(0,fragmentSize);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	697 if( dotdotdot != null )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	698 text += "...";
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	699 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	700 }
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	701 String s = hl.getBestFragment(analyzer,null,text);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	702 return s!=null ? s : text;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	703 } catch(LuanRuntimeException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	704 throw (LuanException)e.getCause();
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	705 } catch(IOException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	706 throw new RuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	707 } catch(InvalidTokenOffsetsException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	708 throw new RuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	709 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	710 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	711 };
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	712 }
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	713
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	714 public int count_tokens(String text)
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	715 throws IOException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	716 {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	717 int n = 0;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	718 TokenStream ts = analyzer.tokenStream(null,text);
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	719 ts.reset();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	720 while( ts.incrementToken() ) {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	721 n++;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	722 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	723 ts.close();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	724 return n;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	725 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	726
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	727 }

Mercurial Hosting > luan

annotate src/luan/modules/lucene/LuceneIndex.java @ 1369:709f7498a363