repo/luan: lucene/src/luan/modules/lucene/LuceneIndex.java annotate

annotate lucene/src/luan/modules/lucene/LuceneIndex.java @ 704:90c89138c234

optimize lucene

author	Franklin Schmidt <fschmidt@gmail.com>
date	Fri, 13 May 2016 17:13:13 -0600
parents	b21d82ee5756
children	1ed9e55f0be8

rev	line source
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	1 package luan.modules.lucene;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	2
527 ef0336efe33c minor Franklin Schmidt <fschmidt@gmail.com> parents: 524 diff changeset	3 import java.io.Closeable;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	4 import java.io.File;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	5 import java.io.FileOutputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	6 import java.io.FileInputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	7 import java.io.IOException;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	8 import java.util.Iterator;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	9 import java.util.Map;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	10 import java.util.List;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	11 import java.util.ArrayList;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	12 import java.util.Set;
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	13 import java.util.HashSet;
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	14 import java.util.Collections;
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	15 import java.util.concurrent.ConcurrentMap;
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	16 import java.util.concurrent.ConcurrentHashMap;
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	17 import java.util.concurrent.atomic.AtomicInteger;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	18 import java.util.concurrent.locks.Lock;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	19 import java.util.concurrent.locks.ReentrantLock;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	20 import java.util.zip.ZipOutputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	21 import java.util.zip.ZipEntry;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	22 import org.apache.lucene.analysis.Analyzer;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	23 import org.apache.lucene.analysis.core.KeywordAnalyzer;
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	24 import org.apache.lucene.document.Document;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	25 import org.apache.lucene.document.Field;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	26 import org.apache.lucene.document.StoredField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	27 import org.apache.lucene.document.StringField;
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	28 import org.apache.lucene.document.TextField;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	29 import org.apache.lucene.document.IntField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	30 import org.apache.lucene.document.LongField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	31 import org.apache.lucene.document.DoubleField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	32 import org.apache.lucene.index.IndexableField;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	33 import org.apache.lucene.index.IndexWriter;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	34 import org.apache.lucene.index.IndexWriterConfig;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	35 import org.apache.lucene.index.DirectoryReader;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	36 import org.apache.lucene.index.Term;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	37 import org.apache.lucene.index.SnapshotDeletionPolicy;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	38 import org.apache.lucene.index.IndexCommit;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	39 import org.apache.lucene.index.AtomicReaderContext;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	40 import org.apache.lucene.store.Directory;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	41 import org.apache.lucene.store.FSDirectory;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	42 import org.apache.lucene.util.Version;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	43 import org.apache.lucene.util.BytesRef;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	44 import org.apache.lucene.util.NumericUtils;
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	45 import org.apache.lucene.search.Query;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	46 import org.apache.lucene.search.TermQuery;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	47 import org.apache.lucene.search.TopDocs;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	48 import org.apache.lucene.search.Sort;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	49 import org.apache.lucene.search.SortField;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	50 import org.apache.lucene.search.IndexSearcher;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	51 import org.apache.lucene.search.TotalHitCountCollector;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	52 import org.apache.lucene.search.ScoreDoc;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	53 import org.apache.lucene.search.Collector;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	54 import org.apache.lucene.search.Scorer;
620 89eb02f9827f add Lucene all_search_terms_must_match() Franklin Schmidt <fschmidt@gmail.com> parents: 618 diff changeset	55 import org.apache.lucene.search.BooleanClause;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	56 import org.apache.lucene.search.highlight.Formatter;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	57 import org.apache.lucene.search.highlight.Highlighter;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	58 import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	59 import org.apache.lucene.search.highlight.NullFragmenter;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	60 import org.apache.lucene.search.highlight.QueryScorer;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	61 import org.apache.lucene.search.highlight.TokenGroup;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	62 import sane.lucene.queryparser.SaneQueryParser;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	63 import sane.lucene.queryparser.FieldParser;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	64 import sane.lucene.queryparser.MultiFieldParser;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	65 import sane.lucene.queryparser.StringFieldParser;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	66 import sane.lucene.queryparser.NumberFieldParser;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	67 import sane.lucene.queryparser.ParseException;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	68 import luan.modules.Utils;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	69 import luan.Luan;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	70 import luan.LuanState;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	71 import luan.LuanTable;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	72 import luan.LuanFunction;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	73 import luan.LuanException;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	74 import luan.LuanMeta;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	75 import luan.LuanRuntimeException;
521 8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	76 import org.slf4j.Logger;
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	77 import org.slf4j.LoggerFactory;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	78
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	79
527 ef0336efe33c minor Franklin Schmidt <fschmidt@gmail.com> parents: 524 diff changeset	80 public final class LuceneIndex implements Closeable {
521 8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	81 private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class);
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	82
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	83 private static final String FLD_NEXT_ID = "nextId";
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	84 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer());
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	85
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	86 private final ReentrantLock writeLock = new ReentrantLock();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	87 private final File indexDir;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	88 final SnapshotDeletionPolicy snapshotDeletionPolicy;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	89 private final IndexWriter writer;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	90 private DirectoryReader reader;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	91 private IndexSearcher searcher;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	92 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>();
521 8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	93 private boolean isClosed = false;
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	94 private final MultiFieldParser mfp;
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	95 public final LuanTable indexed_only_fields = new LuanTable();
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	96 private final Analyzer analyzer;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	97
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	98 private static ConcurrentMap<File,AtomicInteger> globalWriteCounters = new ConcurrentHashMap<File,AtomicInteger>();
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	99 private final File fileDir;
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	100 private int writeCount;
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	101
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	102 public LuceneIndex(LuanState luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields) throws LuanException, IOException {
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	103 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	104 mfp.fields.put( "type", STRING_FIELD_PARSER );
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	105 mfp.fields.put( "id", NumberFieldParser.LONG );
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	106 File indexDir = new File(indexDirStr);
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	107 this.indexDir = indexDir;
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	108 FSDirectory dir = FSDirectory.open(indexDir);
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	109 fileDir = dir.getDirectory();
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	110 globalWriteCounters.putIfAbsent(fileDir,new AtomicInteger());
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	111 Version version = Version.LUCENE_4_9;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	112 Analyzer analyzer = STRING_FIELD_PARSER.analyzer;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	113 if( defaultFieldParser instanceof StringFieldParser ) {
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	114 StringFieldParser sfp = (StringFieldParser)defaultFieldParser;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	115 analyzer = sfp.analyzer;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	116 }
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	117 this.analyzer = analyzer;
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	118 IndexWriterConfig conf = new IndexWriterConfig(version,analyzer);
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	119 snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	120 conf.setIndexDeletionPolicy(snapshotDeletionPolicy);
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	121 writer = new IndexWriter(dir,conf);
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	122 writer.commit(); // commit index creation
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	123 reader = DirectoryReader.open(dir);
527 ef0336efe33c minor Franklin Schmidt <fschmidt@gmail.com> parents: 524 diff changeset	124 luan.onClose(this);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	125 searcher = new IndexSearcher(reader);
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	126 initId();
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	127 }
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	128
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	129 private int globalWriteCount() {
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	130 return globalWriteCounters.get(fileDir).get();
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	131 }
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	132
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	133 private void wrote() {
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	134 globalWriteCounters.get(fileDir).incrementAndGet();
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	135 }
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	136
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	137 public void delete_all() throws IOException {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	138 boolean commit = !writeLock.isHeldByCurrentThread();
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	139 writeLock.lock();
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	140 try {
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	141 writer.deleteAll();
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	142 id = idLim = 0;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	143 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	144 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	145 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	146 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	147 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	148 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	149
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	150 private static Term term(String key,long value) {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	151 BytesRef br = new BytesRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	152 NumericUtils.longToPrefixCoded(value,0,br);
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	153 return new Term(key,br);
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	154 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	155
547 0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	156 public void delete(LuanState luan,String queryStr) throws LuanException, IOException, ParseException {
622 1a53333eb4d5 remove Lucene all_search_terms_must_match() since now query suffix "~a" handles this Franklin Schmidt <fschmidt@gmail.com> parents: 621 diff changeset	157 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	158
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	159 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	160 writeLock.lock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	161 try {
547 0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	162 writer.deleteDocuments(query);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	163 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	164 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	165 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	166 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	167 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	168 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	169
547 0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	170 public void save(LuanState luan,LuanTable doc) throws LuanException, IOException {
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	171 Set indexedOnlySet = new HashSet();
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	172 Object typeObj = doc.get(luan,"type");
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	173 if( typeObj==null )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	174 throw new LuanException("missing 'type' field");
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	175 if( !(typeObj instanceof String) )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	176 throw new LuanException("type must be string");
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	177 String type = (String)typeObj;
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	178 Object indexedOnlyObj = indexed_only_fields.get(luan,type);
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	179 if( indexedOnlyObj != null ) {
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	180 if( !(indexedOnlyObj instanceof LuanTable) )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	181 throw new LuanException("indexed_only_fields elements must be tables");
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	182 LuanTable indexedOnly = (LuanTable)indexedOnlyObj;
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	183 for( Map.Entry<Object,Object> entry : indexedOnly.iterable(luan) ) {
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	184 Object key = entry.getKey();
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	185 if( !(key instanceof String) )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	186 throw new LuanException("indexed_only_fields."+type+" entries must be strings");
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	187 String name = (String)key;
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	188 Object value = entry.getValue();
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	189 if( !(value instanceof LuanFunction) )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	190 throw new LuanException("indexed_only_fields."+type+" values must be functions");
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	191 LuanFunction fn = (LuanFunction)value;
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	192 value = Luan.first(fn.call(luan,new Object[]{doc}));
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	193 doc.put(luan, name, value );
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	194 indexedOnlySet.add(name);
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	195 }
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	196 }
601 d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	197 Object obj = doc.get(luan,"id");
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	198 Long id;
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	199 try {
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	200 id = (Long)obj;
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	201 } catch(ClassCastException e) {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	202 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName());
601 d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	203 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	204
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	205 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	206 writeLock.lock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	207 try {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	208 if( id == null ) {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	209 id = nextId(luan);
575 7c3ad6db8ac3 make LuanState.JAVA private Franklin Schmidt <fschmidt@gmail.com> parents: 574 diff changeset	210 doc.put(luan,"id",id);
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	211 writer.addDocument(toLucene(luan,doc,indexedOnlySet));
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	212 } else {
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	213 writer.updateDocument( term("id",id), toLucene(luan,doc,indexedOnlySet) );
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	214 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	215 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	216 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	217 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	218 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	219 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	220 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	221
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	222 public void update_in_transaction(LuanState luan,LuanFunction fn) throws IOException, LuanException {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	223 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	224 writeLock.lock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	225 try {
574 6cc2f047019b remove LuanState.call() Franklin Schmidt <fschmidt@gmail.com> parents: 572 diff changeset	226 fn.call(luan);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	227 if(commit) writer.commit();
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	228 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	229 wrote();
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	230 writeLock.unlock();
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	231 }
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	232 }
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	233
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	234
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	235 private long id = 0;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	236 private long idLim = 0;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	237 private final int idBatch = 10;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	238
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	239 private void initId() throws LuanException, IOException {
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	240 TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1);
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	241 /*
547 0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	242 // tmp hack
0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	243 if( td.totalHits == 0 ) {
0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	244 td = searcher.search(new TermQuery(new Term("type index","next_id")),1);
0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	245 if( td.totalHits == 1 ) {
0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	246 long idLim = (Long)searcher.doc(td.scoreDocs[0].doc).getField(FLD_NEXT_ID).numericValue();
0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	247 LuanTable doc = new LuanTable();
0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	248 doc.rawPut( "type", "next_id" );
0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	249 doc.rawPut( FLD_NEXT_ID, idLim );
0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	250 writer.addDocument(toLucene(luan,doc));
0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	251 writer.commit();
0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	252 }
0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	253 }
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	254 */
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	255 switch(td.totalHits) {
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	256 case 0:
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	257 break; // do nothing
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	258 case 1:
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	259 idLim = (Long)searcher.doc(td.scoreDocs[0].doc).getField(FLD_NEXT_ID).numericValue();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	260 id = idLim;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	261 break;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	262 default:
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	263 throw new RuntimeException();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	264 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	265 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	266
617 e54c1646eed0 add Lucene.next_id() Franklin Schmidt <fschmidt@gmail.com> parents: 601 diff changeset	267 public synchronized long nextId(LuanState luan) throws LuanException, IOException {
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	268 if( ++id > idLim ) {
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	269 idLim += idBatch;
411 23b99a5039b5 remove Luan.newTable() Franklin Schmidt <fschmidt@gmail.com> parents: 312 diff changeset	270 LuanTable doc = new LuanTable();
427 dae264ad6a7b fix LuanTable.put() to use metatables Franklin Schmidt <fschmidt@gmail.com> parents: 426 diff changeset	271 doc.rawPut( "type", "next_id" );
dae264ad6a7b fix LuanTable.put() to use metatables Franklin Schmidt <fschmidt@gmail.com> parents: 426 diff changeset	272 doc.rawPut( FLD_NEXT_ID, idLim );
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	273 writer.updateDocument(new Term("type","next_id"),toLucene(luan,doc,Collections.EMPTY_SET));
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	274 wrote();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	275 }
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	276 return id;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	277 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	278
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	279
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	280 public void backup(String zipFile) throws LuanException, IOException {
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	281 if( !zipFile.endsWith(".zip") )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	282 throw new LuanException("file "+zipFile+" doesn't end with '.zip'");
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	283 IndexCommit ic = snapshotDeletionPolicy.snapshot();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	284 try {
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	285 ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile));
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	286 for( String fileName : ic.getFileNames() ) {
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	287 out.putNextEntry(new ZipEntry(fileName));
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	288 FileInputStream in = new FileInputStream(new File(indexDir,fileName));
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	289 Utils.copyAll(in,out);
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	290 in.close();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	291 out.closeEntry();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	292 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	293 out.close();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	294 } finally {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	295 snapshotDeletionPolicy.release(ic);
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	296 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	297 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	298
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	299
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	300
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	301 public String to_string() {
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	302 return writer.getDirectory().toString();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	303 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	304
303 fdb4bd391c28 add lucene close(); fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 287 diff changeset	305 public void close() throws IOException {
521 8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	306 if( !isClosed ) {
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	307 writer.close();
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	308 reader.close();
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	309 isClosed = true;
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	310 }
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	311 }
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	312
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	313 protected void finalize() throws Throwable {
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	314 if( !isClosed ) {
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	315 logger.error("not closed");
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	316 close();
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	317 }
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	318 super.finalize();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	319 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	320
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	321
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	322
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	323 private static class DocFn extends LuanFunction {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	324 final IndexSearcher searcher;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	325 int docID;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	326
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	327 DocFn(IndexSearcher searcher) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	328 this.searcher = searcher;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	329 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	330
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	331 @Override public Object call(LuanState luan,Object[] args) throws LuanException {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	332 try {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	333 return toTable(searcher.doc(docID));
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	334 } catch(IOException e) {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	335 throw new LuanException(e);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	336 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	337 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	338 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	339
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	340 private static abstract class MyCollector extends Collector {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	341 int docBase;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	342 int i = 0;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	343
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	344 @Override public void setScorer(Scorer scorer) {}
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	345 @Override public void setNextReader(AtomicReaderContext context) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	346 this.docBase = context.docBase;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	347 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	348 @Override public boolean acceptsDocsOutOfOrder() {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	349 return true;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	350 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	351 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	352
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	353 private synchronized IndexSearcher openSearcher() throws IOException {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	354 int gwc = globalWriteCount();
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	355 if( writeCount != gwc ) {
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	356 writeCount = gwc;
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	357 DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	358 if( newReader != null ) {
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	359 reader.decRef();
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	360 reader = newReader;
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	361 searcher = new IndexSearcher(reader);
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	362 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	363 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	364 reader.incRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	365 return searcher;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	366 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	367
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	368 // call in finally block
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	369 private static void close(IndexSearcher searcher) throws IOException {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	370 searcher.getIndexReader().decRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	371 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	372
591 790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	373 public void ensure_open() throws IOException {
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	374 close(openSearcher());
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	375 }
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	376
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	377 public int advanced_search( final LuanState luan, String queryStr, LuanFunction fn, Integer n, String sortStr ) throws LuanException, IOException, ParseException {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	378 Utils.checkNotNull(queryStr);
622 1a53333eb4d5 remove Lucene all_search_terms_must_match() since now query suffix "~a" handles this Franklin Schmidt <fschmidt@gmail.com> parents: 621 diff changeset	379 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	380 IndexSearcher searcher = threadLocalSearcher.get();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	381 boolean inTransaction = searcher != null;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	382 if( !inTransaction )
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	383 searcher = openSearcher();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	384 try {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	385 if( fn!=null && n==null ) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	386 if( sortStr != null )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	387 throw new LuanException("sort must be nil when n is nil");
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	388 final DocFn docFn = new DocFn(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	389 MyCollector col = new MyCollector() {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	390 @Override public void collect(int doc) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	391 try {
547 0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	392 docFn.docID = docBase + doc;
574 6cc2f047019b remove LuanState.call() Franklin Schmidt <fschmidt@gmail.com> parents: 572 diff changeset	393 fn.call(luan,new Object[]{++i,docFn});
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	394 } catch(LuanException e) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	395 throw new LuanRuntimeException(e);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	396 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	397 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	398 };
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	399 try {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	400 searcher.search(query,col);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	401 } catch(LuanRuntimeException e) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	402 throw (LuanException)e.getCause();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	403 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	404 return col.i;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	405 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	406 if( fn==null \|\| n==0 ) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	407 TotalHitCountCollector thcc = new TotalHitCountCollector();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	408 searcher.search(query,thcc);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	409 return thcc.getTotalHits();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	410 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	411 Sort sort = sortStr==null ? null : SaneQueryParser.parseSort(mfp,sortStr);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	412 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	413 final ScoreDoc[] scoreDocs = td.scoreDocs;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	414 DocFn docFn = new DocFn(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	415 for( int i=0; i<scoreDocs.length; i++ ) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	416 docFn.docID = scoreDocs[i].doc;
574 6cc2f047019b remove LuanState.call() Franklin Schmidt <fschmidt@gmail.com> parents: 572 diff changeset	417 fn.call(luan,new Object[]{i+1,docFn});
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	418 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	419 return td.totalHits;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	420 } finally {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	421 if( !inTransaction )
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	422 close(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	423 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	424 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	425
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	426 public Object search_in_transaction(LuanState luan,LuanFunction fn) throws LuanException, IOException {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	427 if( threadLocalSearcher.get() != null )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	428 throw new LuanException("can't nest search_in_transaction calls");
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	429 IndexSearcher searcher = openSearcher();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	430 threadLocalSearcher.set(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	431 try {
574 6cc2f047019b remove LuanState.call() Franklin Schmidt <fschmidt@gmail.com> parents: 572 diff changeset	432 return fn.call(luan);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	433 } finally {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	434 threadLocalSearcher.set(null);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	435 close(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	436 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	437 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	438
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	439
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	440
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	441 public final LuanMeta indexedFieldsMeta = new LuanMeta() {
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	442
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	443 @Override public boolean canNewindex() {
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	444 return true;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	445 }
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	446
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	447 @Override public Object __index(LuanState luan,LuanTable tbl,Object key) {
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	448 return mfp.fields.get(key);
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	449 }
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	450
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	451 @Override public void __new_index(LuanState luan,LuanTable tbl,Object key,Object value) throws LuanException {
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	452 if( !(key instanceof String) )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	453 throw new LuanException("key must be string");
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	454 String field = (String)key;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	455 if( value==null ) { // delete
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	456 mfp.fields.remove(field);
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	457 return;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	458 }
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	459 if( !(value instanceof FieldParser) )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	460 throw new LuanException("value must be FieldParser like the values of Lucene.type");
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	461 FieldParser parser = (FieldParser)value;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	462 mfp.fields.put( field, parser );
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	463 }
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	464
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	465 @Override public final Iterator keys(LuanTable tbl) {
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	466 return mfp.fields.keySet().iterator();
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	467 }
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	468
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	469 @Override protected String type(LuanTable tbl) {
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	470 return "lucene-indexed-fields";
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	471 }
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	472
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	473 };
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	474
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	475
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	476
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	477 private IndexableField newField(String name,Object value,Field.Store store,Set<String> indexed)
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	478 throws LuanException
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	479 {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	480 if( value instanceof String ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	481 String s = (String)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	482 FieldParser fp = mfp.fields.get(name);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	483 if( fp != null ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	484 if( fp instanceof StringFieldParser && fp != STRING_FIELD_PARSER ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	485 return new TextField(name, s, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	486 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	487 return new StringField(name, s, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	488 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	489 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	490 return new StoredField(name, s);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	491 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	492 } else if( value instanceof Integer ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	493 int i = (Integer)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	494 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	495 return new IntField(name, i, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	496 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	497 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	498 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	499 } else if( value instanceof Long ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	500 long i = (Long)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	501 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	502 return new LongField(name, i, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	503 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	504 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	505 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	506 } else if( value instanceof Double ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	507 double i = (Double)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	508 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	509 return new DoubleField(name, i, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	510 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	511 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	512 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	513 } else if( value instanceof byte[] ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	514 byte[] b = (byte[])value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	515 return new StoredField(name, b);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	516 } else
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	517 throw new LuanException("invalid value type "+value.getClass()+"' for '"+name+"'");
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	518 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	519
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	520 private Document toLucene(LuanState luan,LuanTable table,Set indexOnly) throws LuanException {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	521 Set<String> indexed = mfp.fields.keySet();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	522 Document doc = new Document();
575 7c3ad6db8ac3 make LuanState.JAVA private Franklin Schmidt <fschmidt@gmail.com> parents: 574 diff changeset	523 for( Map.Entry<Object,Object> entry : table.iterable(luan) ) {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	524 Object key = entry.getKey();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	525 if( !(key instanceof String) )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	526 throw new LuanException("key must be string");
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	527 String name = (String)key;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	528 Object value = entry.getValue();
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	529 Field.Store store = indexOnly.contains(name) ? Field.Store.NO : Field.Store.YES;
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	530 if( !(value instanceof LuanTable) ) {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	531 doc.add(newField(name, value, store, indexed));
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	532 } else { // list
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	533 LuanTable list = (LuanTable)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	534 for( Object el : list.asList() ) {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	535 doc.add(newField(name, el, store, indexed));
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	536 }
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	537 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	538 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	539 return doc;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	540 }
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	541
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	542 private static Object getValue(IndexableField ifld) throws LuanException {
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	543 BytesRef br = ifld.binaryValue();
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	544 if( br != null )
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	545 return br.bytes;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	546 Number n = ifld.numericValue();
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	547 if( n != null )
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	548 return n;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	549 String s = ifld.stringValue();
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	550 if( s != null )
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	551 return s;
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	552 throw new LuanException("invalid field type for "+ifld);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	553 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	554
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	555 private static LuanTable toTable(Document doc) throws LuanException {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	556 if( doc==null )
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	557 return null;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	558 LuanTable table = new LuanTable();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	559 for( IndexableField ifld : doc ) {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	560 String name = ifld.name();
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	561 Object value = getValue(ifld);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	562 Object old = table.rawGet(name);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	563 if( old == null ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	564 table.rawPut(name,value);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	565 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	566 LuanTable list;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	567 if( old instanceof LuanTable ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	568 list = (LuanTable)old;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	569 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	570 list = new LuanTable();
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	571 list.rawPut(1,old);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	572 table.rawPut(name,list);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	573 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	574 list.rawPut(list.rawLength()+1,value);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	575 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	576 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	577 return table;
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	578 }
d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	579
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	580
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	581 public LuanFunction highlighter(LuanState luan,String queryStr,LuanFunction formatter) throws ParseException {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	582 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	583 Formatter fmt = new Formatter() {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	584 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	585 if( tokenGroup.getTotalScore() <= 0 )
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	586 return originalText;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	587 try {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	588 return (String)Luan.first(formatter.call(luan,new Object[]{originalText}));
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	589 } catch(LuanException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	590 throw new LuanRuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	591 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	592 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	593 };
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	594 Highlighter hl = new Highlighter( fmt, new QueryScorer(query) );
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	595 hl.setTextFragmenter( new NullFragmenter() );
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	596 return new LuanFunction() {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	597 @Override public String call(LuanState luan,Object[] args) throws LuanException {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	598 String text = (String)args[0];
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	599 try {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	600 String s = hl.getBestFragment(analyzer,null,text);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	601 return s!=null ? s : text;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	602 } catch(LuanRuntimeException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	603 throw (LuanException)e.getCause();
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	604 } catch(IOException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	605 throw new RuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	606 } catch(InvalidTokenOffsetsException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	607 throw new RuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	608 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	609 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	610 };
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	611 }
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	612 }

Mercurial Hosting > luan

annotate lucene/src/luan/modules/lucene/LuceneIndex.java @ 704:90c89138c234