diff lucene/src/luan/modules/lucene/LuceneIndex.java @ 544:c5a93767cc5c

lucene overhaul, untested
author Franklin Schmidt <fschmidt@gmail.com>
date Fri, 12 Jun 2015 19:11:44 -0600
parents ef0336efe33c
children ddcd4296107a
line wrap: on
line diff
--- a/lucene/src/luan/modules/lucene/LuceneIndex.java	Mon Jun 08 01:11:08 2015 -0400
+++ b/lucene/src/luan/modules/lucene/LuceneIndex.java	Fri Jun 12 19:11:44 2015 -0600
@@ -5,12 +5,13 @@
 import java.io.FileOutputStream;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.zip.ZipOutputStream;
 import java.util.zip.ZipEntry;
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
@@ -23,8 +24,15 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser;
-import org.apache.lucene.queryparser.flexible.core.QueryNodeException;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.IndexSearcher;
+import sane.lucene.queryparser.SaneQueryParser;
+import sane.lucene.queryparser.FieldParser;
+import sane.lucene.queryparser.MultiFieldParser;
+import sane.lucene.queryparser.StringFieldParser;
+import sane.lucene.queryparser.NumberFieldParser;
+import sane.lucene.queryparser.ParseException;
 import luan.modules.Utils;
 import luan.Luan;
 import luan.LuanState;
@@ -32,6 +40,7 @@
 import luan.LuanFunction;
 import luan.LuanJavaFunction;
 import luan.LuanException;
+import luan.LuanMeta;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -40,6 +49,8 @@
 	private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class);
 
 	private static final String FLD_NEXT_ID = "nextId";
+	private static final Analyzer analyzer = new KeywordAnalyzer();
+	public static final FieldParser STRING_FIELD_PARSER = new StringFieldParser(analyzer);
 
 	final Lock writeLock = new ReentrantLock();
 	private final File indexDir;
@@ -47,15 +58,16 @@
 	final IndexWriter writer;
 	private DirectoryReader reader;
 	private LuceneSearcher searcher;
-	public final FieldTable fields = new FieldTable();
 	private boolean isClosed = false;
+	private final MultiFieldParser mfp = new MultiFieldParser();
 
 	public LuceneIndex(LuanState luan,String indexDirStr) throws LuanException, IOException {
+		mfp.fields.put( "type", STRING_FIELD_PARSER );
+		mfp.fields.put( "id", NumberFieldParser.LONG );
 		File indexDir = new File(indexDirStr);
 		this.indexDir = indexDir;
 		Directory dir = FSDirectory.open(indexDir);
 		Version version = Version.LUCENE_4_9;
-		Analyzer analyzer = new StandardAnalyzer(version);
 		IndexWriterConfig conf = new IndexWriterConfig(version,analyzer);
 		snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
 		conf.setIndexDeletionPolicy(snapshotDeletionPolicy);
@@ -68,20 +80,7 @@
 	}
 
 	Document toLucene(LuanState luan,LuanTable table) throws LuanException {
-		return LuceneDocument.toLucene(luan,table,fields.map);
-	}
-
-	LuanTable toTable(LuanState luan,Document doc) throws LuanException {
-		return LuceneDocument.toTable(luan,doc,fields.reverseMap);
-	}
-
-	public String map_field_name(String fld) {
-		String s = fields.map.get(fld);
-		return s!=null ? s : fld;
-	}
-
-	Term newTerm(String fld,String text) {
-		return new Term(map_field_name(fld),text);
+		return LuceneDocument.toLucene(luan,table,mfp.fields.keySet());
 	}
 
 	public LuceneWriter openWriter() {
@@ -120,13 +119,13 @@
 	private final int idBatch = 10;
 
 	private void initId(LuanState luan) throws LuanException, IOException {
-		TopDocs td = searcher.search(new TermQuery(newTerm("type","next_id")),1);
+		IndexSearcher searcher = this.searcher.searcher;
+		TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1);
 		switch(td.totalHits) {
 		case 0:
 			break;  // do nothing
 		case 1:
-			LuanTable doc = searcher.doc(luan,td.scoreDocs[0].doc);
-			idLim = (Long)doc.rawGet(FLD_NEXT_ID);
+			idLim = (Long)searcher.doc(td.scoreDocs[0].doc).getField(FLD_NEXT_ID).numericValue();
 			id = idLim;
 			break;
 		default:
@@ -134,16 +133,15 @@
 		}
 	}
 
-	synchronized String nextId(LuanState luan) throws LuanException, IOException {
-		String rtn = Long.toString(++id);
-		if( id > idLim ) {
+	synchronized long nextId(LuanState luan) throws LuanException, IOException {
+		if( ++id > idLim ) {
 			idLim += idBatch;
 			LuanTable doc = new LuanTable();
 			doc.rawPut( "type", "next_id" );
 			doc.rawPut( FLD_NEXT_ID, idLim );
-			writer.updateDocument(newTerm("type","next_id"),toLucene(luan,doc));
+			writer.updateDocument(new Term("type","next_id"),toLucene(luan,doc));
 		}
-		return rtn;
+		return id;
 	}
 
 
@@ -210,10 +208,47 @@
 	}
 
 
-	public Query parse(String s) throws QueryNodeException {
-		StandardQueryParser qp = new StandardQueryParser();
-		qp.setQueryNodeProcessor(new LuanQueryNodeProcessor(this,qp.getQueryNodeProcessor()));
-		return qp.parse(s,null);
+
+	public final LuanMeta indexedFieldsMeta = new LuanMeta() {
+
+		@Override public boolean canNewindex() {
+			return true;
+		}
+
+		@Override public Object __index(LuanState luan,LuanTable tbl,Object key) {
+			return mfp.fields.get(key);
+		}
+
+		@Override public void __new_index(LuanState luan,LuanTable tbl,Object key,Object value) throws LuanException {
+			if( !(key instanceof String) )
+				throw luan.exception("key must be string");
+			String field = (String)key;
+			if( value==null ) {  // delete
+				mfp.fields.remove(field);
+				return;
+			}
+			if( !(value instanceof FieldParser) )
+				throw luan.exception("value must be FieldParser like the values of Lucene.type");
+			FieldParser parser = (FieldParser)value;
+			mfp.fields.put( field, parser );
+		}
+
+		@Override public final Iterator keys(LuanTable tbl) {
+			return mfp.fields.keySet().iterator();
+		}
+
+		@Override protected String type(LuanTable tbl) {
+			return "lucene-indexed-fields";
+		}
+
+	};
+
+	public Query parseQuery(String s) throws ParseException {
+		return SaneQueryParser.parseQuery(mfp,s);
+	}
+
+	public Sort parseSort(String s) throws ParseException {
+		return SaneQueryParser.parseSort(mfp,s);
 	}
 
 }