diff src/luan/modules/lucene/LuceneIndex.java @ 1341:a015a0b5c388

add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
author Franklin Schmidt <fschmidt@gmail.com>
date Tue, 19 Feb 2019 08:14:40 -0700
parents 8b61c8c4e07a
children 60599adc27b8
line wrap: on
line diff
--- a/src/luan/modules/lucene/LuceneIndex.java	Mon Feb 18 05:11:50 2019 -0700
+++ b/src/luan/modules/lucene/LuceneIndex.java	Tue Feb 19 08:14:40 2019 -0700
@@ -20,6 +20,7 @@
 import java.util.zip.ZipOutputStream;
 import java.util.zip.ZipEntry;
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
@@ -166,7 +167,9 @@
 		return new Term(key,br);
 	}
 
-	public void delete(Luan luan,String queryStr) throws LuanException, IOException, ParseException {
+	public void delete(Luan luan,String queryStr)
+		throws LuanException, IOException, ParseException
+	{
 		Query query = SaneQueryParser.parseQuery(mfp,queryStr);
 
 		boolean commit = !writeLock.isHeldByCurrentThread();
@@ -187,7 +190,9 @@
 		map.put(field,fn);
 	}
 
-	public void save(Luan luan,LuanTable doc) throws LuanException, IOException {
+	public void save(Luan luan,LuanTable doc,LuanTable boosts)
+		throws LuanException, IOException
+	{
 		Set indexedOnlySet = new HashSet();
 		Object typeObj = doc.get("type");
 		if( typeObj==null )
@@ -219,9 +224,9 @@
 			if( id == null ) {
 				id = nextId(luan);
 				doc.put("id",id);
-				writer.addDocument(toLucene(doc,indexedOnlySet));
+				writer.addDocument(toLucene(doc,indexedOnlySet,boosts));
 			} else {
-				writer.updateDocument( term("id",id), toLucene(doc,indexedOnlySet) );
+				writer.updateDocument( term("id",id), toLucene(doc,indexedOnlySet,boosts) );
 			}
 			if(commit) writer.commit();
 		} finally {
@@ -283,7 +288,7 @@
 			LuanTable doc = new LuanTable(luan);
 			doc.rawPut( "type", "next_id" );
 			doc.rawPut( FLD_NEXT_ID, idLim );
-			writer.updateDocument(new Term("type","next_id"),toLucene(doc,Collections.EMPTY_SET));
+			writer.updateDocument(new Term("type","next_id"),toLucene(doc,Collections.EMPTY_SET,null));
 			wrote();
 		}
 		return id;
@@ -403,7 +408,9 @@
 		close(openSearcher());
 	}
 
-	public int advanced_search( final Luan luan, String queryStr, LuanFunction fn, Integer n, String sortStr ) throws LuanException, IOException, ParseException {
+	public int advanced_search( final Luan luan, String queryStr, LuanFunction fn, Integer n, String sortStr )
+		throws LuanException, IOException, ParseException
+	{
 		Utils.checkNotNull(queryStr);
 		Query query = SaneQueryParser.parseQuery(mfp,queryStr);
 		IndexSearcher searcher = threadLocalSearcher.get();
@@ -479,7 +486,16 @@
 	}
 
 
-	private IndexableField newField(String name,Object value,Field.Store store,Set<String> indexed)
+	private IndexableField newField(String name,Object value,Field.Store store,Set<String> indexed,Float boost)
+		throws LuanException
+	{
+		IndexableField fld = newField2(name,value,store,indexed);
+		if( boost != null )
+			((Field)fld).setBoost(boost);
+		return fld;
+	}
+
+	private IndexableField newField2(String name,Object value,Field.Store store,Set<String> indexed)
 		throws LuanException
 	{
 		if( value instanceof String ) {
@@ -522,7 +538,7 @@
 			throw new LuanException("invalid value type "+value.getClass()+"' for '"+name+"'");
 	}
 
-	private Document toLucene(LuanTable table,Set indexOnly) throws LuanException {
+	private Document toLucene(LuanTable table,Set indexOnly,LuanTable boosts) throws LuanException {
 		Set<String> indexed = mfp.fields.keySet();
 		Document doc = new Document();
 		for( Map.Entry<Object,Object> entry : table.iterable() ) {
@@ -532,12 +548,21 @@
 			String name = (String)key;
 			Object value = entry.getValue();
 			Field.Store store = indexOnly.contains(name) ? Field.Store.NO : Field.Store.YES;
+			Float boost = null;
+			if( boosts != null ) {
+				Object obj = boosts.get(name);
+				if( obj != null ) {
+					if( !(obj instanceof Number) )
+						throw new LuanException("boost '"+name+"' must be number");
+					boost = ((Number)obj).floatValue();
+				}
+			}
 			if( !(value instanceof LuanTable) ) {
-				doc.add(newField(name, value, store, indexed));
+				doc.add(newField( name, value, store, indexed, boost ));
 			} else { // list
 				LuanTable list = (LuanTable)value;
 				for( Object el : list.asList() ) {
-					doc.add(newField(name, el, store, indexed));
+					doc.add(newField( name, el, store, indexed, boost ));
 				}
 			}
 		}
@@ -642,4 +667,18 @@
 			}
 		};
 	}
+
+	public int count_tokens(String text)
+		throws IOException
+	{
+		int n = 0;
+		TokenStream ts = analyzer.tokenStream(null,text);
+		ts.reset();
+		while( ts.incrementToken() ) {
+			n++;
+		}
+		ts.close();
+		return n;
+	}
+
 }