--- a/src/luan/modules/lucene/LuceneIndex.java	Sun Jul 26 15:06:15 2020 -0600
+++ b/src/luan/modules/lucene/LuceneIndex.java	Sun Jul 26 23:11:53 2020 -0600
@@ -12,6 +12,7 @@
 import java.util.Iterator;
 import java.util.Map;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.ArrayList;
 import java.util.Set;
@@ -74,6 +75,10 @@
 import goodjava.lucene.queryparser.MultiFieldParser;
 import goodjava.lucene.queryparser.StringFieldParser;
 import goodjava.lucene.queryparser.NumberFieldParser;
+import goodjava.lucene.api.GoodIndexWriter;
+import goodjava.lucene.api.LuceneIndexWriter;
+import goodjava.lucene.api.GoodIndexWriterConfig;
+import goodjava.lucene.api.LuceneUtils;
 import goodjava.parser.ParseException;
 import luan.modules.Utils;
 import luan.Luan;
@@ -122,21 +127,17 @@
 
 	private final ReentrantLock writeLock = new ReentrantLock();
 	private final File indexDir;
-	private SnapshotDeletionPolicy snapshotDeletionPolicy;
-	private IndexWriter writer;
+	private GoodIndexWriter writer;
 	private DirectoryReader reader;
 	private IndexSearcher searcher;
 	private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>();
 	private final MultiFieldParser mfp;
-	private final Analyzer analyzer;
+	private final Analyzer analyzer;  // ???
 
 	private FSDirectory fsDir;
 	private int writeCount;
 	private AtomicInteger writeCounter = new AtomicInteger();
-
-	private Set<String> indexOnly = new HashSet<String>();
-//	private final FieldParser defaultFieldParser;
-//	private final String[] defaultFields;
+	private final GoodIndexWriterConfig config;
 
 	private final PostgresBackup postgresBackup;
 	private boolean wasCreated;
@@ -150,10 +151,9 @@
 		LuanTable defaultFieldsTbl = Utils.removeTable(options,"default_fields");
 		String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]);
 		LuanTable postgresSpec = Utils.removeTable(options,"postgres_spec");
+		LuanFunction supplementer = Utils.removeFunction(options,"supplementer");
 		Utils.checkEmpty(options);
 
-//		this.defaultFieldParser = defaultFieldParser;
-//		this.defaultFields = defaultFields;
 		mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
 		mfp.fields.put( "type", STRING_FIELD_PARSER );
 		mfp.fields.put( "id", NumberFieldParser.LONG );
@@ -164,6 +164,7 @@
 			analyzer = sfp.analyzer;
 		}
 		this.analyzer = analyzer;
+		this.config = new SupplementingConfig(luceneVersion,mfp,supplementer);
 		wasCreated = reopen();
 		if( postgresSpec == null ) {
 			postgresBackup = null;
@@ -182,13 +183,9 @@
 	}
 
 	public boolean reopen() throws IOException {
-		IndexWriterConfig conf = new IndexWriterConfig(luceneVersion,analyzer);
-		snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
-		conf.setIndexDeletionPolicy(snapshotDeletionPolicy);
 		fsDir = FSDirectory.open(indexDir);
 		boolean wasCreated = !fsDir.getDirectory().exists();
-		writer = new IndexWriter(fsDir,conf);
-		writer.commit();  // commit index creation
+		writer = new LuceneIndexWriter(fsDir,config);
 		reader = DirectoryReader.open(fsDir);
 		searcher = new IndexSearcher(reader);
 		initId();
@@ -263,16 +260,9 @@
 		}
 	}
 
-	public void indexed_only_fields(List<String> fields) {
-		indexOnly.addAll(fields);
-	}
-
-	public void save(LuanFunction completer,LuanTable doc,LuanTable boosts)
+	public void save(LuanTable doc)
 		throws LuanException, IOException, SQLException
 	{
-		if( boosts!=null && postgresBackup!=null )
-			throw new LuanException("boosts are not saved to postgres backup");
-
 		Object obj = doc.get("id");
 		Long id;
 		try {
@@ -289,11 +279,11 @@
 				doc.put("id",id);
 				if( postgresBackup != null )
 					postgresBackup.add(doc);
-				writer.addDocument(toLucene(completer,doc,boosts));
+				writer.addDocument(toLucene(doc));
 			} else {
 				if( postgresBackup != null )
 					postgresBackup.update(doc);
-				writer.updateDocument( term("id",id), toLucene(completer,doc,boosts) );
+				writer.updateDocument( "id", toLucene(doc) );
 			}
 			if(commit) writer.commit();
 		} finally {
@@ -368,10 +358,10 @@
 	}
 
 	private void saveNextId(long nextId) throws LuanException, IOException {
-		Map doc = new HashMap();
+		Map<String,Object> doc = new HashMap();
 		doc.put( "type", "next_id" );
 		doc.put( FLD_NEXT_ID, idLim );
-		writer.updateDocument(new Term("type","next_id"),toLucene(doc.entrySet(),null));
+		writer.updateDocument("type",doc);
 	}
 
 	public synchronized long nextId() throws LuanException, IOException {
@@ -404,10 +394,11 @@
 	}
 */
 	public SnapshotDeletionPolicy snapshotDeletionPolicy() {
-		return snapshotDeletionPolicy;
+		return (SnapshotDeletionPolicy)writer.getLuceneConfig().getIndexDeletionPolicy();
 	}
 
 	public Object snapshot(LuanFunction fn) throws LuanException, IOException {
+		SnapshotDeletionPolicy snapshotDeletionPolicy = snapshotDeletionPolicy();
 		IndexCommit ic = snapshotDeletionPolicy.snapshot();
 		try {
 			String dir = fsDir.getDirectory().toString();
@@ -587,136 +578,34 @@
 		mfp.fields.put( field, fp );
 	}
 
-
-	private IndexableField newField(String name,Object value,Set<String> indexed,Float boost)
-		throws LuanException
-	{
-		boolean hasBoost = boost!=null;
-		IndexableField fld = newField2(name,value,indexed,hasBoost);
-		if( hasBoost )
-			((Field)fld).setBoost(boost);
-		return fld;
-	}
-
-	private IndexableField newField2(String name,Object value,Set<String> indexed,boolean hasBoost)
-		throws LuanException
-	{
-		Field.Store store = indexOnly.contains(name) ? Field.Store.NO : Field.Store.YES;
-		if( value instanceof String ) {
-			String s = (String)value;
-			FieldParser fp = mfp.fields.get(name);
-			if( fp != null ) {
-				if( fp instanceof StringFieldParser && fp != STRING_FIELD_PARSER ) {
-					return new TextField(name, s, store);
-				} else if (hasBoost) {
-					// fuck you modern lucene developers
-					return new Field(name, s, store, Field.Index.NOT_ANALYZED);
-				} else {
-					return new StringField(name, s, store);
-				}
-			} else {
-				return new StoredField(name, s);
-			}
-		} else if( value instanceof Integer ) {
-			int i = (Integer)value;
-			if( indexed.contains(name) ) {
-				return new IntField(name, i, store);
-			} else {
-				return new StoredField(name, i);
-			}
-		} else if( value instanceof Long ) {
-			long i = (Long)value;
-			if( indexed.contains(name) ) {
-				return new LongField(name, i, store);
-			} else {
-				return new StoredField(name, i);
-			}
-		} else if( value instanceof Double ) {
-			double i = (Double)value;
-			if( indexed.contains(name) ) {
-				return new DoubleField(name, i, store);
-			} else {
-				return new StoredField(name, i);
-			}
-		} else if( value instanceof byte[] ) {
-			byte[] b = (byte[])value;
-			return new StoredField(name, b);
-		} else
-			throw new LuanException("invalid value type "+value.getClass()+"' for '"+name+"'");
-	}
-
-	private Document toLucene(LuanFunction completer,LuanTable table,LuanTable boosts) throws LuanException {
-		if( completer != null )
-			table = (LuanTable)completer.call(table);
-		return toLucene(table.iterable(),boosts);
-	}
-
-	private Document toLucene(Iterable<Map.Entry> iterable,LuanTable boosts) throws LuanException {
-		Set<String> indexed = mfp.fields.keySet();
-		Document doc = new Document();
-		for( Map.Entry<Object,Object> entry : iterable ) {
-			Object key = entry.getKey();
-			if( !(key instanceof String) )
-				throw new LuanException("key must be string");
-			String name = (String)key;
-			Object value = entry.getValue();
-			Float boost = null;
-			if( boosts != null ) {
-				Object obj = boosts.get(name);
-				if( obj != null ) {
-					if( !(obj instanceof Number) )
-						throw new LuanException("boost '"+name+"' must be number");
-					boost = ((Number)obj).floatValue();
-				}
-			}
-			if( !(value instanceof LuanTable) ) {
-				doc.add(newField( name, value, indexed, boost ));
-			} else { // list
+	static Map<String,Object> toLucene(LuanTable table) throws LuanException {
+		Map<String,Object> map = new LinkedHashMap<String,Object>();
+		for( Map.Entry<Object,Object> entry : table.iterable() ) {
+			String name = (String)entry.getKey();
+			Object value  = entry.getValue();
+			if( value instanceof LuanTable ) {
 				LuanTable list = (LuanTable)value;
 				if( !list.isList() )
 					throw new LuanException("table value for '"+name+"' must be a list");
-				for( Object el : list.asList() ) {
-					doc.add(newField( name, el, indexed, boost ));
-				}
+				value = list.asList();
 			}
+			map.put(name,value);
 		}
-		return doc;
-	}
-
-	private static Object getValue(IndexableField ifld) throws LuanException {
-		BytesRef br = ifld.binaryValue();
-		if( br != null )
-			return br.bytes;
-		Number n = ifld.numericValue();
-		if( n != null )
-			return n;
-		String s = ifld.stringValue();
-		if( s != null )
-			return s;
-		throw new LuanException("invalid field type for "+ifld);
+		return map;
 	}
 
 	private static LuanTable toTable(Luan luan,Document doc) throws LuanException {
-		if( doc==null )
-			return null;
+		return doc==null ? null : toTable(luan,LuceneUtils.toMap(doc));
+	}
+
+	static LuanTable toTable(Luan luan,Map map) throws LuanException {
 		LuanTable table = new LuanTable(luan);
-		for( IndexableField ifld : doc ) {
-			String name = ifld.name();
-			Object value = getValue(ifld);
-			Object old = table.rawGet(name);
-			if( old == null ) {
-				table.rawPut(name,value);
-			} else {
-				LuanTable list;
-				if( old instanceof LuanTable ) {
-					list = (LuanTable)old;
-				} else {
-					list = new LuanTable(luan);
-					list.rawPut(1,old);
-					table.rawPut(name,list);
-				}
-				list.rawPut(list.rawLength()+1,value);
-			}
+		for( Object obj : map.entrySet() ) {
+			Map.Entry entry = (Map.Entry)obj;
+			Object value = entry.getValue();
+			if( value instanceof List )
+				value = new LuanTable(luan,(List)value);
+			table.rawPut( entry.getKey(), value );
 		}
 		return table;
 	}
@@ -843,16 +732,16 @@
 		logger.info("end rebuild_postgres_backup");
 	}
 
-	public void restore_from_postgres(LuanFunction completer)
+	public void restore_from_postgres()
 		throws IOException, LuanException, SQLException, ParseException
 	{
 		if( postgresBackup!=null && wasCreated && !postgresBackup.wasCreated ) {
 			logger.error("restoring from postgres");
-			force_restore_from_postgres(completer);
+			force_restore_from_postgres();
 		}
 	}
 
-	public void force_restore_from_postgres(LuanFunction completer)
+	public void force_restore_from_postgres()
 		throws IOException, LuanException, SQLException, ParseException
 	{
 		logger.warn("start restore_from_postgres");
@@ -865,7 +754,7 @@
 		try {
 			writer.deleteAll();
 			long nextId = postgresBackup.maxId() + 1;
-			postgresBackup.restoreLucene(this,completer);
+			postgresBackup.restoreLucene(this);
 			id = idLim = nextId;
 			saveNextId(nextId);
 			ok = true;
@@ -882,10 +771,10 @@
 		logger.warn("end restore_from_postgres");
 	}
 
-	void restore(LuanFunction completer,LuanTable doc)
+	void restore(LuanTable doc)
 		throws LuanException, IOException
 	{
-		writer.addDocument(toLucene(completer,doc,null));
+		writer.addDocument(toLucene(doc));
 	}
 
 	public void check(Luan luan) throws IOException, SQLException, LuanException, ParseException {