Mercurial Hosting > luan
diff src/luan/modules/lucene/LuceneIndex.java @ 1345:6f8988830098
unique LuceneIndex per dir
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Mon, 25 Feb 2019 11:00:10 -0700 |
parents | dc2af9d5463b |
children | efd1c6380f2c |
line wrap: on
line diff
--- a/src/luan/modules/lucene/LuceneIndex.java Mon Feb 25 07:00:55 2019 -0700 +++ b/src/luan/modules/lucene/LuceneIndex.java Mon Feb 25 11:00:10 2019 -0700 @@ -5,15 +5,15 @@ import java.io.FileOutputStream; import java.io.FileInputStream; import java.io.IOException; +import java.util.Arrays; import java.util.Iterator; import java.util.Map; +import java.util.HashMap; import java.util.List; import java.util.ArrayList; import java.util.Set; import java.util.HashSet; import java.util.Collections; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -22,6 +22,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StoredField; @@ -78,13 +79,63 @@ import luan.lib.logging.LoggerFactory; -public final class LuceneIndex implements Closeable { +public final class LuceneIndex { private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class); + private static final class Closer implements Closeable { + final LuceneIndex li; + boolean isClosed = false; + private final Exception created = new Exception("created"); + + Closer(Luan luan,LuceneIndex li) { + this.li = li; + luan.onClose(this); + } + + public void close() throws IOException { + if( !isClosed ) { + li.close(); + isClosed = true; + } + } + + protected void finalize() throws Throwable { + if( !isClosed ) { + logger.error("not closed",created); + close(); + } + super.finalize(); + } + } + + private static Map<String,LuceneIndex> indexes = new HashMap<String,LuceneIndex>(); + + public static Object[] getLuceneIndex(Luan luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields) + throws LuanException, IOException + { + String key = new File(indexDirStr).getCanonicalPath(); + synchronized(indexes) { + LuceneIndex li = indexes.get(key); + if( li == null ) { + li = new LuceneIndex(indexDirStr,defaultFieldParser,defaultFields,key); + li.openCount = 1; + indexes.put(key,li); + } else { + if( defaultFieldParser != li.defaultFieldParser ) + throw new LuanException("default_type doesn't match previous use"); + if( !Arrays.equals(defaultFields,li.defaultFields) ) + throw new LuanException("default_fields don't match previous use"); + li.openCount++; + } + return new Object[]{li,new Closer(luan,li)}; + } + } + + private static final Version version = Version.LUCENE_4_9; private static final String FLD_NEXT_ID = "nextId"; public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer()); + public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(version)); - private static final Version version = Version.LUCENE_4_9; private final ReentrantLock writeLock = new ReentrantLock(); private final File indexDir; private SnapshotDeletionPolicy snapshotDeletionPolicy; @@ -92,20 +143,26 @@ private DirectoryReader reader; private IndexSearcher searcher; private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>(); - private boolean isClosed = true; private final MultiFieldParser mfp; private final Analyzer analyzer; - private final Exception created = new Exception("created"); - private static ConcurrentMap<File,AtomicInteger> globalWriteCounters = new ConcurrentHashMap<File,AtomicInteger>(); private File fileDir; private int writeCount; + private AtomicInteger writeCounter = new AtomicInteger(); private Set<String> indexOnly = new HashSet<String>(); - public LuceneIndex(Luan luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields) + private int openCount; + private final String key; + private final FieldParser defaultFieldParser; + private final String[] defaultFields; + + private LuceneIndex(String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields,String key) throws LuanException, IOException { + this.key = key; + this.defaultFieldParser = defaultFieldParser; + this.defaultFields = defaultFields; mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields); mfp.fields.put( "type", STRING_FIELD_PARSER ); mfp.fields.put( "id", NumberFieldParser.LONG ); @@ -117,19 +174,15 @@ analyzer = sfp.analyzer; } this.analyzer = analyzer; - luan.onClose(this); reopen(); } public void reopen() throws LuanException, IOException { - if( !isClosed ) throw new RuntimeException(); - isClosed = false; IndexWriterConfig conf = new IndexWriterConfig(version,analyzer); snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()); conf.setIndexDeletionPolicy(snapshotDeletionPolicy); FSDirectory dir = FSDirectory.open(indexDir); fileDir = dir.getDirectory(); - globalWriteCounters.putIfAbsent(fileDir,new AtomicInteger()); writer = new IndexWriter(dir,conf); writer.commit(); // commit index creation reader = DirectoryReader.open(dir); @@ -137,12 +190,8 @@ initId(); } - private int globalWriteCount() { - return globalWriteCounters.get(fileDir).get(); - } - private void wrote() { - globalWriteCounters.get(fileDir).incrementAndGet(); + writeCounter.incrementAndGet(); } public void delete_all() throws IOException { @@ -312,24 +361,23 @@ return writer.getDirectory().toString(); } - public void close() throws IOException { - if( !isClosed ) { - writer.close(); - reader.close(); - isClosed = true; + private synchronized void close() throws IOException { + if( openCount > 0 ) { + if( --openCount == 0 ) { + doClose(); + synchronized(indexes) { + indexes.remove(key); + } + } } } - protected void finalize() throws Throwable { - if( !isClosed ) { - logger.error("not closed",created); - close(); - } - super.finalize(); + public void doClose() throws IOException { + writer.close(); + reader.close(); } - private static class DocFn extends LuanFunction { final IndexSearcher searcher; final Query query; @@ -370,7 +418,7 @@ } private synchronized IndexSearcher openSearcher() throws IOException { - int gwc = globalWriteCount(); + int gwc = writeCounter.get(); if( writeCount != gwc ) { writeCount = gwc; DirectoryReader newReader = DirectoryReader.openIfChanged(reader);