Mercurial Hosting > luan
diff src/luan/modules/lucene/LuceneIndex.java @ 1391:94f48cc76de8
add lucene check
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Thu, 05 Sep 2019 01:29:57 -0600 |
parents | 179c4882c6b6 |
children | 002152af497a |
line wrap: on
line diff
--- a/src/luan/modules/lucene/LuceneIndex.java Wed Sep 04 00:06:42 2019 -0600 +++ b/src/luan/modules/lucene/LuceneIndex.java Thu Sep 05 01:29:57 2019 -0600 @@ -5,6 +5,7 @@ import java.io.FileOutputStream; import java.io.FileInputStream; import java.io.IOException; +import java.sql.SQLException; import java.util.Arrays; import java.util.Iterator; import java.util.Map; @@ -39,6 +40,7 @@ import org.apache.lucene.index.SnapshotDeletionPolicy; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.CheckIndex; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.Version; @@ -76,6 +78,7 @@ import luan.LuanFunction; import luan.LuanException; import luan.LuanRuntimeException; +import luan.modules.parsers.LuanToString; import luan.lib.logging.Logger; import luan.lib.logging.LoggerFactory; @@ -147,7 +150,7 @@ private final MultiFieldParser mfp; private final Analyzer analyzer; - private File fileDir; + private FSDirectory fsDir; private int writeCount; private AtomicInteger writeCounter = new AtomicInteger(); @@ -193,12 +196,11 @@ IndexWriterConfig conf = new IndexWriterConfig(version,analyzer); snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()); conf.setIndexDeletionPolicy(snapshotDeletionPolicy); - FSDirectory dir = FSDirectory.open(indexDir); - fileDir = dir.getDirectory(); - boolean wasCreated = !fileDir.exists(); - writer = new IndexWriter(dir,conf); + fsDir = FSDirectory.open(indexDir); + boolean wasCreated = !fsDir.getDirectory().exists(); + writer = new IndexWriter(fsDir,conf); writer.commit(); // commit index creation - reader = DirectoryReader.open(dir); + reader = DirectoryReader.open(fsDir); searcher = new IndexSearcher(reader); initId(); return wasCreated; @@ -235,14 +237,18 @@ if( postgresBackup != null ) { final List<Long> ids = new ArrayList<Long>(); IndexSearcher searcher = openSearcher(); - MyCollector col = new MyCollector() { - @Override public void collect(int iDoc) throws IOException { - Document doc = searcher.doc( docBase + iDoc ); - Long id = (Long)doc.getField("id").numericValue(); - ids.add(id); - } - }; - searcher.search(query,col); + try { + MyCollector col = new MyCollector() { + @Override public void collect(int iDoc) throws IOException { + Document doc = searcher.doc( docBase + iDoc ); + Long id = (Long)doc.getField("id").numericValue(); + ids.add(id); + } + }; + searcher.search(query,col); + } finally { + close(searcher); + } postgresBackup.begin(); for( Long id : ids ) { postgresBackup.delete(id); @@ -292,13 +298,13 @@ if( id == null ) { id = nextId(); doc.put("id",id); + if( postgresBackup != null ) + postgresBackup.add(doc); writer.addDocument(toLucene(doc,boosts)); - if( postgresBackup != null ) - postgresBackup.add(id,doc); } else { + if( postgresBackup != null ) + postgresBackup.update(doc); writer.updateDocument( term("id",id), toLucene(doc,boosts) ); - if( postgresBackup != null ) - postgresBackup.update(id,doc); } if(commit) writer.commit(); } finally { @@ -413,7 +419,7 @@ public Object snapshot(LuanFunction fn) throws LuanException, IOException { IndexCommit ic = snapshotDeletionPolicy.snapshot(); try { - String dir = fileDir.toString(); + String dir = fsDir.getDirectory().toString(); LuanTable fileNames = new LuanTable(fn.luan(),new ArrayList(ic.getFileNames())); return fn.call(dir,fileNames); } finally { @@ -803,21 +809,19 @@ throws IOException, LuanException { writeLock.lock(); + IndexSearcher searcher = openSearcher(); boolean ok = false; try { postgresBackup.begin(); postgresBackup.deleteAll(); Query query = new PrefixQuery(new Term("id")); - IndexSearcher searcher = openSearcher(); MyCollector col = new MyCollector() { @Override public void collect(int iDoc) throws IOException { try { Document doc = searcher.doc( docBase + iDoc ); LuanTable tbl = toTable(completer.luan(),doc); tbl = (LuanTable)completer.call(tbl); - Long id = (Long)tbl.get("id"); - //logger.info("id = "+id); - postgresBackup.add(id,tbl); + postgresBackup.add(tbl); } catch(LuanException e) { throw new LuanRuntimeException(e); } @@ -831,6 +835,7 @@ ok = true; postgresBackup.commit(); } finally { + close(searcher); if( !ok ) postgresBackup.rollback(); writeLock.unlock(); @@ -869,4 +874,87 @@ writer.addDocument(toLucene(doc,null)); } + public void check(LuanFunction completer) throws IOException, SQLException, LuanException { + logger.info("start check"); + CheckIndex.Status status = new CheckIndex(fsDir).checkIndex(); + if( !status.clean ) + logger.error("index not clean"); + if( postgresBackup != null ) + checkPostgres(completer); + logger.info("end check"); + } + + private void checkPostgres(LuanFunction completer) throws IOException, SQLException, LuanException { + final PostgresBackup.Checker postgresChecker; + final IndexSearcher searcher; + writeLock.lock(); + try { + postgresChecker = postgresBackup.newChecker(); + searcher = openSearcher(); + } finally { + writeLock.unlock(); + } + try { + final List<Long> idsLucene = new ArrayList<Long>(); + Query query = new PrefixQuery(new Term("id")); + MyCollector col = new MyCollector() { + @Override public void collect(int iDoc) throws IOException { + Document doc = searcher.doc( docBase + iDoc ); + Long id = (Long)doc.getField("id").numericValue(); + idsLucene.add(id); + } + }; + searcher.search(query,col); + Collections.sort(idsLucene); + final List<Long> idsPostgres = postgresChecker.getIds(); + final int nLucene = idsLucene.size(); + final int nPostgres = idsPostgres.size(); + int iLucene = 0; + int iPostgres = 0; + LuanToString lts = new LuanToString(); + lts.strict = true; + lts.numberTypes = true; + while( iLucene < nLucene && iPostgres < nPostgres ) { + long idLucene = idsLucene.get(iLucene); + long idPostgres = idsPostgres.get(iPostgres); + if( idLucene < idPostgres ) { + iLucene++; + logger.error("id "+idLucene+" found in lucene but not postgres"); + } else if( idLucene > idPostgres ) { + iPostgres++; + logger.error("id "+idPostgres+" found in postgres but not lucene"); + } else { // == + LuanTable docPostgres = postgresChecker.getDoc(idPostgres); + TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1); + if( td.totalHits != 1 ) throw new RuntimeException(); + Document doc = searcher.doc( td.scoreDocs[0].doc ); + LuanTable docLucene = toTable(completer.luan(),doc); + docLucene = (LuanTable)completer.call(docLucene); + if( !equal(docPostgres,docLucene) ) { + logger.error("id "+idLucene+" not equal"); + logger.error("lucene = "+lts.toString(docLucene)); + logger.error("postgres = "+lts.toString(docPostgres)); + } + iLucene++; + iPostgres++; + } + } + while( iLucene < nLucene ) { + long idLucene = idsLucene.get(iLucene++); + logger.error("id "+idLucene+" found in lucene but not postgres"); + } + while( iPostgres < nPostgres ) { + long idPostgres = idsPostgres.get(iPostgres++); + logger.error("id "+idPostgres+" found in postgres but not lucene"); + } + } finally { + close(searcher); + postgresChecker.close(); + } + } + + private boolean equal(LuanTable t1,LuanTable t2) throws LuanException { + return t1.asMap().equals(t2.asMap()); + } + }