diff src/goodjava/lucene/logging/LoggingIndexWriter.java @ 1544:35601f15ecc3

add lucene log tag and restore_from_log
author Franklin Schmidt <fschmidt@gmail.com>
date Sun, 20 Sep 2020 20:36:55 -0600
parents c27dc6af87ca
children 736ec76bbf42
line wrap: on
line diff
--- a/src/goodjava/lucene/logging/LoggingIndexWriter.java	Sun Sep 20 16:45:31 2020 -0600
+++ b/src/goodjava/lucene/logging/LoggingIndexWriter.java	Sun Sep 20 20:36:55 2020 -0600
@@ -44,9 +44,11 @@
 	private static final int OP_DELETE_DOCUMENTS = 2;
 	private static final int OP_ADD_DOCUMENT = 3;
 	private static final int OP_UPDATE_DOCUMENT = 4;
+	private static final int OP_TAG = 5;
 	private static final Random rnd = new Random();
 
 	public final LuceneIndexWriter indexWriter;
+	public boolean wasCreated;
 	private final File logDir;
 	protected final List<LogFile> logs = new ArrayList<LogFile>();
 	private LogOutputStream log;
@@ -71,6 +73,7 @@
 					}
 					deleteUnusedFiles();
 					setLog();
+					wasCreated = false;
 					return;
 				}
 			} finally {
@@ -78,6 +81,7 @@
 			}
 		}
 		newLogs();
+		wasCreated = true;
 	}
 
 	public IndexWriter getLuceneIndexWriter() {
@@ -201,8 +205,8 @@
 			throw new RuntimeException();
 		Directory dir = FSDirectory.open(dirFile);
 		LuceneIndexWriter mergeWriter = new LuceneIndexWriter( dir, indexWriter.goodConfig );
-		playLog( first.input(), mergeWriter );
-		playLog( second.input(), mergeWriter );
+		playLog( first.input(), mergeWriter, null );
+		playLog( second.input(), mergeWriter, null );
 		mergeWriter.commit();
 		LogFile merge = newLogFile();
 		logLucene( lastTime, merge, mergeWriter );
@@ -273,18 +277,17 @@
 			IoUtils.deleteRecursively(dirFile);
 			Directory dir = FSDirectory.open(dirFile);
 			LuceneIndexWriter checkWriter = new LuceneIndexWriter( dir, indexWriter.goodConfig );
-			playLogs(logReaders,checkWriter);
+			playLogs(logReaders,checkWriter,null);
 			//logger.info("check lucene");
 			IndexReader checkReader = checkWriter.openReader();
+			int nCheck = checkReader.numDocs();
+			int nOrig = indexReader.numDocs();
+			if( nCheck != nOrig ) {
+				logger.error("numDocs mismatch: lucene="+nOrig+" logs="+nCheck);
+				ok = false;
+			}
 			if( sortField == null ) {
-				int nCheck = checkReader.numDocs();
-				int nOrig = indexReader.numDocs();
-				if( nCheck != nOrig ) {
-					logger.error("numDocs mismatch: lucene="+nOrig+" logs="+nCheck);
-					ok = false;
-				}
-				//logger.info("numDocs="+nOrig);
-				if( hash(indexReader) != hash(checkReader) ) {
+				if( ok && hash(indexReader) != hash(checkReader) ) {
 					logger.error("hash mismatch");
 					ok = false;
 				}
@@ -413,6 +416,11 @@
 		log.writeMap(storedFields);
 	}
 
+	public synchronized void tag(String tag) throws IOException {
+		writeOp(OP_TAG);
+		log.writeUTF(tag);
+	}
+
 	public synchronized void reindexDocuments(String keyFieldName,Query query) throws IOException {
 		indexWriter.reindexDocuments(keyFieldName,query);
 	}
@@ -422,8 +430,9 @@
 		log.writeByte(op);
 	}
 
-	public synchronized void playLogs() throws IOException {
-		playLogs( logReaders(logs), indexWriter );
+	// return whether stopped at tag
+	public synchronized boolean playLogs(String upToTag) throws IOException {
+		return playLogs( logReaders(logs), indexWriter, upToTag );
 	}
 
 	private static List<LogInputStream> logReaders(List<LogFile> logs) throws IOException {
@@ -434,13 +443,20 @@
 		return logReaders;
 	}
 
-	private static void playLogs(List<LogInputStream> logReaders,LuceneIndexWriter indexWriter) throws IOException {
+	private static boolean playLogs(List<LogInputStream> logReaders,LuceneIndexWriter indexWriter,String upToTag)
+		throws IOException
+	{
 		if( numDocs(indexWriter) != 0 )
 			throw new RuntimeException ("not empty");
+		boolean rtn = false;
 		for( LogInputStream reader : logReaders ) {
-			playLog(reader,indexWriter);
+			if( playLog(reader,indexWriter,upToTag) ) {
+				rtn = true;
+				break;
+			}
 		}
 		indexWriter.commit();
+		return rtn;
 	}
 
 	private static int numDocs(LuceneIndexWriter indexWriter) throws IOException {
@@ -450,35 +466,51 @@
 		return n;
 	}
 
-	private static void playLog(LogInputStream in,LuceneIndexWriter indexWriter) throws IOException {
+	private static boolean playLog(LogInputStream in,LuceneIndexWriter indexWriter,String upToTag)
+		throws IOException
+	{
+		boolean rtn = false;
 		while( in.available() > 0 ) {
-			playOp(in,indexWriter);
+			if( playOp(in,indexWriter,upToTag) ) {
+				rtn = true;
+				break;
+			}
 		}
 		in.close();
+		return rtn;
 	}
 
-	private static void playOp(LogInputStream in,LuceneIndexWriter indexWriter) throws IOException {
+	private static boolean playOp(LogInputStream in,LuceneIndexWriter indexWriter,String upToTag) throws IOException {
 		in.readLong();  // time
 		int op = in.readByte();
 		switch(op) {
 		case OP_DELETE_ALL:
 			indexWriter.deleteAll();
-			return;
+			return false;
 		case OP_DELETE_DOCUMENTS:
-			indexWriter.deleteDocuments( in.readQuery() );
-			return;
+			{
+				Query query = in.readQuery();
+				//System.out.println("OP_DELETE_DOCUMENTS "+query);
+				indexWriter.deleteDocuments(query);
+				return false;
+			}
 		case OP_ADD_DOCUMENT:
 			{
 				Map storedFields = in.readMap();
 				indexWriter.addDocument(storedFields);
-				return;
+				return false;
 			}
 		case OP_UPDATE_DOCUMENT:
 			{
 				String keyFieldName = in.readUTF();
 				Map storedFields = in.readMap();
 				indexWriter.updateDocument(keyFieldName,storedFields);
-				return;
+				return false;
+			}
+		case OP_TAG:
+			{
+				String tag = in.readUTF();
+				return tag.equals(upToTag);
 			}
 		default:
 			throw new RuntimeException("invalid op "+op);