annotate src/goodjava/lucene/backup/BackupIndexWriter.java @ 1502:8a7b6b32c691

minor threads
author Franklin Schmidt <fschmidt@gmail.com>
date Sun, 10 May 2020 22:48:15 -0600
parents e66e3d50b289
children f443542d8650
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
1 package goodjava.lucene.backup;
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
2
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
3 import java.io.File;
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
4 import java.io.InputStream;
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
5 import java.io.IOException;
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
6 import java.net.Socket;
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
7 import java.util.List;
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
8 import java.util.ArrayList;
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
9 import java.util.Map;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
10 import java.util.HashMap;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
11 import java.util.Arrays;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
12 import javax.net.ssl.SSLSocketFactory;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
13 import javax.net.ssl.SSLSocket;
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
14 import goodjava.io.IoUtils;
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
15 import goodjava.rpc.RpcClient;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
16 import goodjava.rpc.RpcCall;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
17 import goodjava.rpc.RpcResult;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
18 import goodjava.rpc.RpcException;
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
19 import goodjava.lucene.api.LuceneIndexWriter;
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
20 import goodjava.logging.Logger;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
21 import goodjava.logging.LoggerFactory;
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
22 import goodjava.lucene.logging.LoggingIndexWriter;
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
23 import goodjava.lucene.logging.LogFile;
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
24
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
25
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
26 public class BackupIndexWriter extends LoggingIndexWriter {
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
27 private static final Logger logger = LoggerFactory.getLogger(BackupIndexWriter.class);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
28 public static String[] backupDomains;
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
29 private final String name;
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
30 private final File dir;
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
31 private boolean isSyncPending = false;
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
32
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
33 public BackupIndexWriter(LuceneIndexWriter indexWriter,File logDir,String name) throws IOException {
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
34 super(indexWriter,logDir);
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
35 if( backupDomains == null )
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
36 throw new RuntimeException("must set backupDomains");
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
37 this.name = name;
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
38 File f = new File(System.getProperty("java.io.tmpdir"));
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
39 dir = new File(f,"goodjava.lucene/"+name);
1501
Franklin Schmidt <fschmidt@gmail.com>
parents: 1500
diff changeset
40 IoUtils.mkdirs(dir);
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
41 }
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
42
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
43 public synchronized void commit() throws IOException {
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
44 super.commit();
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
45 //sync();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
46 if( !isSyncPending ) {
1502
8a7b6b32c691 minor threads
Franklin Schmidt <fschmidt@gmail.com>
parents: 1501
diff changeset
47 threadPool.execute(sync);
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
48 isSyncPending = true;
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
49 }
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
50 }
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
51
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
52 public void runSync() {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
53 sync.run();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
54 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
55
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
56 private final Runnable sync = new Runnable() {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
57 public synchronized void run() {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
58 try {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
59 sync();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
60 } catch(IOException e) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
61 throw new RuntimeException(e);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
62 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
63 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
64 };
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
65
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
66 private void sync() throws IOException {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
67 List<LogFile> logs = new ArrayList<LogFile>();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
68 synchronized(this) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
69 isSyncPending = false;
1500
Franklin Schmidt <fschmidt@gmail.com>
parents: 1499
diff changeset
70 clearDir();
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
71 for( LogFile log : this.logs ) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
72 File f = new File(dir,log.file.getName());
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
73 IoUtils.link(log.file,f);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
74 logs.add( new LogFile(f) );
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
75 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
76 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
77 List logInfo = new ArrayList();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
78 Map<String,LogFile> logMap = new HashMap<String,LogFile>();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
79 for( LogFile log : logs ) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
80 Map fileInfo = new HashMap();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
81 fileInfo.put("name",log.file.getName());
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
82 fileInfo.put("end",log.end());
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
83 logInfo.add(fileInfo);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
84 logMap.put(log.file.getName(),log);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
85 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
86 for( String backupDomain : backupDomains ) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
87 RpcClient rpc = rpcClient(backupDomain);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
88 RpcCall call = new RpcCall("check",name,logInfo);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
89 try {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
90 while(true) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
91 rpc.write(call);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
92 RpcResult result = rpc.read();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
93 logger.info(Arrays.asList(result.returnValues).toString());
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
94 String status = (String)result.returnValues[0];
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
95 if( status.equals("ok") ) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
96 break;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
97 } else if( status.equals("missing") ) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
98 String fileName = (String)result.returnValues[1];
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
99 LogFile log = logMap.get(fileName);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
100 long len = log.end() - 8;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
101 InputStream in = log.input();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
102 call = new RpcCall(in,len,"add",name,logInfo,fileName);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
103 } else if( status.equals("incomplete") ) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
104 String fileName = (String)result.returnValues[1];
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
105 long logEnd = (Long)result.returnValues[2];
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
106 LogFile log = logMap.get(fileName);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
107 long len = log.end() - logEnd;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
108 InputStream in = log.input();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
109 in.skip(logEnd-8);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
110 call = new RpcCall(in,len,"append",name,logInfo,fileName);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
111 } else
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
112 throw new RuntimeException("status "+status);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
113 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
114 } catch(RpcException e) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
115 logger.warn("",e);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
116 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
117 rpc.close();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
118 }
1500
Franklin Schmidt <fschmidt@gmail.com>
parents: 1499
diff changeset
119 clearDir();
Franklin Schmidt <fschmidt@gmail.com>
parents: 1499
diff changeset
120 }
Franklin Schmidt <fschmidt@gmail.com>
parents: 1499
diff changeset
121
Franklin Schmidt <fschmidt@gmail.com>
parents: 1499
diff changeset
122 private void clearDir() throws IOException {
Franklin Schmidt <fschmidt@gmail.com>
parents: 1499
diff changeset
123 for( File f : dir.listFiles() ) {
Franklin Schmidt <fschmidt@gmail.com>
parents: 1499
diff changeset
124 IoUtils.delete(f);
Franklin Schmidt <fschmidt@gmail.com>
parents: 1499
diff changeset
125 }
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
126 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
127
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
128 static RpcClient rpcClient(String backupDomain) throws IOException {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
129 Socket socket;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
130 if( BackupServer.cipherSuites == null ) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
131 socket = new Socket(backupDomain,BackupServer.port);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
132 } else {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
133 socket = SSLSocketFactory.getDefault().createSocket(backupDomain,BackupServer.port);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
134 ((SSLSocket)socket).setEnabledCipherSuites(BackupServer.cipherSuites);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
135 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
136 return new RpcClient(socket);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
137 }
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
138 }