Mercurial Hosting > luan

comparison src/goodjava/lucene/backup/BackupIndexWriter.java @ 1508:86c5e7000ecf

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lucene.backup checksum
author Franklin Schmidt <fschmidt@gmail.com>
date Sat, 16 May 2020 17:56:02 -0600
parents d80395468b4e
children 0ba144491a42
comparison
equal deleted inserted replaced
1507:c8f4867fd083 1508:86c5e7000ecf
10 import java.util.HashMap; 10 import java.util.HashMap;
11 import java.util.Arrays; 11 import java.util.Arrays;
12 import java.util.concurrent.Executors; 12 import java.util.concurrent.Executors;
13 import java.util.concurrent.ExecutorService; 13 import java.util.concurrent.ExecutorService;
14 import javax.net.ssl.SSLSocket; 14 import javax.net.ssl.SSLSocket;
15 import org.apache.lucene.search.SortField;
15 import goodjava.io.IoUtils; 16 import goodjava.io.IoUtils;
16 import goodjava.rpc.RpcClient; 17 import goodjava.rpc.RpcClient;
17 import goodjava.rpc.RpcCall; 18 import goodjava.rpc.RpcCall;
18 import goodjava.rpc.RpcResult; 19 import goodjava.rpc.RpcResult;
19 import goodjava.rpc.RpcException; 20 import goodjava.rpc.RpcException;
54 exec.execute(sync); 55 exec.execute(sync);
55 isSyncPending = true; 56 isSyncPending = true;
56 } 57 }
57 } 58 }
58 59
60 protected void doCheck(SortField sortField) throws IOException {
61 super.doCheck(sortField);
62 runSyncWithChecksum();
63 }
64
59 public void runSync() { 65 public void runSync() {
60 try { 66 try {
61 exec.submit(sync).get(); 67 exec.submit(sync).get();
62 } catch(Exception e) { 68 } catch(Exception e) {
63 throw new RuntimeException(e); 69 throw new RuntimeException(e);
64 } 70 }
65 } 71 }
66 72
73 public void runSyncWithChecksum() {
74 try {
75 exec.submit(syncWithChecksum).get();
76 } catch(Exception e) {
77 throw new RuntimeException(e);
78 }
79 }
80
67 private final Runnable sync = new Runnable() { 81 private final Runnable sync = new Runnable() {
68 public void run() { 82 public void run() {
69 try { 83 try {
70 sync(); 84 sync(false);
71 } catch(IOException e) { 85 } catch(IOException e) {
72 throw new RuntimeException(e); 86 throw new RuntimeException(e);
73 } 87 }
74 } 88 }
75 }; 89 };
76 90
77 private void sync() throws IOException { 91 private final Runnable syncWithChecksum = new Runnable() {
92 public void run() {
93 try {
94 sync(true);
95 } catch(IOException e) {
96 throw new RuntimeException(e);
97 }
98 }
99 };
100
101 private void sync(boolean withChecksum) throws IOException {
78 List<LogFile> logs = new ArrayList<LogFile>(); 102 List<LogFile> logs = new ArrayList<LogFile>();
79 synchronized(this) { 103 synchronized(this) {
80 isSyncPending = false; 104 isSyncPending = false;
81 clearDir(); 105 clearDir();
82 for( LogFile log : this.logs ) { 106 for( LogFile log : this.logs ) {
89 Map<String,LogFile> logMap = new HashMap<String,LogFile>(); 113 Map<String,LogFile> logMap = new HashMap<String,LogFile>();
90 for( LogFile log : logs ) { 114 for( LogFile log : logs ) {
91 Map fileInfo = new HashMap(); 115 Map fileInfo = new HashMap();
92 fileInfo.put("name",log.file.getName()); 116 fileInfo.put("name",log.file.getName());
93 fileInfo.put("end",log.end()); 117 fileInfo.put("end",log.end());
118 if( withChecksum )
119 fileInfo.put("checksum",log.checksum());
94 logInfo.add(fileInfo); 120 logInfo.add(fileInfo);
95 logMap.put(log.file.getName(),log); 121 logMap.put(log.file.getName(),log);
96 } 122 }
97 for( String backupDomain : backupDomains ) { 123 for( String backupDomain : backupDomains ) {
98 RpcClient rpc = rpcClient(backupDomain); 124 RpcClient rpc = rpcClient(backupDomain);
103 RpcResult result = rpc.read(); 129 RpcResult result = rpc.read();
104 logger.info(Arrays.asList(result.returnValues).toString()); 130 logger.info(Arrays.asList(result.returnValues).toString());
105 String status = (String)result.returnValues[0]; 131 String status = (String)result.returnValues[0];
106 if( status.equals("ok") ) { 132 if( status.equals("ok") ) {
107 break; 133 break;
108 } else if( status.equals("missing") ) { 134 } else if( status.equals("missing") || status.equals("bad_checksum") ) {
109 String fileName = (String)result.returnValues[1]; 135 String fileName = (String)result.returnValues[1];
136 if( status.equals("bad_checksum") )
137 logger.error("bad_checksum "+fileName);
110 LogFile log = logMap.get(fileName); 138 LogFile log = logMap.get(fileName);
111 long len = log.end() - 8; 139 long len = log.end() - 8;
112 InputStream in = log.input(); 140 InputStream in = log.input();
113 call = new RpcCall(in,len,"add",name,logInfo,fileName); 141 call = new RpcCall(in,len,"add",name,logInfo,fileName);
114 } else if( status.equals("incomplete") ) { 142 } else if( status.equals("incomplete") ) {