Mercurial Hosting > luan

changeset 1509:0ba144491a42

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lucene.backup zip
author Franklin Schmidt <fschmidt@gmail.com>
date Sun, 17 May 2020 14:29:33 -0600
parents 86c5e7000ecf
children 20375c20289d
files src/goodjava/io/IoUtils.java src/goodjava/lucene/backup/Backup.java src/goodjava/lucene/backup/BackupIndexWriter.java src/goodjava/lucene/backup/BackupServer.java src/goodjava/rpc/Rpc.java src/goodjava/rpc/RpcResult.java src/luan/modules/IoLuan.java src/luan/modules/PackageLuan.java src/luan/modules/Rpc.luan src/luan/modules/Utils.java src/luan/modules/url/LuanUrl.java src/luan/modules/url/UrlCall.java
diffstat 12 files changed, 137 insertions(+), 141 deletions(-) [+]
line wrap: on
line diff
--- a/src/goodjava/io/IoUtils.java	Sat May 16 17:56:02 2020 -0600
+++ b/src/goodjava/io/IoUtils.java	Sun May 17 14:29:33 2020 -0600
@@ -3,14 +3,22 @@
 import java.io.File;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.io.Reader;
+import java.io.InputStreamReader;
+import java.io.Writer;
+import java.io.StringWriter;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.security.Security;
 import javax.net.ssl.SSLSocketFactory;
 import javax.net.ssl.SSLServerSocketFactory;
+import goodjava.logging.Logger;
+import goodjava.logging.LoggerFactory;
 
 
 public final class IoUtils {
+	private static final Logger logger = LoggerFactory.getLogger(IoUtils.class);
+
 	private IoUtils() {}  // never
 
 	public static void move( File from, File to ) throws IOException {
@@ -57,6 +65,25 @@
 		in.close();
 	}
 
+	public static void copyAll(Reader in,Writer out)
+		throws IOException
+	{
+		char[] a = new char[8192];
+		int n;
+		while( (n=in.read(a)) != -1 ) {
+			out.write(a,0,n);
+		}
+		in.close();
+	}
+
+	public static String readAll(Reader in)
+		throws IOException
+	{
+		StringWriter sw = new StringWriter();
+		copyAll(in,sw);
+		return sw.toString();
+	}
+
 	public static long checksum(InputStream in) throws IOException {
 		long cs = 0;
 		int c;
@@ -68,6 +95,32 @@
 	}
 
 
+
+	public static class ProcException extends IOException {
+		private ProcException(String msg) {
+			super(msg);
+		}
+	}
+
+	public static void waitFor(Process proc)
+		throws IOException, ProcException
+	{
+		try {
+			proc.waitFor();
+		} catch(InterruptedException e) {
+			throw new RuntimeException(e);
+		}
+		int exitVal = proc.exitValue();
+		if( exitVal != 0 ) {
+			StringWriter sw = new StringWriter();
+			copyAll( new InputStreamReader(proc.getInputStream()), sw );
+			copyAll( new InputStreamReader(proc.getErrorStream()), sw );
+			String error = sw.toString();
+			throw new ProcException(error);
+		}
+	}
+
+
 	static {
 		// undo restrictions of modern scum
 		Security.setProperty("jdk.tls.disabledAlgorithms","SSLv3, RC4, DES, MD5withRSA, DH keySize < 1024, EC keySize < 224, 3DES_EDE_CBC");
--- a/src/goodjava/lucene/backup/Backup.java	Sat May 16 17:56:02 2020 -0600
+++ b/src/goodjava/lucene/backup/Backup.java	Sun May 17 14:29:33 2020 -0600
@@ -1,12 +1,15 @@
 package goodjava.lucene.backup;
 
 import java.io.File;
+import java.io.InputStream;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.util.List;
 import java.util.ArrayList;
 import java.util.Map;
 import java.util.Arrays;
 import goodjava.io.IoUtils;
+import goodjava.io.BufferedInputStream;
 import goodjava.rpc.RpcServer;
 import goodjava.rpc.RpcCall;
 import goodjava.rpc.RpcResult;
@@ -28,16 +31,22 @@
 		this.index = new File(dir,"index");
 	}
 
-	synchronized void handle(RpcServer rpc,RpcCall call) {
+	void handle(RpcServer rpc,RpcCall call) {
 		try {
-			handle2(rpc,call);
+			IoUtils.mkdirs(dir);
+			if( call.cmd.equals("zip") ) {
+				handleZip(rpc);
+			} else {
+				handle2(rpc,call);
+			}
 		} catch(IOException e) {
 			throw new RuntimeException(e);
 		}
 	}
 
-	void handle2(RpcServer rpc,RpcCall call) throws IOException {
-		IoUtils.mkdirs(dir);
+	private static final RpcResult OK = new RpcResult(new Object[]{"ok"});
+
+	synchronized void handle2(RpcServer rpc,RpcCall call) throws IOException {
 		//logger.info(call.cmd+" "+Arrays.asList(call.args));
 		String fileName = null;
 		if( call.cmd.equals("check") ) {
@@ -57,14 +66,14 @@
 			throw new RuntimeException("cmd "+call.cmd);
 		List logInfo = (List)call.args[1];
 		logger.info("check "+logInfo);
-		RpcResult result = new RpcResult("ok");
+		RpcResult result = OK;
 		for( Object obj : logInfo ) {
 			Map fileInfo = (Map)obj;
 			String name = (String)fileInfo.get("name");
 			File f = new File(dir,name);
 			if( !f.exists() ) {
 				if( name.equals(fileName) )  logger.error("missing");
-				result = new RpcResult("missing",name);
+				result = new RpcResult(new Object[]{"missing",name});
 				break;
 			}
 			long end = (Long)fileInfo.get("end");
@@ -72,12 +81,12 @@
 			long logEnd = log.end();
 			if( logEnd > end ) {
 				logger.error("logEnd > end - shouldn't happen, file="+name+" logEnd="+logEnd+" end="+end);
-				result = new RpcResult("missing",name);
+				result = new RpcResult(new Object[]{"missing",name});
 				break;
 			}
 			if( logEnd < end ) {
 				if( name.equals(fileName) )  logger.error("incomplete");
-				result = new RpcResult("incomplete",name,logEnd);
+				result = new RpcResult(new Object[]{"incomplete",name,logEnd});
 				break;
 			}
 			Object checksumObj = fileInfo.get("checksum");
@@ -85,7 +94,7 @@
 				long checksum = (Long)checksumObj;
 				if( log.checksum() != checksum ) {
 					index.delete();
-					result = new RpcResult("bad_checksum",name);
+					result = new RpcResult(new Object[]{"bad_checksum",name});
 					break;
 				}
 			}
@@ -111,5 +120,19 @@
 		rpc.write(result);
 	}
 
+	void handleZip(RpcServer rpc) throws IOException {
+		File zip = File.createTempFile("luan_",".zip");
+		IoUtils.delete(zip);
+		String cmd = "zip -r " + zip + " " + dir.getName();
+logger.info("cmd = "+cmd);
+		synchronized(this) {
+			Process proc = Runtime.getRuntime().exec(cmd,null,dir.getParentFile());
+			IoUtils.waitFor(proc);
+		}
+		InputStream in = new BufferedInputStream(new FileInputStream(zip));
+		RpcResult result = new RpcResult(in,zip.length(),new Object[0]);
+		rpc.write(result);
+		IoUtils.delete(zip);
+	}
 
 }
--- a/src/goodjava/lucene/backup/BackupIndexWriter.java	Sat May 16 17:56:02 2020 -0600
+++ b/src/goodjava/lucene/backup/BackupIndexWriter.java	Sun May 17 14:29:33 2020 -0600
@@ -11,7 +11,6 @@
 import java.util.Arrays;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ExecutorService;
-import javax.net.ssl.SSLSocket;
 import org.apache.lucene.search.SortField;
 import goodjava.io.IoUtils;
 import goodjava.rpc.RpcClient;
@@ -121,7 +120,7 @@
 			logMap.put(log.file.getName(),log);
 		}
 		for( String backupDomain : backupDomains ) {
-			RpcClient rpc = rpcClient(backupDomain);
+			RpcClient rpc = BackupServer.rpcClient(backupDomain);
 			RpcCall call = new RpcCall("check",name,logInfo);
 			try {
 				while(true) {
@@ -164,14 +163,4 @@
 		}
 	}
 
-	static RpcClient rpcClient(String backupDomain) throws IOException {
-		Socket socket;
-		if( BackupServer.cipherSuites == null ) {
-			socket = new Socket(backupDomain,BackupServer.port);
-		} else {
-			socket = IoUtils.getSSLSocketFactory().createSocket(backupDomain,BackupServer.port);
-			((SSLSocket)socket).setEnabledCipherSuites(BackupServer.cipherSuites);
-		}
-		return new RpcClient(socket);
-	}
 }
--- a/src/goodjava/lucene/backup/BackupServer.java	Sat May 16 17:56:02 2020 -0600
+++ b/src/goodjava/lucene/backup/BackupServer.java	Sun May 17 14:29:33 2020 -0600
@@ -1,17 +1,24 @@
 package goodjava.lucene.backup;
 
 import java.io.File;
+import java.io.OutputStream;
+import java.io.FileOutputStream;
+import java.io.BufferedOutputStream;
 import java.io.IOException;
 import java.util.Map;
 import java.net.Socket;
 import java.net.ServerSocket;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ExecutorService;
+import javax.net.ssl.SSLSocket;
 import javax.net.ssl.SSLServerSocket;
 import goodjava.util.SoftCacheMap;
 import goodjava.io.IoUtils;
 import goodjava.rpc.RpcServer;
 import goodjava.rpc.RpcCall;
+import goodjava.rpc.RpcClient;
+import goodjava.rpc.RpcResult;
+import goodjava.rpc.RpcException;
 import goodjava.logging.Logger;
 import goodjava.logging.LoggerFactory;
 
@@ -85,4 +92,29 @@
 		}
 	}
 
+
+	// for client
+
+	public static RpcClient rpcClient(String backupDomain) throws IOException {
+		Socket socket;
+		if( BackupServer.cipherSuites == null ) {
+			socket = new Socket(backupDomain,BackupServer.port);
+		} else {
+			socket = IoUtils.getSSLSocketFactory().createSocket(backupDomain,BackupServer.port);
+			((SSLSocket)socket).setEnabledCipherSuites(BackupServer.cipherSuites);
+		}
+		return new RpcClient(socket);
+	}
+
+	public static void getBackup(String backupDomain,String name,File zip) throws IOException, RpcException {
+		RpcClient rpc = BackupServer.rpcClient(backupDomain);
+		RpcCall call = new RpcCall("zip",name);
+		rpc.write(call);
+		RpcResult result = rpc.read();
+		OutputStream out = new BufferedOutputStream(new FileOutputStream(zip));
+		IoUtils.copyAll(result.in,out);
+		out.close();
+		rpc.close();
+	}
+
 }
--- a/src/goodjava/rpc/Rpc.java	Sat May 16 17:56:02 2020 -0600
+++ b/src/goodjava/rpc/Rpc.java	Sun May 17 14:29:33 2020 -0600
@@ -7,7 +7,7 @@
 public final class Rpc {
 	private Rpc() {}  // never
 
-	public static final RpcResult OK = new RpcResult();
+	public static final RpcResult OK = new RpcResult(new Object[0]);
 
 	public static final RpcCall CLOSE = new RpcCall("close");
 	public static final RpcCall PING = new RpcCall("ping");
--- a/src/goodjava/rpc/RpcResult.java	Sat May 16 17:56:02 2020 -0600
+++ b/src/goodjava/rpc/RpcResult.java	Sun May 17 14:29:33 2020 -0600
@@ -8,11 +8,11 @@
 	public final long lenIn;
 	public final Object[] returnValues;
 
-	public RpcResult(Object... returnValues) {
+	public RpcResult(Object[] returnValues) {
 		this(null,-1L,returnValues);
 	}
 
-	public RpcResult(InputStream in,long lenIn,Object... returnValues) {
+	public RpcResult(InputStream in,long lenIn,Object[] returnValues) {
 		this.in = in;
 		this.lenIn = lenIn;
 		this.returnValues = returnValues;
--- a/src/luan/modules/IoLuan.java	Sat May 16 17:56:02 2020 -0600
+++ b/src/luan/modules/IoLuan.java	Sun May 17 14:29:33 2020 -0600
@@ -170,7 +170,7 @@
 
 		public String read_text() throws IOException, LuanException {
 			Reader in = reader();
-			String s = Utils.readAll(in);
+			String s = IoUtils.readAll(in);
 			return s;
 		}
 
@@ -232,7 +232,7 @@
 		}
 
 		@Override public String read_text() throws IOException {
-			return Utils.readAll(new InputStreamReader(System.in));
+			return IoUtils.readAll(new InputStreamReader(System.in));
 		}
 
 		@Override public byte[] read_binary() throws IOException {
@@ -567,20 +567,9 @@
 		}
 
 		public void wait_for()
-			throws IOException, LuanException
+			throws IOException
 		{
-			try {
-				proc.waitFor();
-			} catch(InterruptedException e) {
-				throw new RuntimeException(e);
-			}
-			int exitVal = proc.exitValue();
-			if( exitVal != 0 ) {
-				Reader err = new InputStreamReader(proc.getErrorStream());
-				String error = "exit value "+exitVal+" in: "+cmd+"\n"+Utils.readAll(err);
-				err.close();
-				throw new LuanException(error);
-			}
+			IoUtils.waitFor(proc);
 		}
 
 		@Override public String read_text() throws IOException, LuanException {
--- a/src/luan/modules/PackageLuan.java	Sat May 16 17:56:02 2020 -0600
+++ b/src/luan/modules/PackageLuan.java	Sun May 17 14:29:33 2020 -0600
@@ -5,6 +5,7 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
+import goodjava.io.IoUtils;
 import luan.Luan;
 import luan.LuanTable;
 import luan.LuanFunction;
@@ -56,7 +57,7 @@
 				String src;
 				try {
 					Reader in = new InputStreamReader(ClassLoader.getSystemResourceAsStream("luan/modules/Boot.luan"));
-					src = Utils.readAll(in);
+					src = IoUtils.readAll(in);
 					in.close();
 				} catch(IOException e) {
 					throw new RuntimeException(e);
--- a/src/luan/modules/Rpc.luan	Sat May 16 17:56:02 2020 -0600
+++ b/src/luan/modules/Rpc.luan	Sun May 17 14:29:33 2020 -0600
@@ -47,11 +47,15 @@
 	"SSL_DH_anon_EXPORT_WITH_RC4_40_MD5"
 }
 
-local function java_args(list)
+local function java_array(list)
 	for i,v in ipairs(list) do
 		list[i] = LuanJava.toJava(v)
 	end
-	return unpack(list)
+	return list
+end
+
+local function java_args(list)
+	return unpack(java_array(list))
 end
 
 local function luan_args(list,binary_in)
@@ -163,9 +167,9 @@
 		local binary_in, len_in = encode_binary(rtn)
 		local result
 		if binary_in == nil then
-			result = RpcResult.new(java_args(rtn))
+			result = RpcResult.new(java_array(rtn))
 		else
-			result = RpcResult.new(binary_in,len_in,java_args(rtn))
+			result = RpcResult.new(binary_in,len_in,java_array(rtn))
 		end
 		server.write(result)
 	end
--- a/src/luan/modules/Utils.java	Sat May 16 17:56:02 2020 -0600
+++ b/src/luan/modules/Utils.java	Sun May 17 14:29:33 2020 -0600
@@ -67,18 +67,6 @@
 		checkNotNull(fn,1);
 	}
 
-	public static String readAll(Reader in)
-		throws IOException
-	{
-		char[] a = new char[bufSize];
-		StringBuilder buf = new StringBuilder();
-		int n;
-		while( (n=in.read(a)) != -1 ) {
-			buf.append(a,0,n);
-		}
-		return buf.toString();
-	}
-
 	public static byte[] readAll(InputStream in)
 		throws IOException
 	{
--- a/src/luan/modules/url/LuanUrl.java	Sat May 16 17:56:02 2020 -0600
+++ b/src/luan/modules/url/LuanUrl.java	Sun May 17 14:29:33 2020 -0600
@@ -17,6 +17,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Base64;
+import goodjava.io.IoUtils;
 import goodjava.parser.ParseException;
 import luan.Luan;
 import luan.LuanTable;
@@ -322,7 +323,7 @@
 			if( is == null )
 				throw e;
 			Reader in = new InputStreamReader(is);
-			String msg = Utils.readAll(in);
+			String msg = IoUtils.readAll(in);
 			in.close();
 			LuanException le = new LuanException(msg,e);
 			le.put("response_code",responseCode);
--- a/src/luan/modules/url/UrlCall.java	Sat May 16 17:56:02 2020 -0600
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,84 +0,0 @@
-// not used, just for reference
-
-package luan.modules.url;
-
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.io.Reader;
-import java.io.IOException;
-import java.net.URLConnection;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.Map;
-import java.util.HashMap;
-import luan.modules.Utils;
-
-
-public final class UrlCall {
-	public final URLConnection connection;
-
-	public UrlCall(String url) throws IOException {
-		this(new URL(url));
-	}
-
-	public UrlCall(URL url) throws IOException {
-		connection = url.openConnection();
-	}
-
-	public void acceptJson() {
-		connection.setRequestProperty("accept","application/json");
-	}
-
-	public String get() throws IOException {
-		Reader in = new InputStreamReader(connection.getInputStream());
-		String rtn = Utils.readAll(in);
-		in.close();
-		return rtn;
-	}
-
-	public String post(String content,String contentType) throws IOException {
-		HttpURLConnection connection = (HttpURLConnection)this.connection;
-
-		connection.setRequestProperty("content-type",contentType);
-		connection.setDoOutput(true);
-		connection.setRequestMethod("POST");
-
-		byte[] post = content.getBytes();
-		connection.setRequestProperty("Content-Length",Integer.toString(post.length));
-		OutputStream out = connection.getOutputStream();
-		out.write(post);
-		out.flush();
-
-		Reader in;
-		try {
-			in = new InputStreamReader(connection.getInputStream());
-		} catch(IOException e) {
-			InputStream is = connection.getErrorStream();
-			if( is == null )
-				throw e;
-			in = new InputStreamReader(is);
-			String msg = Utils.readAll(in);
-			in.close();
-			throw new UrlCallException(msg,e);
-		}
-		String rtn = Utils.readAll(in);
-		in.close();
-		out.close();
-		return rtn;
-	}
-
-	public String post(String content) throws IOException {
-		return post(content,"application/x-www-form-urlencoded");
-	}
-
-	public String postJson(String content) throws IOException {
-		return post(content,"application/json");
-	}
-
-	public static final class UrlCallException extends IOException {
-		UrlCallException(String msg,IOException e) {
-			super(msg,e);
-		}
-	}
-}