Mercurial Hosting > luan

comparison src/luan/modules/lucene/LuceneIndex.java @ 1454:219f2b937f2b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
remove log4j
author Franklin Schmidt <fschmidt@gmail.com>
date Sun, 08 Mar 2020 14:11:30 -0600
parents 928be2a4d565
children 6b6c11c9164e
comparison
equal deleted inserted replaced
1453:928be2a4d565 1454:219f2b937f2b
84 import goodjava.logging.Logger; 84 import goodjava.logging.Logger;
85 import goodjava.logging.LoggerFactory; 85 import goodjava.logging.LoggerFactory;
86 86
87 87
88 public final class LuceneIndex { 88 public final class LuceneIndex {
89 private static final Logger sysLogger = LoggerFactory.getLogger(LuceneIndex.class); 89 private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class);
90 90
91 private static Map<String,Reference<LuceneIndex>> indexes = new HashMap<String,Reference<LuceneIndex>>(); 91 private static Map<String,Reference<LuceneIndex>> indexes = new HashMap<String,Reference<LuceneIndex>>();
92 92
93 public static LuceneIndex getLuceneIndex(Luan luan,File indexDir,LuanTable options) 93 public static LuceneIndex getLuceneIndex(Luan luan,File indexDir,LuanTable options)
94 throws LuanException, IOException, ClassNotFoundException, SQLException 94 throws LuanException, IOException, ClassNotFoundException, SQLException
114 private static final Version luceneVersion = Version.LUCENE_4_9; 114 private static final Version luceneVersion = Version.LUCENE_4_9;
115 private static final String FLD_NEXT_ID = "nextId"; 115 private static final String FLD_NEXT_ID = "nextId";
116 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer()); 116 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer());
117 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(luceneVersion)); 117 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(luceneVersion));
118 118
119 private final Logger luanLogger;
120 private final Object version; 119 private final Object version;
121 120
122 private final ReentrantLock writeLock = new ReentrantLock(); 121 private final ReentrantLock writeLock = new ReentrantLock();
123 private final File indexDir; 122 private final File indexDir;
124 private SnapshotDeletionPolicy snapshotDeletionPolicy; 123 private SnapshotDeletionPolicy snapshotDeletionPolicy;
149 LuanTable defaultFieldsTbl = Utils.removeTable(options,"default_fields"); 148 LuanTable defaultFieldsTbl = Utils.removeTable(options,"default_fields");
150 String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]); 149 String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]);
151 LuanTable postgresSpec = Utils.removeTable(options,"postgres_spec"); 150 LuanTable postgresSpec = Utils.removeTable(options,"postgres_spec");
152 Utils.checkEmpty(options); 151 Utils.checkEmpty(options);
153 152
154 this.luanLogger = luan.getLogger(LuceneIndex.class);
155 // this.defaultFieldParser = defaultFieldParser; 153 // this.defaultFieldParser = defaultFieldParser;
156 // this.defaultFields = defaultFields; 154 // this.defaultFields = defaultFields;
157 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields); 155 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
158 mfp.fields.put( "type", STRING_FIELD_PARSER ); 156 mfp.fields.put( "type", STRING_FIELD_PARSER );
159 mfp.fields.put( "id", NumberFieldParser.LONG ); 157 mfp.fields.put( "id", NumberFieldParser.LONG );
168 if( postgresSpec == null ) { 166 if( postgresSpec == null ) {
169 postgresBackup = null; 167 postgresBackup = null;
170 } else { 168 } else {
171 postgresBackup = new PostgresBackup(luan,postgresSpec); 169 postgresBackup = new PostgresBackup(luan,postgresSpec);
172 if( !wasCreated && postgresBackup.wasCreated ) { 170 if( !wasCreated && postgresBackup.wasCreated ) {
173 luanLogger.error("rebuilding postgres backup"); 171 logger.error("rebuilding postgres backup");
174 rebuild_postgres_backup(luan); 172 rebuild_postgres_backup(luan);
175 /* 173 /*
176 } else if( wasCreated && !postgresBackup.wasCreated ) { 174 } else if( wasCreated && !postgresBackup.wasCreated ) {
177 luanLogger.error("restoring from postgres"); 175 logger.error("restoring from postgres");
178 restore_from_postgres(); 176 restore_from_postgres();
179 */ 177 */
180 } 178 }
181 } 179 }
182 } 180 }
804 } 802 }
805 803
806 public void rebuild_postgres_backup(Luan luan) 804 public void rebuild_postgres_backup(Luan luan)
807 throws IOException, LuanException, SQLException 805 throws IOException, LuanException, SQLException
808 { 806 {
809 luanLogger.info("start rebuild_postgres_backup"); 807 logger.info("start rebuild_postgres_backup");
810 writeLock.lock(); 808 writeLock.lock();
811 IndexSearcher searcher = openSearcher(); 809 IndexSearcher searcher = openSearcher();
812 boolean ok = false; 810 boolean ok = false;
813 try { 811 try {
814 postgresBackup.begin(); 812 postgresBackup.begin();
838 close(searcher); 836 close(searcher);
839 if( !ok ) 837 if( !ok )
840 postgresBackup.rollback(); 838 postgresBackup.rollback();
841 writeLock.unlock(); 839 writeLock.unlock();
842 } 840 }
843 luanLogger.info("end rebuild_postgres_backup"); 841 logger.info("end rebuild_postgres_backup");
844 } 842 }
845 843
846 public void restore_from_postgres(LuanFunction completer) 844 public void restore_from_postgres(LuanFunction completer)
847 throws IOException, LuanException, SQLException, ParseException 845 throws IOException, LuanException, SQLException, ParseException
848 { 846 {
849 if( postgresBackup!=null && wasCreated && !postgresBackup.wasCreated ) { 847 if( postgresBackup!=null && wasCreated && !postgresBackup.wasCreated ) {
850 luanLogger.error("restoring from postgres"); 848 logger.error("restoring from postgres");
851 force_restore_from_postgres(completer); 849 force_restore_from_postgres(completer);
852 } 850 }
853 } 851 }
854 852
855 public void force_restore_from_postgres(LuanFunction completer) 853 public void force_restore_from_postgres(LuanFunction completer)
856 throws IOException, LuanException, SQLException, ParseException 854 throws IOException, LuanException, SQLException, ParseException
857 { 855 {
858 luanLogger.warn("start restore_from_postgres"); 856 logger.warn("start restore_from_postgres");
859 if( postgresBackup==null ) 857 if( postgresBackup==null )
860 throw new NullPointerException(); 858 throw new NullPointerException();
861 if( writeLock.isHeldByCurrentThread() ) 859 if( writeLock.isHeldByCurrentThread() )
862 throw new RuntimeException(); 860 throw new RuntimeException();
863 writeLock.lock(); 861 writeLock.lock();
877 reopen(); 875 reopen();
878 } 876 }
879 wrote(); 877 wrote();
880 writeLock.unlock(); 878 writeLock.unlock();
881 } 879 }
882 luanLogger.warn("end restore_from_postgres"); 880 logger.warn("end restore_from_postgres");
883 } 881 }
884 882
885 void restore(LuanFunction completer,LuanTable doc) 883 void restore(LuanFunction completer,LuanTable doc)
886 throws LuanException, IOException 884 throws LuanException, IOException
887 { 885 {
891 public void check(Luan luan) throws IOException, SQLException, LuanException, ParseException { 889 public void check(Luan luan) throws IOException, SQLException, LuanException, ParseException {
892 boolean hasPostgres = postgresBackup != null; 890 boolean hasPostgres = postgresBackup != null;
893 String msg = "start check"; 891 String msg = "start check";
894 if( hasPostgres ) 892 if( hasPostgres )
895 msg += " with postgres"; 893 msg += " with postgres";
896 luanLogger.info(msg); 894 logger.info(msg);
897 CheckIndex.Status status = new CheckIndex(fsDir).checkIndex(); 895 CheckIndex.Status status = new CheckIndex(fsDir).checkIndex();
898 if( !status.clean ) 896 if( !status.clean )
899 luanLogger.error("index not clean"); 897 logger.error("index not clean");
900 if( hasPostgres ) 898 if( hasPostgres )
901 checkPostgres(luan); 899 checkPostgres(luan);
902 luanLogger.info("end check"); 900 logger.info("end check");
903 } 901 }
904 902
905 private void checkPostgres(Luan luan) 903 private void checkPostgres(Luan luan)
906 throws IOException, SQLException, LuanException, ParseException 904 throws IOException, SQLException, LuanException, ParseException
907 { 905 {
908 //luanLogger.info("start postgres check"); 906 //logger.info("start postgres check");
909 final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker(); 907 final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker();
910 final IndexSearcher searcher = openSearcher(); 908 final IndexSearcher searcher = openSearcher();
911 try { 909 try {
912 final List<Long> idsLucene = new ArrayList<Long>(); 910 final List<Long> idsLucene = new ArrayList<Long>();
913 Query query = new PrefixQuery(new Term("id")); 911 Query query = new PrefixQuery(new Term("id"));
965 } 963 }
966 964
967 private void checkPostgres(Luan luan,PostgresBackup.Checker postgresChecker,LuanToString lts,long id) 965 private void checkPostgres(Luan luan,PostgresBackup.Checker postgresChecker,LuanToString lts,long id)
968 throws IOException, SQLException, LuanException, ParseException 966 throws IOException, SQLException, LuanException, ParseException
969 { 967 {
970 //luanLogger.info("check id "+id); 968 //logger.info("check id "+id);
971 writeLock.lock(); 969 writeLock.lock();
972 try { 970 try {
973 final IndexSearcher searcher = openSearcher(); 971 final IndexSearcher searcher = openSearcher();
974 try { 972 try {
975 LuanTable docPostgres = postgresChecker.getDoc(id); 973 LuanTable docPostgres = postgresChecker.getDoc(id);
982 docLucene = toTable(luan,doc); 980 docLucene = toTable(luan,doc);
983 } else 981 } else
984 throw new RuntimeException(); 982 throw new RuntimeException();
985 if( docPostgres == null ) { 983 if( docPostgres == null ) {
986 if( docLucene != null ) 984 if( docLucene != null )
987 luanLogger.error("id "+id+" found in lucene but not postgres"); 985 logger.error("id "+id+" found in lucene but not postgres");
988 return; 986 return;
989 } 987 }
990 if( docLucene == null ) { 988 if( docLucene == null ) {
991 luanLogger.error("id "+id+" found in postgres but not lucene"); 989 logger.error("id "+id+" found in postgres but not lucene");
992 return; 990 return;
993 } 991 }
994 if( !equal(docPostgres,docLucene) ) { 992 if( !equal(docPostgres,docLucene) ) {
995 luanLogger.error("id "+id+" not equal"); 993 logger.error("id "+id+" not equal");
996 luanLogger.error("lucene = "+lts.toString(docLucene)); 994 logger.error("lucene = "+lts.toString(docLucene));
997 luanLogger.error("postgres = "+lts.toString(docPostgres)); 995 logger.error("postgres = "+lts.toString(docPostgres));
998 } 996 }
999 } finally { 997 } finally {
1000 close(searcher); 998 close(searcher);
1001 } 999 }
1002 } finally { 1000 } finally {
1014 Map.Entry entry = (Map.Entry)obj; 1012 Map.Entry entry = (Map.Entry)obj;
1015 Object value = entry.getValue(); 1013 Object value = entry.getValue();
1016 if( value instanceof LuanTable ) { 1014 if( value instanceof LuanTable ) {
1017 LuanTable v = (LuanTable)value; 1015 LuanTable v = (LuanTable)value;
1018 if( !v.isList() ) 1016 if( !v.isList() )
1019 sysLogger.error("not list"); 1017 logger.error("not list");
1020 entry.setValue(v.asList()); 1018 entry.setValue(v.asList());
1021 } 1019 }
1022 } 1020 }
1023 return map; 1021 return map;
1024 } 1022 }