Mercurial Hosting > luan

comparison src/luan/modules/lucene/LuceneIndex.java @ 1538:634f6765830e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
use goodjava/lucene/logging
author Franklin Schmidt <fschmidt@gmail.com>
date Fri, 07 Aug 2020 21:42:16 -0600
parents e73b72a510b4
children c27dc6af87ca
comparison
equal deleted inserted replaced
1537:f7649ad6e3e7 1538:634f6765830e
76 import goodjava.lucene.queryparser.NumberFieldParser; 76 import goodjava.lucene.queryparser.NumberFieldParser;
77 import goodjava.lucene.api.GoodIndexWriter; 77 import goodjava.lucene.api.GoodIndexWriter;
78 import goodjava.lucene.api.LuceneIndexWriter; 78 import goodjava.lucene.api.LuceneIndexWriter;
79 import goodjava.lucene.api.GoodIndexWriterConfig; 79 import goodjava.lucene.api.GoodIndexWriterConfig;
80 import goodjava.lucene.api.LuceneUtils; 80 import goodjava.lucene.api.LuceneUtils;
81 import goodjava.lucene.logging.LoggingIndexWriter;
81 import goodjava.parser.ParseException; 82 import goodjava.parser.ParseException;
82 import luan.modules.Utils; 83 import luan.modules.Utils;
83 import luan.Luan; 84 import luan.Luan;
84 import luan.LuanTable; 85 import luan.LuanTable;
85 import luan.LuanFunction; 86 import luan.LuanFunction;
119 private static final Version luceneVersion = Version.LUCENE_4_9; 120 private static final Version luceneVersion = Version.LUCENE_4_9;
120 private static final String FLD_NEXT_ID = "nextId"; 121 private static final String FLD_NEXT_ID = "nextId";
121 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer()); 122 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer());
122 public static final StringFieldParser LOWERCASE_FIELD_PARSER = new StringFieldParser(new LowercaseAnalyzer(luceneVersion)); 123 public static final StringFieldParser LOWERCASE_FIELD_PARSER = new StringFieldParser(new LowercaseAnalyzer(luceneVersion));
123 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(luceneVersion)); 124 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(luceneVersion));
125 private static final SortField ID_SORT = new SortField("id",SortField.Type.LONG);
124 126
125 private final Object version; 127 private final Object version;
126 128
127 private final ReentrantLock writeLock = new ReentrantLock(); 129 private final ReentrantLock writeLock = new ReentrantLock();
128 private final File indexDir; 130 private final File indexDir;
138 private AtomicInteger writeCounter = new AtomicInteger(); 140 private AtomicInteger writeCounter = new AtomicInteger();
139 private final GoodIndexWriterConfig config; 141 private final GoodIndexWriterConfig config;
140 142
141 private final PostgresBackup postgresBackup; 143 private final PostgresBackup postgresBackup;
142 private boolean wasCreated; 144 private boolean wasCreated;
145 private final File logDir;
143 146
144 private LuceneIndex(Luan luan,File indexDir,LuanTable options) 147 private LuceneIndex(Luan luan,File indexDir,LuanTable options)
145 throws LuanException, IOException, ClassNotFoundException, SQLException 148 throws LuanException, IOException, ClassNotFoundException, SQLException
146 { 149 {
147 options = new LuanTable(options); 150 options = new LuanTable(options);
149 FieldParser defaultFieldParser = (FieldParser)options.remove("default_type"); 152 FieldParser defaultFieldParser = (FieldParser)options.remove("default_type");
150 LuanTable defaultFieldsTbl = Utils.removeTable(options,"default_fields"); 153 LuanTable defaultFieldsTbl = Utils.removeTable(options,"default_fields");
151 String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]); 154 String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]);
152 LuanTable postgresSpec = Utils.removeTable(options,"postgres_spec"); 155 LuanTable postgresSpec = Utils.removeTable(options,"postgres_spec");
153 LuanFunction supplementer = Utils.removeFunction(options,"supplementer"); 156 LuanFunction supplementer = Utils.removeFunction(options,"supplementer");
157 logDir = (File)options.remove("log_dir");
154 Utils.checkEmpty(options); 158 Utils.checkEmpty(options);
155 159
156 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields); 160 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
157 mfp.fields.put( "type", STRING_FIELD_PARSER ); 161 mfp.fields.put( "type", STRING_FIELD_PARSER );
158 mfp.fields.put( "id", NumberFieldParser.LONG ); 162 mfp.fields.put( "id", NumberFieldParser.LONG );
183 187
184 public boolean reopen() throws IOException { 188 public boolean reopen() throws IOException {
185 fsDir = FSDirectory.open(indexDir); 189 fsDir = FSDirectory.open(indexDir);
186 boolean wasCreated = !fsDir.getDirectory().exists(); 190 boolean wasCreated = !fsDir.getDirectory().exists();
187 writer = new LuceneIndexWriter(fsDir,config); 191 writer = new LuceneIndexWriter(fsDir,config);
192 if( logDir != null )
193 writer = new LoggingIndexWriter((LuceneIndexWriter)writer,logDir);
188 reader = DirectoryReader.open(fsDir); 194 reader = DirectoryReader.open(fsDir);
189 searcher = new IndexSearcher(reader); 195 searcher = new IndexSearcher(reader);
190 initId(); 196 initId();
191 return wasCreated; 197 return wasCreated;
192 } 198 }
779 msg += " with postgres"; 785 msg += " with postgres";
780 logger.info(msg); 786 logger.info(msg);
781 CheckIndex.Status status = new CheckIndex(fsDir).checkIndex(); 787 CheckIndex.Status status = new CheckIndex(fsDir).checkIndex();
782 if( !status.clean ) 788 if( !status.clean )
783 logger.error("index not clean"); 789 logger.error("index not clean");
784 if( hasPostgres ) 790 if( writer instanceof LoggingIndexWriter ) {
791 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer;
792 logger.info("log check");
793 boolean ok = loggingWriter.check(ID_SORT);
794 }
795 if( hasPostgres ) {
796 logger.info("postgres check");
785 checkPostgres(luan); 797 checkPostgres(luan);
798 }
786 logger.info("end check"); 799 logger.info("end check");
800 }
801
802 public void rebuild_log() throws IOException {
803 logger.info("start rebuild_log");
804 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer;
805 loggingWriter.newLogs();
806 logger.info("end rebuild_log");
787 } 807 }
788 808
789 private void checkPostgres(Luan luan) 809 private void checkPostgres(Luan luan)
790 throws IOException, SQLException, LuanException, ParseException 810 throws IOException, SQLException, LuanException, ParseException
791 { 811 {
792 //logger.info("start postgres check");
793 final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker(); 812 final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker();
794 final IndexSearcher searcher = openSearcher(); 813 final IndexSearcher searcher = openSearcher();
795 try { 814 try {
796 final List<Long> idsLucene = new ArrayList<Long>(); 815 final List<Long> idsLucene = new ArrayList<Long>();
797 Query query = new PrefixQuery(new Term("id")); 816 Query query = new PrefixQuery(new Term("id"));