repo/luan: src/luan/modules/lucene/LuceneIndex.java annotate

annotate src/luan/modules/lucene/LuceneIndex.java @ 1420:225808b90cee

options handling

author	Franklin Schmidt <fschmidt@gmail.com>
date	Sat, 26 Oct 2019 22:21:09 -0600
parents	dc625408def8
children	103d0ce70385

rev	line source
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	1 package luan.modules.lucene;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	2
527 ef0336efe33c minor Franklin Schmidt <fschmidt@gmail.com> parents: 524 diff changeset	3 import java.io.Closeable;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	4 import java.io.File;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	5 import java.io.FileOutputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	6 import java.io.FileInputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	7 import java.io.IOException;
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	8 import java.lang.ref.Reference;
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	9 import java.lang.ref.WeakReference;
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	10 import java.sql.SQLException;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	11 import java.util.Arrays;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	12 import java.util.Iterator;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	13 import java.util.Map;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	14 import java.util.HashMap;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	15 import java.util.List;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	16 import java.util.ArrayList;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	17 import java.util.Set;
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	18 import java.util.HashSet;
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	19 import java.util.Collections;
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	20 import java.util.concurrent.atomic.AtomicInteger;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	21 import java.util.concurrent.locks.Lock;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	22 import java.util.concurrent.locks.ReentrantLock;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	23 import java.util.zip.ZipOutputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	24 import java.util.zip.ZipEntry;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	25 import org.apache.lucene.analysis.Analyzer;
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	26 import org.apache.lucene.analysis.TokenStream;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	27 import org.apache.lucene.analysis.core.KeywordAnalyzer;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	28 import org.apache.lucene.analysis.en.EnglishAnalyzer;
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	29 import org.apache.lucene.document.Document;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	30 import org.apache.lucene.document.Field;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	31 import org.apache.lucene.document.StoredField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	32 import org.apache.lucene.document.StringField;
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	33 import org.apache.lucene.document.TextField;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	34 import org.apache.lucene.document.IntField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	35 import org.apache.lucene.document.LongField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	36 import org.apache.lucene.document.DoubleField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	37 import org.apache.lucene.index.IndexableField;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	38 import org.apache.lucene.index.IndexWriter;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	39 import org.apache.lucene.index.IndexWriterConfig;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	40 import org.apache.lucene.index.DirectoryReader;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	41 import org.apache.lucene.index.Term;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	42 import org.apache.lucene.index.SnapshotDeletionPolicy;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	43 import org.apache.lucene.index.IndexCommit;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	44 import org.apache.lucene.index.AtomicReaderContext;
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	45 import org.apache.lucene.index.CheckIndex;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	46 import org.apache.lucene.store.Directory;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	47 import org.apache.lucene.store.FSDirectory;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	48 import org.apache.lucene.util.Version;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	49 import org.apache.lucene.util.BytesRef;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	50 import org.apache.lucene.util.NumericUtils;
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	51 import org.apache.lucene.search.Query;
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	52 import org.apache.lucene.search.PrefixQuery;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	53 import org.apache.lucene.search.TermQuery;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	54 import org.apache.lucene.search.TopDocs;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	55 import org.apache.lucene.search.Sort;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	56 import org.apache.lucene.search.SortField;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	57 import org.apache.lucene.search.IndexSearcher;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	58 import org.apache.lucene.search.TotalHitCountCollector;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	59 import org.apache.lucene.search.ScoreDoc;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	60 import org.apache.lucene.search.Collector;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	61 import org.apache.lucene.search.Scorer;
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	62 import org.apache.lucene.search.Explanation;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	63 import org.apache.lucene.search.highlight.Formatter;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	64 import org.apache.lucene.search.highlight.Highlighter;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	65 import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	66 import org.apache.lucene.search.highlight.Fragmenter;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	67 import org.apache.lucene.search.highlight.NullFragmenter;
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	68 import org.apache.lucene.search.highlight.SimpleSpanFragmenter;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	69 import org.apache.lucene.search.highlight.QueryScorer;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	70 import org.apache.lucene.search.highlight.TokenGroup;
1402 27efb1fcbcb5 move luan.lib to goodjava Franklin Schmidt <fschmidt@gmail.com> parents: 1399 diff changeset	71 import goodjava.queryparser.SaneQueryParser;
27efb1fcbcb5 move luan.lib to goodjava Franklin Schmidt <fschmidt@gmail.com> parents: 1399 diff changeset	72 import goodjava.queryparser.FieldParser;
27efb1fcbcb5 move luan.lib to goodjava Franklin Schmidt <fschmidt@gmail.com> parents: 1399 diff changeset	73 import goodjava.queryparser.MultiFieldParser;
27efb1fcbcb5 move luan.lib to goodjava Franklin Schmidt <fschmidt@gmail.com> parents: 1399 diff changeset	74 import goodjava.queryparser.StringFieldParser;
27efb1fcbcb5 move luan.lib to goodjava Franklin Schmidt <fschmidt@gmail.com> parents: 1399 diff changeset	75 import goodjava.queryparser.NumberFieldParser;
27efb1fcbcb5 move luan.lib to goodjava Franklin Schmidt <fschmidt@gmail.com> parents: 1399 diff changeset	76 import goodjava.parser.ParseException;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	77 import luan.modules.Utils;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	78 import luan.Luan;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	79 import luan.LuanTable;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	80 import luan.LuanFunction;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	81 import luan.LuanException;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	82 import luan.LuanRuntimeException;
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	83 import luan.modules.parsers.LuanToString;
1402 27efb1fcbcb5 move luan.lib to goodjava Franklin Schmidt <fschmidt@gmail.com> parents: 1399 diff changeset	84 import goodjava.logging.Logger;
27efb1fcbcb5 move luan.lib to goodjava Franklin Schmidt <fschmidt@gmail.com> parents: 1399 diff changeset	85 import goodjava.logging.LoggerFactory;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	86
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	87
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	88 public final class LuceneIndex {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	89 private static final Logger sysLogger = LoggerFactory.getLogger(LuceneIndex.class);
521 8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	90
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	91 private static Map<String,Reference<LuceneIndex>> indexes = new HashMap<String,Reference<LuceneIndex>>();
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	92
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	93 public static LuceneIndex getLuceneIndex(Luan luan,File indexDir,LuanTable options)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	94 throws LuanException, IOException, ClassNotFoundException, SQLException
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	95 {
1369 709f7498a363 change Lucene.index() and add Lucene.recover() Franklin Schmidt <fschmidt@gmail.com> parents: 1347 diff changeset	96 String key = indexDir.getCanonicalPath();
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	97 synchronized(indexes) {
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	98 Reference<LuceneIndex> ref = indexes.get(key);
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	99 if( ref != null ) {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	100 LuceneIndex li = ref.get();
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	101 if( li != null ) {
67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	102 Object version = options.get("version");
67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	103 if( version==null \|\| version.equals(li.version) )
67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	104 return li;
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	105 li.closeWriter();
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	106 }
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	107 }
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	108 LuceneIndex li = new LuceneIndex(luan,indexDir,options);
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	109 indexes.put(key, new WeakReference<LuceneIndex>(li));
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	110 return li;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	111 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	112 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	113
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	114 private static final Version luceneVersion = Version.LUCENE_4_9;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	115 private static final String FLD_NEXT_ID = "nextId";
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	116 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer());
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	117 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(luceneVersion));
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	118
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	119 private final Logger luanLogger;
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	120 private final Object version;
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	121
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	122 private final ReentrantLock writeLock = new ReentrantLock();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	123 private final File indexDir;
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	124 private SnapshotDeletionPolicy snapshotDeletionPolicy;
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	125 private IndexWriter writer;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	126 private DirectoryReader reader;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	127 private IndexSearcher searcher;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	128 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>();
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	129 private final MultiFieldParser mfp;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	130 private final Analyzer analyzer;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	131
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	132 private FSDirectory fsDir;
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	133 private int writeCount;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	134 private AtomicInteger writeCounter = new AtomicInteger();
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	135
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	136 private Set<String> indexOnly = new HashSet<String>();
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	137 private final FieldParser defaultFieldParser;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	138 private final String[] defaultFields;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	139
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	140 private final PostgresBackup postgresBackup;
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	141 private boolean wasCreated;
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	142
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	143 private LuceneIndex(Luan luan,File indexDir,LuanTable options)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	144 throws LuanException, IOException, ClassNotFoundException, SQLException
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	145 {
1420 225808b90cee options handling Franklin Schmidt <fschmidt@gmail.com> parents: 1410 diff changeset	146 options = new LuanTable(options);
225808b90cee options handling Franklin Schmidt <fschmidt@gmail.com> parents: 1410 diff changeset	147 this.version = options.remove("version");
225808b90cee options handling Franklin Schmidt <fschmidt@gmail.com> parents: 1410 diff changeset	148 FieldParser defaultFieldParser = (FieldParser)options.remove("default_type");
225808b90cee options handling Franklin Schmidt <fschmidt@gmail.com> parents: 1410 diff changeset	149 LuanTable defaultFieldsTbl = Utils.removeTable(options,"default_fields");
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	150 String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]);
1420 225808b90cee options handling Franklin Schmidt <fschmidt@gmail.com> parents: 1410 diff changeset	151 LuanFunction completer = Utils.removeFunction(options,"completer");
225808b90cee options handling Franklin Schmidt <fschmidt@gmail.com> parents: 1410 diff changeset	152 LuanTable postgresSpec = Utils.removeTable(options,"postgres_spec");
225808b90cee options handling Franklin Schmidt <fschmidt@gmail.com> parents: 1410 diff changeset	153 Utils.checkEmpty(options);
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	154
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	155 this.luanLogger = luan.getLogger(LuceneIndex.class);
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	156 this.defaultFieldParser = defaultFieldParser;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	157 this.defaultFields = defaultFields;
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	158 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	159 mfp.fields.put( "type", STRING_FIELD_PARSER );
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	160 mfp.fields.put( "id", NumberFieldParser.LONG );
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	161 this.indexDir = indexDir;
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	162 Analyzer analyzer = STRING_FIELD_PARSER.analyzer;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	163 if( defaultFieldParser instanceof StringFieldParser ) {
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	164 StringFieldParser sfp = (StringFieldParser)defaultFieldParser;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	165 analyzer = sfp.analyzer;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	166 }
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	167 this.analyzer = analyzer;
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	168 wasCreated = reopen();
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	169 if( postgresSpec == null ) {
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	170 postgresBackup = null;
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	171 } else {
1395 9dfff82dfc59 finish postgres work Franklin Schmidt <fschmidt@gmail.com> parents: 1393 diff changeset	172 if( completer == null )
9dfff82dfc59 finish postgres work Franklin Schmidt <fschmidt@gmail.com> parents: 1393 diff changeset	173 throw new LuanException("completer is required for postgres_spec");
1420 225808b90cee options handling Franklin Schmidt <fschmidt@gmail.com> parents: 1410 diff changeset	174 postgresBackup = new PostgresBackup(luan,postgresSpec);
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	175 if( !wasCreated && postgresBackup.wasCreated ) {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	176 luanLogger.error("rebuilding postgres backup");
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	177 rebuild_postgres_backup(completer);
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	178 /*
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	179 } else if( wasCreated && !postgresBackup.wasCreated ) {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	180 luanLogger.error("restoring from postgres");
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	181 restore_from_postgres();
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	182 */
1390 179c4882c6b6 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1388 diff changeset	183 }
179c4882c6b6 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1388 diff changeset	184 }
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	185 }
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	186
1390 179c4882c6b6 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1388 diff changeset	187 public boolean reopen() throws IOException {
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	188 IndexWriterConfig conf = new IndexWriterConfig(luceneVersion,analyzer);
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	189 snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	190 conf.setIndexDeletionPolicy(snapshotDeletionPolicy);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	191 fsDir = FSDirectory.open(indexDir);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	192 boolean wasCreated = !fsDir.getDirectory().exists();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	193 writer = new IndexWriter(fsDir,conf);
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	194 writer.commit(); // commit index creation
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	195 reader = DirectoryReader.open(fsDir);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	196 searcher = new IndexSearcher(reader);
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	197 initId();
1390 179c4882c6b6 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1388 diff changeset	198 return wasCreated;
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	199 }
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	200
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	201 private void wrote() {
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	202 writeCounter.incrementAndGet();
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	203 }
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	204
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	205 public void delete_all() throws IOException, SQLException {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	206 boolean commit = !writeLock.isHeldByCurrentThread();
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	207 writeLock.lock();
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	208 try {
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	209 writer.deleteAll();
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	210 id = idLim = 0;
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	211 if( postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	212 postgresBackup.deleteAll();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	213 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	214 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	215 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	216 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	217 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	218 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	219
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	220 private static Term term(String key,long value) {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	221 BytesRef br = new BytesRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	222 NumericUtils.longToPrefixCoded(value,0,br);
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	223 return new Term(key,br);
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	224 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	225
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	226 private void backupDelete(Query query)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	227 throws IOException, SQLException, LuanException
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	228 {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	229 if( postgresBackup != null ) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	230 final List<Long> ids = new ArrayList<Long>();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	231 IndexSearcher searcher = openSearcher();
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	232 try {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	233 MyCollector col = new MyCollector() {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	234 @Override public void collect(int iDoc) throws IOException {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	235 Document doc = searcher.doc( docBase + iDoc );
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	236 Long id = (Long)doc.getField("id").numericValue();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	237 ids.add(id);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	238 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	239 };
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	240 searcher.search(query,col);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	241 } finally {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	242 close(searcher);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	243 }
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	244 postgresBackup.begin();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	245 for( Long id : ids ) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	246 postgresBackup.delete(id);
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	247 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	248 postgresBackup.commit();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	249 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	250 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	251
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	252 public void delete(String queryStr)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	253 throws IOException, ParseException, SQLException, LuanException
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	254 {
622 1a53333eb4d5 remove Lucene all_search_terms_must_match() since now query suffix "~a" handles this Franklin Schmidt <fschmidt@gmail.com> parents: 621 diff changeset	255 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	256
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	257 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	258 writeLock.lock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	259 try {
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	260 backupDelete(query);
547 0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	261 writer.deleteDocuments(query);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	262 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	263 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	264 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	265 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	266 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	267 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	268
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	269 public void indexed_only_fields(List<String> fields) {
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	270 indexOnly.addAll(fields);
1271 48f302bdc187 fix indexed_only_field Franklin Schmidt <fschmidt@gmail.com> parents: 1267 diff changeset	271 }
48f302bdc187 fix indexed_only_field Franklin Schmidt <fschmidt@gmail.com> parents: 1267 diff changeset	272
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	273 public void save(LuanTable doc,LuanTable boosts)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	274 throws LuanException, IOException, SQLException
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	275 {
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	276 if( boosts!=null && postgresBackup!=null )
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	277 throw new LuanException("boosts are not saved to postgres backup");
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	278
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	279 Object obj = doc.get("id");
601 d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	280 Long id;
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	281 try {
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	282 id = (Long)obj;
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	283 } catch(ClassCastException e) {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	284 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName());
601 d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	285 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	286
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	287 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	288 writeLock.lock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	289 try {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	290 if( id == null ) {
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	291 id = nextId();
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	292 doc.put("id",id);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	293 if( postgresBackup != null )
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	294 postgresBackup.add(doc);
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	295 writer.addDocument(toLucene(doc,boosts));
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	296 } else {
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	297 if( postgresBackup != null )
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	298 postgresBackup.update(doc);
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	299 writer.updateDocument( term("id",id), toLucene(doc,boosts) );
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	300 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	301 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	302 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	303 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	304 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	305 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	306 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	307
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	308 public Object run_in_transaction(LuanFunction fn)
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	309 throws IOException, LuanException, SQLException
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	310 {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	311 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	312 writeLock.lock();
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	313 boolean ok = false;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	314 try {
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	315 if( commit && postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	316 postgresBackup.begin();
1379 87a3738d7cc5 run_in_transaction Franklin Schmidt <fschmidt@gmail.com> parents: 1369 diff changeset	317 Object rtn = fn.call();
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	318 ok = true;
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	319 if(commit) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	320 if( postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	321 postgresBackup.commit();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	322 writer.commit();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	323 }
1379 87a3738d7cc5 run_in_transaction Franklin Schmidt <fschmidt@gmail.com> parents: 1369 diff changeset	324 return rtn;
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	325 } finally {
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	326 if( !ok && commit ) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	327 if( postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	328 postgresBackup.rollback();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	329 writer.rollback();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	330 reopen();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	331 }
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	332 wrote();
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	333 writeLock.unlock();
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	334 }
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	335 }
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	336
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	337 // ???
1379 87a3738d7cc5 run_in_transaction Franklin Schmidt <fschmidt@gmail.com> parents: 1369 diff changeset	338 public Object run_in_lock(LuanFunction fn) throws IOException, LuanException {
756 9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	339 if( writeLock.isHeldByCurrentThread() )
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	340 throw new RuntimeException();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	341 writeLock.lock();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	342 try {
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	343 synchronized(this) {
1379 87a3738d7cc5 run_in_transaction Franklin Schmidt <fschmidt@gmail.com> parents: 1369 diff changeset	344 return fn.call();
756 9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	345 }
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	346 } finally {
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	347 wrote();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	348 writeLock.unlock();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	349 }
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	350 }
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	351
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	352
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	353 private long id;
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	354 private long idLim;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	355 private final int idBatch = 10;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	356
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	357 private void initId() throws IOException {
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	358 TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1);
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	359 switch(td.totalHits) {
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	360 case 0:
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	361 id = 0;
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	362 idLim = 0;
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	363 break;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	364 case 1:
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	365 idLim = (Long)searcher.doc(td.scoreDocs[0].doc).getField(FLD_NEXT_ID).numericValue();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	366 id = idLim;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	367 break;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	368 default:
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	369 throw new RuntimeException();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	370 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	371 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	372
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	373 private void saveNextId(long nextId) throws LuanException, IOException {
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	374 Map doc = new HashMap();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	375 doc.put( "type", "next_id" );
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	376 doc.put( FLD_NEXT_ID, idLim );
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	377 writer.updateDocument(new Term("type","next_id"),toLucene(doc.entrySet(),null));
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	378 }
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	379
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	380 public synchronized long nextId() throws LuanException, IOException {
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	381 if( ++id > idLim ) {
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	382 idLim += idBatch;
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	383 saveNextId(idLim);
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	384 wrote();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	385 }
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	386 return id;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	387 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	388
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	389 /*
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	390 public void backup(String zipFile) throws LuanException, IOException {
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	391 if( !zipFile.endsWith(".zip") )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	392 throw new LuanException("file "+zipFile+" doesn't end with '.zip'");
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	393 IndexCommit ic = snapshotDeletionPolicy.snapshot();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	394 try {
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	395 ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile));
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	396 for( String fileName : ic.getFileNames() ) {
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	397 out.putNextEntry(new ZipEntry(fileName));
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	398 FileInputStream in = new FileInputStream(new File(indexDir,fileName));
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	399 Utils.copyAll(in,out);
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	400 in.close();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	401 out.closeEntry();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	402 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	403 out.close();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	404 } finally {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	405 snapshotDeletionPolicy.release(ic);
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	406 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	407 }
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	408 */
1129 3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	409 public SnapshotDeletionPolicy snapshotDeletionPolicy() {
3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	410 return snapshotDeletionPolicy;
3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	411 }
3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	412
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	413 public Object snapshot(LuanFunction fn) throws LuanException, IOException {
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	414 IndexCommit ic = snapshotDeletionPolicy.snapshot();
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	415 try {
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	416 String dir = fsDir.getDirectory().toString();
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	417 LuanTable fileNames = new LuanTable(fn.luan(),new ArrayList(ic.getFileNames()));
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	418 return fn.call(dir,fileNames);
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	419 } finally {
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	420 snapshotDeletionPolicy.release(ic);
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	421 }
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	422 }
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	423
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	424
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	425
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	426 public String to_string() {
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	427 return writer.getDirectory().toString();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	428 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	429
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	430 protected void finalize() throws Throwable {
67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	431 close();
67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	432 super.finalize();
521 8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	433 }
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	434
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	435 public void close() throws IOException, SQLException {
67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	436 closeWriter();
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	437 reader.close();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	438 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	439
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	440 private void closeWriter() throws IOException, SQLException {
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	441 writeLock.lock();
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	442 try {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	443 writer.close();
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	444 if( postgresBackup != null )
67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	445 postgresBackup.close();
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	446 } finally {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	447 writeLock.unlock();
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	448 }
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	449 }
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	450
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	451
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	452 private static class DocFn extends LuanFunction {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	453 final IndexSearcher searcher;
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	454 final Query query;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	455 int docID;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	456
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	457 DocFn(Luan luan,IndexSearcher searcher,Query query) {
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	458 super(luan);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	459 this.searcher = searcher;
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	460 this.query = query;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	461 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	462
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	463 @Override public Object call(Object[] args) throws LuanException {
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	464 try {
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	465 LuanTable doc = toTable(luan(),searcher.doc(docID));
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	466 if( args.length > 0 && "explain".equals(args[0]) ) {
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	467 Explanation explanation = searcher.explain(query,docID);
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	468 return new Object[]{doc,explanation};
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	469 } else {
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	470 return doc;
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	471 }
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	472 } catch(IOException e) {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	473 throw new LuanException(e);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	474 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	475 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	476 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	477
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	478 private static abstract class MyCollector extends Collector {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	479 int docBase;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	480 int i = 0;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	481
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	482 @Override public void setScorer(Scorer scorer) {}
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	483 @Override public void setNextReader(AtomicReaderContext context) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	484 this.docBase = context.docBase;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	485 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	486 @Override public boolean acceptsDocsOutOfOrder() {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	487 return true;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	488 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	489 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	490
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	491 private synchronized IndexSearcher openSearcher() throws IOException {
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	492 int gwc = writeCounter.get();
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	493 if( writeCount != gwc ) {
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	494 writeCount = gwc;
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	495 DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	496 if( newReader != null ) {
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	497 reader.decRef();
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	498 reader = newReader;
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	499 searcher = new IndexSearcher(reader);
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	500 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	501 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	502 reader.incRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	503 return searcher;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	504 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	505
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	506 // call in finally block
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	507 private static void close(IndexSearcher searcher) throws IOException {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	508 searcher.getIndexReader().decRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	509 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	510
591 790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	511 public void ensure_open() throws IOException {
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	512 close(openSearcher());
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	513 }
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	514
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	515 public int advanced_search( String queryStr, LuanFunction fn, Integer n, String sortStr )
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	516 throws LuanException, IOException, ParseException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	517 {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	518 Utils.checkNotNull(queryStr);
622 1a53333eb4d5 remove Lucene all_search_terms_must_match() since now query suffix "~a" handles this Franklin Schmidt <fschmidt@gmail.com> parents: 621 diff changeset	519 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	520 IndexSearcher searcher = threadLocalSearcher.get();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	521 boolean inTransaction = searcher != null;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	522 if( !inTransaction )
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	523 searcher = openSearcher();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	524 try {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	525 if( fn!=null && n==null ) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	526 if( sortStr != null )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	527 throw new LuanException("sort must be nil when n is nil");
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	528 final DocFn docFn = new DocFn(fn.luan(),searcher,query);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	529 MyCollector col = new MyCollector() {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	530 @Override public void collect(int doc) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	531 try {
547 0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	532 docFn.docID = docBase + doc;
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	533 fn.call(++i,docFn);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	534 } catch(LuanException e) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	535 throw new LuanRuntimeException(e);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	536 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	537 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	538 };
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	539 try {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	540 searcher.search(query,col);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	541 } catch(LuanRuntimeException e) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	542 throw (LuanException)e.getCause();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	543 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	544 return col.i;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	545 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	546 if( fn==null \|\| n==0 ) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	547 TotalHitCountCollector thcc = new TotalHitCountCollector();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	548 searcher.search(query,thcc);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	549 return thcc.getTotalHits();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	550 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	551 Sort sort = sortStr==null ? null : SaneQueryParser.parseSort(mfp,sortStr);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	552 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	553 final ScoreDoc[] scoreDocs = td.scoreDocs;
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	554 DocFn docFn = new DocFn(fn.luan(),searcher,query);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	555 for( int i=0; i<scoreDocs.length; i++ ) {
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	556 ScoreDoc scoreDoc = scoreDocs[i];
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	557 docFn.docID = scoreDoc.doc;
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	558 fn.call(i+1,docFn,scoreDoc.score);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	559 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	560 return td.totalHits;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	561 } finally {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	562 if( !inTransaction )
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	563 close(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	564 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	565 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	566
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	567 public Object search_in_transaction(LuanFunction fn) throws LuanException, IOException {
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	568 if( threadLocalSearcher.get() != null )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	569 throw new LuanException("can't nest search_in_transaction calls");
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	570 IndexSearcher searcher = openSearcher();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	571 threadLocalSearcher.set(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	572 try {
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	573 return fn.call();
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	574 } finally {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	575 threadLocalSearcher.set(null);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	576 close(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	577 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	578 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	579
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	580
796 6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	581 public FieldParser getIndexedFieldParser(String field) {
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	582 return mfp.fields.get(field);
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	583 }
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	584
796 6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	585 public void setIndexedFieldParser(String field,FieldParser fp) {
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	586 if( fp==null ) { // delete
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	587 mfp.fields.remove(field);
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	588 return;
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	589 }
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	590 mfp.fields.put( field, fp );
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	591 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	592
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	593
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	594 private IndexableField newField(String name,Object value,Set<String> indexed,Float boost)
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	595 throws LuanException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	596 {
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	597 boolean hasBoost = boost!=null;
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	598 IndexableField fld = newField2(name,value,indexed,hasBoost);
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	599 if( hasBoost )
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	600 ((Field)fld).setBoost(boost);
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	601 return fld;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	602 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	603
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	604 private IndexableField newField2(String name,Object value,Set<String> indexed,boolean hasBoost)
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	605 throws LuanException
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	606 {
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	607 Field.Store store = indexOnly.contains(name) ? Field.Store.NO : Field.Store.YES;
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	608 if( value instanceof String ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	609 String s = (String)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	610 FieldParser fp = mfp.fields.get(name);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	611 if( fp != null ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	612 if( fp instanceof StringFieldParser && fp != STRING_FIELD_PARSER ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	613 return new TextField(name, s, store);
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	614 } else if (hasBoost) {
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	615 // fuck you modern lucene developers
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	616 return new Field(name, s, store, Field.Index.NOT_ANALYZED);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	617 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	618 return new StringField(name, s, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	619 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	620 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	621 return new StoredField(name, s);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	622 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	623 } else if( value instanceof Integer ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	624 int i = (Integer)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	625 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	626 return new IntField(name, i, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	627 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	628 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	629 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	630 } else if( value instanceof Long ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	631 long i = (Long)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	632 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	633 return new LongField(name, i, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	634 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	635 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	636 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	637 } else if( value instanceof Double ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	638 double i = (Double)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	639 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	640 return new DoubleField(name, i, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	641 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	642 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	643 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	644 } else if( value instanceof byte[] ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	645 byte[] b = (byte[])value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	646 return new StoredField(name, b);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	647 } else
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	648 throw new LuanException("invalid value type "+value.getClass()+"' for '"+name+"'");
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	649 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	650
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	651 private Document toLucene(LuanTable table,LuanTable boosts) throws LuanException {
1347 643cf1c37723 move webserver to lib and bug fixes Franklin Schmidt <fschmidt@gmail.com> parents: 1346 diff changeset	652 return toLucene(table.iterable(),boosts);
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	653 }
efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	654
efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	655 private Document toLucene(Iterable<Map.Entry> iterable,LuanTable boosts) throws LuanException {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	656 Set<String> indexed = mfp.fields.keySet();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	657 Document doc = new Document();
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	658 for( Map.Entry<Object,Object> entry : iterable ) {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	659 Object key = entry.getKey();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	660 if( !(key instanceof String) )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	661 throw new LuanException("key must be string");
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	662 String name = (String)key;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	663 Object value = entry.getValue();
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	664 Float boost = null;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	665 if( boosts != null ) {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	666 Object obj = boosts.get(name);
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	667 if( obj != null ) {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	668 if( !(obj instanceof Number) )
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	669 throw new LuanException("boost '"+name+"' must be number");
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	670 boost = ((Number)obj).floatValue();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	671 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	672 }
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	673 if( !(value instanceof LuanTable) ) {
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	674 doc.add(newField( name, value, indexed, boost ));
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	675 } else { // list
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	676 LuanTable list = (LuanTable)value;
1410 dc625408def8 minor Franklin Schmidt <fschmidt@gmail.com> parents: 1406 diff changeset	677 if( !list.isList() )
dc625408def8 minor Franklin Schmidt <fschmidt@gmail.com> parents: 1406 diff changeset	678 throw new LuanException("table value for '"+name+"' must be a list");
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	679 for( Object el : list.asList() ) {
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	680 doc.add(newField( name, el, indexed, boost ));
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	681 }
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	682 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	683 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	684 return doc;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	685 }
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	686
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	687 private static Object getValue(IndexableField ifld) throws LuanException {
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	688 BytesRef br = ifld.binaryValue();
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	689 if( br != null )
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	690 return br.bytes;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	691 Number n = ifld.numericValue();
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	692 if( n != null )
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	693 return n;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	694 String s = ifld.stringValue();
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	695 if( s != null )
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	696 return s;
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	697 throw new LuanException("invalid field type for "+ifld);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	698 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	699
1333 25746915a241 merge Luan and LuanState Franklin Schmidt <fschmidt@gmail.com> parents: 1271 diff changeset	700 private static LuanTable toTable(Luan luan,Document doc) throws LuanException {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	701 if( doc==null )
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	702 return null;
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	703 LuanTable table = new LuanTable(luan);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	704 for( IndexableField ifld : doc ) {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	705 String name = ifld.name();
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	706 Object value = getValue(ifld);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	707 Object old = table.rawGet(name);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	708 if( old == null ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	709 table.rawPut(name,value);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	710 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	711 LuanTable list;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	712 if( old instanceof LuanTable ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	713 list = (LuanTable)old;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	714 } else {
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	715 list = new LuanTable(luan);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	716 list.rawPut(1,old);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	717 table.rawPut(name,list);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	718 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	719 list.rawPut(list.rawLength()+1,value);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	720 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	721 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	722 return table;
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	723 }
d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	724
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	725
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	726 private static final Formatter nullFormatter = new Formatter() {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	727 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	728 return originalText;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	729 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	730 };
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	731
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	732 public LuanFunction highlighter(String queryStr,final LuanFunction formatter,final Integer fragmentSize,String dotdotdot)
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	733 throws ParseException
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	734 {
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	735 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	736 Formatter fmt = new Formatter() {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	737 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	738 if( tokenGroup.getTotalScore() <= 0 )
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	739 return originalText;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	740 try {
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	741 return (String)Luan.first(formatter.call(originalText));
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	742 } catch(LuanException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	743 throw new LuanRuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	744 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	745 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	746 };
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	747 QueryScorer queryScorer = new QueryScorer(query);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	748 final Highlighter chooser = fragmentSize==null ? null : new Highlighter(nullFormatter,queryScorer);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	749 if( chooser != null )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	750 chooser.setTextFragmenter( new SimpleSpanFragmenter(queryScorer,fragmentSize) );
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	751 final Highlighter hl = new Highlighter(fmt,queryScorer);
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	752 hl.setTextFragmenter( new NullFragmenter() );
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	753 return new LuanFunction(false) { // ???
e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	754 @Override public String call(Object[] args) throws LuanException {
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	755 String text = (String)args[0];
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	756 try {
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	757 if( chooser != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	758 String s = chooser.getBestFragment(analyzer,null,text);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	759 if( s != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	760 if( dotdotdot != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	761 boolean atStart = text.startsWith(s);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	762 boolean atEnd = text.endsWith(s);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	763 if( !atStart )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	764 s = dotdotdot + s;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	765 if( !atEnd )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	766 s = s + dotdotdot;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	767 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	768 text = s;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	769 } else if( text.length() > fragmentSize ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	770 text = text.substring(0,fragmentSize);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	771 if( dotdotdot != null )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	772 text += "...";
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	773 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	774 }
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	775 String s = hl.getBestFragment(analyzer,null,text);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	776 return s!=null ? s : text;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	777 } catch(LuanRuntimeException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	778 throw (LuanException)e.getCause();
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	779 } catch(IOException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	780 throw new RuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	781 } catch(InvalidTokenOffsetsException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	782 throw new RuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	783 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	784 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	785 };
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	786 }
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	787
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	788 public int count_tokens(String text)
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	789 throws IOException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	790 {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	791 int n = 0;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	792 TokenStream ts = analyzer.tokenStream(null,text);
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	793 ts.reset();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	794 while( ts.incrementToken() ) {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	795 n++;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	796 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	797 ts.close();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	798 return n;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	799 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	800
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	801
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	802
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	803 public boolean hasPostgresBackup() {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	804 return postgresBackup != null;
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	805 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	806
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	807 public void rebuild_postgres_backup(LuanFunction completer)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	808 throws IOException, LuanException, SQLException
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	809 {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	810 luanLogger.info("start rebuild_postgres_backup");
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	811 writeLock.lock();
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	812 IndexSearcher searcher = openSearcher();
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	813 boolean ok = false;
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	814 try {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	815 postgresBackup.begin();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	816 postgresBackup.deleteAll();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	817 Query query = new PrefixQuery(new Term("id"));
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	818 MyCollector col = new MyCollector() {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	819 @Override public void collect(int iDoc) throws IOException {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	820 try {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	821 Document doc = searcher.doc( docBase + iDoc );
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	822 LuanTable tbl = toTable(completer.luan(),doc);
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	823 tbl = (LuanTable)completer.call(tbl);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	824 postgresBackup.add(tbl);
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	825 } catch(LuanException e) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	826 throw new LuanRuntimeException(e);
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	827 } catch(SQLException e) {
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	828 throw new RuntimeException(e);
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	829 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	830 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	831 };
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	832 try {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	833 searcher.search(query,col);
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	834 } catch(LuanRuntimeException e) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	835 throw (LuanException)e.getCause();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	836 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	837 ok = true;
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	838 postgresBackup.commit();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	839 } finally {
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	840 close(searcher);
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	841 if( !ok )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	842 postgresBackup.rollback();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	843 writeLock.unlock();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	844 }
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	845 luanLogger.info("end rebuild_postgres_backup");
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	846 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	847
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	848 public void restore_from_postgres()
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	849 throws IOException, LuanException, SQLException
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	850 {
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	851 if( postgresBackup!=null && wasCreated && !postgresBackup.wasCreated ) {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	852 luanLogger.error("restoring from postgres");
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	853 force_restore_from_postgres();
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	854 }
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	855 }
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	856
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	857 public void force_restore_from_postgres()
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	858 throws IOException, LuanException, SQLException
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	859 {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	860 luanLogger.warn("start restore_from_postgres");
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	861 if( postgresBackup==null )
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	862 throw new NullPointerException();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	863 if( writeLock.isHeldByCurrentThread() )
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	864 throw new RuntimeException();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	865 writeLock.lock();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	866 boolean ok = false;
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	867 try {
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	868 writer.deleteAll();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	869 long nextId = postgresBackup.maxId() + 1;
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	870 postgresBackup.restoreLucene(this);
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	871 id = idLim = nextId;
1395 9dfff82dfc59 finish postgres work Franklin Schmidt <fschmidt@gmail.com> parents: 1393 diff changeset	872 saveNextId(nextId);
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	873 ok = true;
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	874 writer.commit();
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	875 wasCreated = false;
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	876 } finally {
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	877 if( !ok ) {
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	878 writer.rollback();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	879 reopen();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	880 }
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	881 wrote();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	882 writeLock.unlock();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	883 }
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	884 luanLogger.warn("end restore_from_postgres");
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	885 }
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	886
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	887 void restore(LuanTable doc)
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	888 throws LuanException, IOException
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	889 {
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	890 writer.addDocument(toLucene(doc,null));
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	891 }
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	892
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	893 public void check(LuanFunction completer) throws IOException, SQLException, LuanException {
1406 8187ddb0e827 minor Franklin Schmidt <fschmidt@gmail.com> parents: 1402 diff changeset	894 boolean hasPostgres = postgresBackup != null;
8187ddb0e827 minor Franklin Schmidt <fschmidt@gmail.com> parents: 1402 diff changeset	895 String msg = "start check";
8187ddb0e827 minor Franklin Schmidt <fschmidt@gmail.com> parents: 1402 diff changeset	896 if( hasPostgres )
8187ddb0e827 minor Franklin Schmidt <fschmidt@gmail.com> parents: 1402 diff changeset	897 msg += " with postgres";
8187ddb0e827 minor Franklin Schmidt <fschmidt@gmail.com> parents: 1402 diff changeset	898 luanLogger.info(msg);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	899 CheckIndex.Status status = new CheckIndex(fsDir).checkIndex();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	900 if( !status.clean )
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	901 luanLogger.error("index not clean");
1406 8187ddb0e827 minor Franklin Schmidt <fschmidt@gmail.com> parents: 1402 diff changeset	902 if( hasPostgres )
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	903 checkPostgres(completer);
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	904 luanLogger.info("end check");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	905 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	906
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	907 private void checkPostgres(LuanFunction completer) throws IOException, SQLException, LuanException {
1406 8187ddb0e827 minor Franklin Schmidt <fschmidt@gmail.com> parents: 1402 diff changeset	908 //luanLogger.info("start postgres check");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	909 final PostgresBackup.Checker postgresChecker;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	910 final IndexSearcher searcher;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	911 writeLock.lock();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	912 try {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	913 postgresChecker = postgresBackup.newChecker();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	914 searcher = openSearcher();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	915 } finally {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	916 writeLock.unlock();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	917 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	918 try {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	919 final List<Long> idsLucene = new ArrayList<Long>();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	920 Query query = new PrefixQuery(new Term("id"));
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	921 MyCollector col = new MyCollector() {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	922 @Override public void collect(int iDoc) throws IOException {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	923 Document doc = searcher.doc( docBase + iDoc );
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	924 Long id = (Long)doc.getField("id").numericValue();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	925 idsLucene.add(id);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	926 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	927 };
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	928 searcher.search(query,col);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	929 Collections.sort(idsLucene);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	930 final List<Long> idsPostgres = postgresChecker.getIds();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	931 final int nLucene = idsLucene.size();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	932 final int nPostgres = idsPostgres.size();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	933 int iLucene = 0;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	934 int iPostgres = 0;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	935 LuanToString lts = new LuanToString();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	936 lts.strict = true;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	937 lts.numberTypes = true;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	938 while( iLucene < nLucene && iPostgres < nPostgres ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	939 long idLucene = idsLucene.get(iLucene);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	940 long idPostgres = idsPostgres.get(iPostgres);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	941 if( idLucene < idPostgres ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	942 iLucene++;
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	943 luanLogger.error("id "+idLucene+" found in lucene but not postgres");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	944 } else if( idLucene > idPostgres ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	945 iPostgres++;
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	946 luanLogger.error("id "+idPostgres+" found in postgres but not lucene");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	947 } else { // ==
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	948 LuanTable docPostgres = postgresChecker.getDoc(idPostgres);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	949 TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	950 if( td.totalHits != 1 ) throw new RuntimeException();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	951 Document doc = searcher.doc( td.scoreDocs[0].doc );
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	952 LuanTable docLucene = toTable(completer.luan(),doc);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	953 docLucene = (LuanTable)completer.call(docLucene);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	954 if( !equal(docPostgres,docLucene) ) {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	955 luanLogger.error("id "+idLucene+" not equal");
cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	956 luanLogger.error("lucene = "+lts.toString(docLucene));
cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	957 luanLogger.error("postgres = "+lts.toString(docPostgres));
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	958 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	959 iLucene++;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	960 iPostgres++;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	961 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	962 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	963 while( iLucene < nLucene ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	964 long idLucene = idsLucene.get(iLucene++);
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	965 luanLogger.error("id "+idLucene+" found in lucene but not postgres");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	966 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	967 while( iPostgres < nPostgres ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	968 long idPostgres = idsPostgres.get(iPostgres++);
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	969 luanLogger.error("id "+idPostgres+" found in postgres but not lucene");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	970 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	971 } finally {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	972 close(searcher);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	973 postgresChecker.close();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	974 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	975 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	976
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	977 private boolean equal(LuanTable t1,LuanTable t2) throws LuanException {
1399 38a1c1b4279a fix Franklin Schmidt <fschmidt@gmail.com> parents: 1398 diff changeset	978 return t1!=null && t2!=null && t1.asMap().equals(t2.asMap());
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	979 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	980
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	981 }

Mercurial Hosting > luan

annotate src/luan/modules/lucene/LuceneIndex.java @ 1420:225808b90cee