repo/luan: src/luan/modules/lucene/LuceneIndex.java annotate

annotate src/luan/modules/lucene/LuceneIndex.java @ 1397:0dc9837c16be

fix lucene issues

author	Franklin Schmidt <fschmidt@gmail.com>
date	Wed, 11 Sep 2019 01:31:21 -0600
parents	9dfff82dfc59
children	67c0e47b5be3

rev	line source
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	1 package luan.modules.lucene;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	2
527 ef0336efe33c minor Franklin Schmidt <fschmidt@gmail.com> parents: 524 diff changeset	3 import java.io.Closeable;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	4 import java.io.File;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	5 import java.io.FileOutputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	6 import java.io.FileInputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	7 import java.io.IOException;
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	8 import java.lang.ref.Reference;
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	9 import java.lang.ref.WeakReference;
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	10 import java.sql.SQLException;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	11 import java.util.Arrays;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	12 import java.util.Iterator;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	13 import java.util.Map;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	14 import java.util.HashMap;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	15 import java.util.List;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	16 import java.util.ArrayList;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	17 import java.util.Set;
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	18 import java.util.HashSet;
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	19 import java.util.Collections;
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	20 import java.util.concurrent.atomic.AtomicInteger;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	21 import java.util.concurrent.locks.Lock;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	22 import java.util.concurrent.locks.ReentrantLock;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	23 import java.util.zip.ZipOutputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	24 import java.util.zip.ZipEntry;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	25 import org.apache.lucene.analysis.Analyzer;
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	26 import org.apache.lucene.analysis.TokenStream;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	27 import org.apache.lucene.analysis.core.KeywordAnalyzer;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	28 import org.apache.lucene.analysis.en.EnglishAnalyzer;
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	29 import org.apache.lucene.document.Document;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	30 import org.apache.lucene.document.Field;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	31 import org.apache.lucene.document.StoredField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	32 import org.apache.lucene.document.StringField;
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	33 import org.apache.lucene.document.TextField;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	34 import org.apache.lucene.document.IntField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	35 import org.apache.lucene.document.LongField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	36 import org.apache.lucene.document.DoubleField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	37 import org.apache.lucene.index.IndexableField;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	38 import org.apache.lucene.index.IndexWriter;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	39 import org.apache.lucene.index.IndexWriterConfig;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	40 import org.apache.lucene.index.DirectoryReader;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	41 import org.apache.lucene.index.Term;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	42 import org.apache.lucene.index.SnapshotDeletionPolicy;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	43 import org.apache.lucene.index.IndexCommit;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	44 import org.apache.lucene.index.AtomicReaderContext;
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	45 import org.apache.lucene.index.CheckIndex;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	46 import org.apache.lucene.store.Directory;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	47 import org.apache.lucene.store.FSDirectory;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	48 import org.apache.lucene.util.Version;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	49 import org.apache.lucene.util.BytesRef;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	50 import org.apache.lucene.util.NumericUtils;
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	51 import org.apache.lucene.search.Query;
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	52 import org.apache.lucene.search.PrefixQuery;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	53 import org.apache.lucene.search.TermQuery;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	54 import org.apache.lucene.search.TopDocs;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	55 import org.apache.lucene.search.Sort;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	56 import org.apache.lucene.search.SortField;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	57 import org.apache.lucene.search.IndexSearcher;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	58 import org.apache.lucene.search.TotalHitCountCollector;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	59 import org.apache.lucene.search.ScoreDoc;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	60 import org.apache.lucene.search.Collector;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	61 import org.apache.lucene.search.Scorer;
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	62 import org.apache.lucene.search.Explanation;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	63 import org.apache.lucene.search.highlight.Formatter;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	64 import org.apache.lucene.search.highlight.Highlighter;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	65 import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	66 import org.apache.lucene.search.highlight.Fragmenter;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	67 import org.apache.lucene.search.highlight.NullFragmenter;
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	68 import org.apache.lucene.search.highlight.SimpleSpanFragmenter;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	69 import org.apache.lucene.search.highlight.QueryScorer;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	70 import org.apache.lucene.search.highlight.TokenGroup;
1344 dc2af9d5463b move queryparser to lib Franklin Schmidt <fschmidt@gmail.com> parents: 1343 diff changeset	71 import luan.lib.queryparser.SaneQueryParser;
dc2af9d5463b move queryparser to lib Franklin Schmidt <fschmidt@gmail.com> parents: 1343 diff changeset	72 import luan.lib.queryparser.FieldParser;
dc2af9d5463b move queryparser to lib Franklin Schmidt <fschmidt@gmail.com> parents: 1343 diff changeset	73 import luan.lib.queryparser.MultiFieldParser;
dc2af9d5463b move queryparser to lib Franklin Schmidt <fschmidt@gmail.com> parents: 1343 diff changeset	74 import luan.lib.queryparser.StringFieldParser;
dc2af9d5463b move queryparser to lib Franklin Schmidt <fschmidt@gmail.com> parents: 1343 diff changeset	75 import luan.lib.queryparser.NumberFieldParser;
1111 88b5b81cad4a move Parser to luan.lib.parser Franklin Schmidt <fschmidt@gmail.com> parents: 1110 diff changeset	76 import luan.lib.parser.ParseException;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	77 import luan.modules.Utils;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	78 import luan.Luan;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	79 import luan.LuanTable;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	80 import luan.LuanFunction;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	81 import luan.LuanException;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	82 import luan.LuanRuntimeException;
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	83 import luan.modules.parsers.LuanToString;
1337 8b61c8c4e07a remove slf4j Franklin Schmidt <fschmidt@gmail.com> parents: 1335 diff changeset	84 import luan.lib.logging.Logger;
8b61c8c4e07a remove slf4j Franklin Schmidt <fschmidt@gmail.com> parents: 1335 diff changeset	85 import luan.lib.logging.LoggerFactory;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	86
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	87
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	88 public final class LuceneIndex implements Closeable {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	89 private static final Logger sysLogger = LoggerFactory.getLogger(LuceneIndex.class);
521 8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	90
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	91 private static Map<String,Reference<LuceneIndex>> indexes = new HashMap<String,Reference<LuceneIndex>>();
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	92
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	93 public static LuceneIndex getLuceneIndex(Luan luan,File indexDir,LuanTable options)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	94 throws LuanException, IOException, ClassNotFoundException, SQLException
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	95 {
1369 709f7498a363 change Lucene.index() and add Lucene.recover() Franklin Schmidt <fschmidt@gmail.com> parents: 1347 diff changeset	96 String key = indexDir.getCanonicalPath();
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	97 synchronized(indexes) {
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	98 Reference<LuceneIndex> ref = indexes.get(key);
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	99 if( ref != null ) {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	100 LuceneIndex li = ref.get();
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	101 if( li != null )
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	102 li.closeWriter();
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	103 }
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	104 LuceneIndex li = new LuceneIndex(luan,indexDir,options);
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	105 indexes.put(key, new WeakReference<LuceneIndex>(li));
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	106 return li;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	107 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	108 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	109
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	110 private static final Version version = Version.LUCENE_4_9;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	111 private static final String FLD_NEXT_ID = "nextId";
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	112 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer());
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	113 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(version));
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	114
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	115 private final Logger luanLogger;
cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	116
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	117 private final ReentrantLock writeLock = new ReentrantLock();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	118 private final File indexDir;
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	119 private SnapshotDeletionPolicy snapshotDeletionPolicy;
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	120 private IndexWriter writer;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	121 private DirectoryReader reader;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	122 private IndexSearcher searcher;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	123 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>();
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	124 private final MultiFieldParser mfp;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	125 private final Analyzer analyzer;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	126
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	127 private FSDirectory fsDir;
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	128 private int writeCount;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	129 private AtomicInteger writeCounter = new AtomicInteger();
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	130
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	131 private Set<String> indexOnly = new HashSet<String>();
1271 48f302bdc187 fix indexed_only_field Franklin Schmidt <fschmidt@gmail.com> parents: 1267 diff changeset	132
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	133 private boolean isClosed = false;
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	134 private final Exception created = new Exception("created");
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	135 private final FieldParser defaultFieldParser;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	136 private final String[] defaultFields;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	137
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	138 private final PostgresBackup postgresBackup;
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	139 private boolean wasCreated;
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	140
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	141 private LuceneIndex(Luan luan,File indexDir,LuanTable options)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	142 throws LuanException, IOException, ClassNotFoundException, SQLException
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	143 {
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	144 Map map = options.asMap();
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	145 FieldParser defaultFieldParser = (FieldParser)map.remove("default_type");
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	146 LuanTable defaultFieldsTbl = Utils.removeTable(map,"default_fields");
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	147 String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]);
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	148 LuanFunction completer = Utils.removeFunction(map,"completer");
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	149 LuanTable postgresSpec = Utils.removeTable(map,"postgres_spec");
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	150 Utils.checkEmpty(map);
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	151
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	152 this.luanLogger = luan.getLogger(LuceneIndex.class);
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	153 this.defaultFieldParser = defaultFieldParser;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	154 this.defaultFields = defaultFields;
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	155 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	156 mfp.fields.put( "type", STRING_FIELD_PARSER );
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	157 mfp.fields.put( "id", NumberFieldParser.LONG );
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	158 this.indexDir = indexDir;
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	159 Analyzer analyzer = STRING_FIELD_PARSER.analyzer;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	160 if( defaultFieldParser instanceof StringFieldParser ) {
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	161 StringFieldParser sfp = (StringFieldParser)defaultFieldParser;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	162 analyzer = sfp.analyzer;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	163 }
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	164 this.analyzer = analyzer;
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	165 wasCreated = reopen();
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	166 if( postgresSpec == null ) {
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	167 postgresBackup = null;
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	168 } else {
1395 9dfff82dfc59 finish postgres work Franklin Schmidt <fschmidt@gmail.com> parents: 1393 diff changeset	169 if( completer == null )
9dfff82dfc59 finish postgres work Franklin Schmidt <fschmidt@gmail.com> parents: 1393 diff changeset	170 throw new LuanException("completer is required for postgres_spec");
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	171 Map spec = postgresSpec.asMap();
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	172 postgresBackup = new PostgresBackup(luan,spec);
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	173 if( !wasCreated && postgresBackup.wasCreated ) {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	174 luanLogger.error("rebuilding postgres backup");
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	175 rebuild_postgres_backup(completer);
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	176 /*
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	177 } else if( wasCreated && !postgresBackup.wasCreated ) {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	178 luanLogger.error("restoring from postgres");
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	179 restore_from_postgres();
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	180 */
1390 179c4882c6b6 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1388 diff changeset	181 }
179c4882c6b6 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1388 diff changeset	182 }
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	183 luan.onClose(this);
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	184 }
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	185
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	186 protected void finalize() throws Throwable {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	187 if( !isClosed ) {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	188 sysLogger.error("not closed",created);
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	189 close();
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	190 }
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	191 super.finalize();
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	192 }
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	193
1390 179c4882c6b6 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1388 diff changeset	194 public boolean reopen() throws IOException {
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	195 IndexWriterConfig conf = new IndexWriterConfig(version,analyzer);
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	196 snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	197 conf.setIndexDeletionPolicy(snapshotDeletionPolicy);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	198 fsDir = FSDirectory.open(indexDir);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	199 boolean wasCreated = !fsDir.getDirectory().exists();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	200 writer = new IndexWriter(fsDir,conf);
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	201 writer.commit(); // commit index creation
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	202 reader = DirectoryReader.open(fsDir);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	203 searcher = new IndexSearcher(reader);
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	204 initId();
1390 179c4882c6b6 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1388 diff changeset	205 return wasCreated;
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	206 }
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	207
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	208 private void wrote() {
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	209 writeCounter.incrementAndGet();
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	210 }
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	211
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	212 public void delete_all() throws IOException, SQLException {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	213 boolean commit = !writeLock.isHeldByCurrentThread();
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	214 writeLock.lock();
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	215 try {
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	216 writer.deleteAll();
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	217 id = idLim = 0;
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	218 if( postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	219 postgresBackup.deleteAll();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	220 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	221 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	222 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	223 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	224 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	225 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	226
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	227 private static Term term(String key,long value) {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	228 BytesRef br = new BytesRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	229 NumericUtils.longToPrefixCoded(value,0,br);
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	230 return new Term(key,br);
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	231 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	232
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	233 private void backupDelete(Query query)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	234 throws IOException, SQLException, LuanException
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	235 {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	236 if( postgresBackup != null ) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	237 final List<Long> ids = new ArrayList<Long>();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	238 IndexSearcher searcher = openSearcher();
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	239 try {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	240 MyCollector col = new MyCollector() {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	241 @Override public void collect(int iDoc) throws IOException {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	242 Document doc = searcher.doc( docBase + iDoc );
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	243 Long id = (Long)doc.getField("id").numericValue();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	244 ids.add(id);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	245 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	246 };
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	247 searcher.search(query,col);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	248 } finally {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	249 close(searcher);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	250 }
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	251 postgresBackup.begin();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	252 for( Long id : ids ) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	253 postgresBackup.delete(id);
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	254 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	255 postgresBackup.commit();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	256 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	257 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	258
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	259 public void delete(String queryStr)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	260 throws IOException, ParseException, SQLException, LuanException
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	261 {
622 1a53333eb4d5 remove Lucene all_search_terms_must_match() since now query suffix "~a" handles this Franklin Schmidt <fschmidt@gmail.com> parents: 621 diff changeset	262 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	263
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	264 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	265 writeLock.lock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	266 try {
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	267 backupDelete(query);
547 0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	268 writer.deleteDocuments(query);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	269 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	270 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	271 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	272 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	273 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	274 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	275
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	276 public void indexed_only_fields(List<String> fields) {
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	277 indexOnly.addAll(fields);
1271 48f302bdc187 fix indexed_only_field Franklin Schmidt <fschmidt@gmail.com> parents: 1267 diff changeset	278 }
48f302bdc187 fix indexed_only_field Franklin Schmidt <fschmidt@gmail.com> parents: 1267 diff changeset	279
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	280 public void save(LuanTable doc,LuanTable boosts)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	281 throws LuanException, IOException, SQLException
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	282 {
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	283 if( boosts!=null && postgresBackup!=null )
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	284 throw new LuanException("boosts are not saved to postgres backup");
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	285
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	286 Object obj = doc.get("id");
601 d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	287 Long id;
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	288 try {
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	289 id = (Long)obj;
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	290 } catch(ClassCastException e) {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	291 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName());
601 d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	292 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	293
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	294 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	295 writeLock.lock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	296 try {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	297 if( id == null ) {
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	298 id = nextId();
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	299 doc.put("id",id);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	300 if( postgresBackup != null )
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	301 postgresBackup.add(doc);
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	302 writer.addDocument(toLucene(doc,boosts));
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	303 } else {
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	304 if( postgresBackup != null )
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	305 postgresBackup.update(doc);
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	306 writer.updateDocument( term("id",id), toLucene(doc,boosts) );
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	307 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	308 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	309 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	310 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	311 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	312 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	313 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	314
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	315 public Object run_in_transaction(LuanFunction fn)
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	316 throws IOException, LuanException, SQLException
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	317 {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	318 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	319 writeLock.lock();
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	320 boolean ok = false;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	321 try {
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	322 if( commit && postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	323 postgresBackup.begin();
1379 87a3738d7cc5 run_in_transaction Franklin Schmidt <fschmidt@gmail.com> parents: 1369 diff changeset	324 Object rtn = fn.call();
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	325 ok = true;
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	326 if(commit) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	327 if( postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	328 postgresBackup.commit();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	329 writer.commit();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	330 }
1379 87a3738d7cc5 run_in_transaction Franklin Schmidt <fschmidt@gmail.com> parents: 1369 diff changeset	331 return rtn;
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	332 } finally {
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	333 if( !ok && commit ) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	334 if( postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	335 postgresBackup.rollback();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	336 writer.rollback();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	337 reopen();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	338 }
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	339 wrote();
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	340 writeLock.unlock();
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	341 }
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	342 }
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	343
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	344 // ???
1379 87a3738d7cc5 run_in_transaction Franklin Schmidt <fschmidt@gmail.com> parents: 1369 diff changeset	345 public Object run_in_lock(LuanFunction fn) throws IOException, LuanException {
756 9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	346 if( writeLock.isHeldByCurrentThread() )
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	347 throw new RuntimeException();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	348 writeLock.lock();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	349 try {
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	350 synchronized(this) {
1379 87a3738d7cc5 run_in_transaction Franklin Schmidt <fschmidt@gmail.com> parents: 1369 diff changeset	351 return fn.call();
756 9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	352 }
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	353 } finally {
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	354 wrote();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	355 writeLock.unlock();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	356 }
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	357 }
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	358
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	359
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	360 private long id;
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	361 private long idLim;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	362 private final int idBatch = 10;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	363
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	364 private void initId() throws IOException {
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	365 TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1);
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	366 switch(td.totalHits) {
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	367 case 0:
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	368 id = 0;
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	369 idLim = 0;
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	370 break;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	371 case 1:
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	372 idLim = (Long)searcher.doc(td.scoreDocs[0].doc).getField(FLD_NEXT_ID).numericValue();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	373 id = idLim;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	374 break;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	375 default:
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	376 throw new RuntimeException();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	377 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	378 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	379
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	380 private void saveNextId(long nextId) throws LuanException, IOException {
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	381 Map doc = new HashMap();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	382 doc.put( "type", "next_id" );
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	383 doc.put( FLD_NEXT_ID, idLim );
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	384 writer.updateDocument(new Term("type","next_id"),toLucene(doc.entrySet(),null));
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	385 }
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	386
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	387 public synchronized long nextId() throws LuanException, IOException {
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	388 if( ++id > idLim ) {
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	389 idLim += idBatch;
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	390 saveNextId(idLim);
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	391 wrote();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	392 }
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	393 return id;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	394 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	395
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	396 /*
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	397 public void backup(String zipFile) throws LuanException, IOException {
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	398 if( !zipFile.endsWith(".zip") )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	399 throw new LuanException("file "+zipFile+" doesn't end with '.zip'");
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	400 IndexCommit ic = snapshotDeletionPolicy.snapshot();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	401 try {
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	402 ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile));
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	403 for( String fileName : ic.getFileNames() ) {
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	404 out.putNextEntry(new ZipEntry(fileName));
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	405 FileInputStream in = new FileInputStream(new File(indexDir,fileName));
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	406 Utils.copyAll(in,out);
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	407 in.close();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	408 out.closeEntry();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	409 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	410 out.close();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	411 } finally {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	412 snapshotDeletionPolicy.release(ic);
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	413 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	414 }
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	415 */
1129 3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	416 public SnapshotDeletionPolicy snapshotDeletionPolicy() {
3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	417 return snapshotDeletionPolicy;
3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	418 }
3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	419
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	420 public Object snapshot(LuanFunction fn) throws LuanException, IOException {
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	421 IndexCommit ic = snapshotDeletionPolicy.snapshot();
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	422 try {
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	423 String dir = fsDir.getDirectory().toString();
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	424 LuanTable fileNames = new LuanTable(fn.luan(),new ArrayList(ic.getFileNames()));
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	425 return fn.call(dir,fileNames);
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	426 } finally {
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	427 snapshotDeletionPolicy.release(ic);
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	428 }
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	429 }
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	430
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	431
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	432
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	433 public String to_string() {
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	434 return writer.getDirectory().toString();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	435 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	436
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	437 public synchronized void close() throws IOException {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	438 try {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	439 doClose();
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	440 } catch(SQLException e) {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	441 throw new RuntimeException(e);
521 8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	442 }
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	443 isClosed = true;
521 8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	444 }
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	445
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	446 public void doClose() throws IOException, SQLException {
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	447 writer.close();
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	448 reader.close();
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	449 if( postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	450 postgresBackup.close();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	451 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	452
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	453 private void closeWriter() throws IOException {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	454 writeLock.lock();
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	455 try {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	456 writer.close();
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	457 } finally {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	458 writeLock.unlock();
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	459 }
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	460 }
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	461
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	462
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	463 private static class DocFn extends LuanFunction {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	464 final IndexSearcher searcher;
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	465 final Query query;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	466 int docID;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	467
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	468 DocFn(Luan luan,IndexSearcher searcher,Query query) {
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	469 super(luan);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	470 this.searcher = searcher;
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	471 this.query = query;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	472 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	473
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	474 @Override public Object call(Object[] args) throws LuanException {
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	475 try {
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	476 LuanTable doc = toTable(luan(),searcher.doc(docID));
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	477 if( args.length > 0 && "explain".equals(args[0]) ) {
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	478 Explanation explanation = searcher.explain(query,docID);
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	479 return new Object[]{doc,explanation};
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	480 } else {
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	481 return doc;
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	482 }
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	483 } catch(IOException e) {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	484 throw new LuanException(e);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	485 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	486 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	487 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	488
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	489 private static abstract class MyCollector extends Collector {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	490 int docBase;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	491 int i = 0;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	492
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	493 @Override public void setScorer(Scorer scorer) {}
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	494 @Override public void setNextReader(AtomicReaderContext context) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	495 this.docBase = context.docBase;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	496 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	497 @Override public boolean acceptsDocsOutOfOrder() {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	498 return true;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	499 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	500 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	501
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	502 private synchronized IndexSearcher openSearcher() throws IOException {
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	503 int gwc = writeCounter.get();
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	504 if( writeCount != gwc ) {
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	505 writeCount = gwc;
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	506 DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	507 if( newReader != null ) {
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	508 reader.decRef();
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	509 reader = newReader;
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	510 searcher = new IndexSearcher(reader);
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	511 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	512 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	513 reader.incRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	514 return searcher;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	515 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	516
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	517 // call in finally block
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	518 private static void close(IndexSearcher searcher) throws IOException {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	519 searcher.getIndexReader().decRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	520 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	521
591 790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	522 public void ensure_open() throws IOException {
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	523 close(openSearcher());
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	524 }
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	525
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	526 public int advanced_search( String queryStr, LuanFunction fn, Integer n, String sortStr )
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	527 throws LuanException, IOException, ParseException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	528 {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	529 Utils.checkNotNull(queryStr);
622 1a53333eb4d5 remove Lucene all_search_terms_must_match() since now query suffix "~a" handles this Franklin Schmidt <fschmidt@gmail.com> parents: 621 diff changeset	530 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	531 IndexSearcher searcher = threadLocalSearcher.get();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	532 boolean inTransaction = searcher != null;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	533 if( !inTransaction )
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	534 searcher = openSearcher();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	535 try {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	536 if( fn!=null && n==null ) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	537 if( sortStr != null )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	538 throw new LuanException("sort must be nil when n is nil");
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	539 final DocFn docFn = new DocFn(fn.luan(),searcher,query);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	540 MyCollector col = new MyCollector() {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	541 @Override public void collect(int doc) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	542 try {
547 0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	543 docFn.docID = docBase + doc;
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	544 fn.call(++i,docFn);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	545 } catch(LuanException e) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	546 throw new LuanRuntimeException(e);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	547 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	548 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	549 };
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	550 try {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	551 searcher.search(query,col);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	552 } catch(LuanRuntimeException e) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	553 throw (LuanException)e.getCause();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	554 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	555 return col.i;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	556 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	557 if( fn==null \|\| n==0 ) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	558 TotalHitCountCollector thcc = new TotalHitCountCollector();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	559 searcher.search(query,thcc);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	560 return thcc.getTotalHits();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	561 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	562 Sort sort = sortStr==null ? null : SaneQueryParser.parseSort(mfp,sortStr);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	563 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	564 final ScoreDoc[] scoreDocs = td.scoreDocs;
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	565 DocFn docFn = new DocFn(fn.luan(),searcher,query);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	566 for( int i=0; i<scoreDocs.length; i++ ) {
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	567 ScoreDoc scoreDoc = scoreDocs[i];
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	568 docFn.docID = scoreDoc.doc;
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	569 fn.call(i+1,docFn,scoreDoc.score);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	570 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	571 return td.totalHits;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	572 } finally {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	573 if( !inTransaction )
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	574 close(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	575 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	576 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	577
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	578 public Object search_in_transaction(LuanFunction fn) throws LuanException, IOException {
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	579 if( threadLocalSearcher.get() != null )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	580 throw new LuanException("can't nest search_in_transaction calls");
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	581 IndexSearcher searcher = openSearcher();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	582 threadLocalSearcher.set(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	583 try {
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	584 return fn.call();
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	585 } finally {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	586 threadLocalSearcher.set(null);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	587 close(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	588 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	589 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	590
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	591
796 6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	592 public FieldParser getIndexedFieldParser(String field) {
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	593 return mfp.fields.get(field);
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	594 }
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	595
796 6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	596 public void setIndexedFieldParser(String field,FieldParser fp) {
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	597 if( fp==null ) { // delete
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	598 mfp.fields.remove(field);
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	599 return;
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	600 }
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	601 mfp.fields.put( field, fp );
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	602 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	603
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	604
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	605 private IndexableField newField(String name,Object value,Set<String> indexed,Float boost)
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	606 throws LuanException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	607 {
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	608 boolean hasBoost = boost!=null;
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	609 IndexableField fld = newField2(name,value,indexed,hasBoost);
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	610 if( hasBoost )
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	611 ((Field)fld).setBoost(boost);
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	612 return fld;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	613 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	614
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	615 private IndexableField newField2(String name,Object value,Set<String> indexed,boolean hasBoost)
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	616 throws LuanException
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	617 {
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	618 Field.Store store = indexOnly.contains(name) ? Field.Store.NO : Field.Store.YES;
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	619 if( value instanceof String ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	620 String s = (String)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	621 FieldParser fp = mfp.fields.get(name);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	622 if( fp != null ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	623 if( fp instanceof StringFieldParser && fp != STRING_FIELD_PARSER ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	624 return new TextField(name, s, store);
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	625 } else if (hasBoost) {
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	626 // fuck you modern lucene developers
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	627 return new Field(name, s, store, Field.Index.NOT_ANALYZED);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	628 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	629 return new StringField(name, s, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	630 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	631 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	632 return new StoredField(name, s);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	633 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	634 } else if( value instanceof Integer ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	635 int i = (Integer)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	636 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	637 return new IntField(name, i, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	638 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	639 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	640 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	641 } else if( value instanceof Long ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	642 long i = (Long)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	643 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	644 return new LongField(name, i, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	645 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	646 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	647 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	648 } else if( value instanceof Double ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	649 double i = (Double)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	650 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	651 return new DoubleField(name, i, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	652 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	653 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	654 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	655 } else if( value instanceof byte[] ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	656 byte[] b = (byte[])value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	657 return new StoredField(name, b);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	658 } else
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	659 throw new LuanException("invalid value type "+value.getClass()+"' for '"+name+"'");
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	660 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	661
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	662 private Document toLucene(LuanTable table,LuanTable boosts) throws LuanException {
1347 643cf1c37723 move webserver to lib and bug fixes Franklin Schmidt <fschmidt@gmail.com> parents: 1346 diff changeset	663 return toLucene(table.iterable(),boosts);
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	664 }
efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	665
efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	666 private Document toLucene(Iterable<Map.Entry> iterable,LuanTable boosts) throws LuanException {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	667 Set<String> indexed = mfp.fields.keySet();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	668 Document doc = new Document();
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	669 for( Map.Entry<Object,Object> entry : iterable ) {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	670 Object key = entry.getKey();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	671 if( !(key instanceof String) )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	672 throw new LuanException("key must be string");
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	673 String name = (String)key;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	674 Object value = entry.getValue();
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	675 Float boost = null;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	676 if( boosts != null ) {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	677 Object obj = boosts.get(name);
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	678 if( obj != null ) {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	679 if( !(obj instanceof Number) )
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	680 throw new LuanException("boost '"+name+"' must be number");
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	681 boost = ((Number)obj).floatValue();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	682 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	683 }
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	684 if( !(value instanceof LuanTable) ) {
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	685 doc.add(newField( name, value, indexed, boost ));
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	686 } else { // list
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	687 LuanTable list = (LuanTable)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	688 for( Object el : list.asList() ) {
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	689 doc.add(newField( name, el, indexed, boost ));
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	690 }
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	691 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	692 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	693 return doc;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	694 }
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	695
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	696 private static Object getValue(IndexableField ifld) throws LuanException {
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	697 BytesRef br = ifld.binaryValue();
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	698 if( br != null )
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	699 return br.bytes;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	700 Number n = ifld.numericValue();
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	701 if( n != null )
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	702 return n;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	703 String s = ifld.stringValue();
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	704 if( s != null )
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	705 return s;
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	706 throw new LuanException("invalid field type for "+ifld);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	707 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	708
1333 25746915a241 merge Luan and LuanState Franklin Schmidt <fschmidt@gmail.com> parents: 1271 diff changeset	709 private static LuanTable toTable(Luan luan,Document doc) throws LuanException {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	710 if( doc==null )
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	711 return null;
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	712 LuanTable table = new LuanTable(luan);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	713 for( IndexableField ifld : doc ) {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	714 String name = ifld.name();
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	715 Object value = getValue(ifld);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	716 Object old = table.rawGet(name);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	717 if( old == null ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	718 table.rawPut(name,value);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	719 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	720 LuanTable list;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	721 if( old instanceof LuanTable ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	722 list = (LuanTable)old;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	723 } else {
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	724 list = new LuanTable(luan);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	725 list.rawPut(1,old);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	726 table.rawPut(name,list);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	727 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	728 list.rawPut(list.rawLength()+1,value);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	729 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	730 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	731 return table;
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	732 }
d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	733
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	734
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	735 private static final Formatter nullFormatter = new Formatter() {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	736 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	737 return originalText;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	738 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	739 };
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	740
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	741 public LuanFunction highlighter(String queryStr,final LuanFunction formatter,final Integer fragmentSize,String dotdotdot)
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	742 throws ParseException
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	743 {
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	744 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	745 Formatter fmt = new Formatter() {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	746 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	747 if( tokenGroup.getTotalScore() <= 0 )
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	748 return originalText;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	749 try {
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	750 return (String)Luan.first(formatter.call(originalText));
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	751 } catch(LuanException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	752 throw new LuanRuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	753 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	754 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	755 };
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	756 QueryScorer queryScorer = new QueryScorer(query);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	757 final Highlighter chooser = fragmentSize==null ? null : new Highlighter(nullFormatter,queryScorer);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	758 if( chooser != null )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	759 chooser.setTextFragmenter( new SimpleSpanFragmenter(queryScorer,fragmentSize) );
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	760 final Highlighter hl = new Highlighter(fmt,queryScorer);
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	761 hl.setTextFragmenter( new NullFragmenter() );
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	762 return new LuanFunction(false) { // ???
e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	763 @Override public String call(Object[] args) throws LuanException {
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	764 String text = (String)args[0];
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	765 try {
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	766 if( chooser != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	767 String s = chooser.getBestFragment(analyzer,null,text);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	768 if( s != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	769 if( dotdotdot != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	770 boolean atStart = text.startsWith(s);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	771 boolean atEnd = text.endsWith(s);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	772 if( !atStart )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	773 s = dotdotdot + s;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	774 if( !atEnd )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	775 s = s + dotdotdot;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	776 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	777 text = s;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	778 } else if( text.length() > fragmentSize ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	779 text = text.substring(0,fragmentSize);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	780 if( dotdotdot != null )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	781 text += "...";
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	782 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	783 }
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	784 String s = hl.getBestFragment(analyzer,null,text);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	785 return s!=null ? s : text;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	786 } catch(LuanRuntimeException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	787 throw (LuanException)e.getCause();
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	788 } catch(IOException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	789 throw new RuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	790 } catch(InvalidTokenOffsetsException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	791 throw new RuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	792 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	793 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	794 };
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	795 }
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	796
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	797 public int count_tokens(String text)
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	798 throws IOException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	799 {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	800 int n = 0;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	801 TokenStream ts = analyzer.tokenStream(null,text);
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	802 ts.reset();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	803 while( ts.incrementToken() ) {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	804 n++;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	805 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	806 ts.close();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	807 return n;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	808 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	809
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	810
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	811
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	812 public boolean hasPostgresBackup() {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	813 return postgresBackup != null;
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	814 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	815
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	816 public void rebuild_postgres_backup(LuanFunction completer)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	817 throws IOException, LuanException, SQLException
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	818 {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	819 luanLogger.info("start rebuild_postgres_backup");
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	820 writeLock.lock();
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	821 IndexSearcher searcher = openSearcher();
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	822 boolean ok = false;
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	823 try {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	824 postgresBackup.begin();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	825 postgresBackup.deleteAll();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	826 Query query = new PrefixQuery(new Term("id"));
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	827 MyCollector col = new MyCollector() {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	828 @Override public void collect(int iDoc) throws IOException {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	829 try {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	830 Document doc = searcher.doc( docBase + iDoc );
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	831 LuanTable tbl = toTable(completer.luan(),doc);
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	832 tbl = (LuanTable)completer.call(tbl);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	833 postgresBackup.add(tbl);
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	834 } catch(LuanException e) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	835 throw new LuanRuntimeException(e);
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	836 } catch(SQLException e) {
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	837 throw new RuntimeException(e);
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	838 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	839 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	840 };
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	841 try {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	842 searcher.search(query,col);
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	843 } catch(LuanRuntimeException e) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	844 throw (LuanException)e.getCause();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	845 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	846 ok = true;
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	847 postgresBackup.commit();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	848 } finally {
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	849 close(searcher);
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	850 if( !ok )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	851 postgresBackup.rollback();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	852 writeLock.unlock();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	853 }
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	854 luanLogger.info("end rebuild_postgres_backup");
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	855 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	856
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	857 public void restore_from_postgres()
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	858 throws IOException, LuanException, SQLException
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	859 {
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	860 if( postgresBackup!=null && wasCreated && !postgresBackup.wasCreated ) {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	861 luanLogger.error("restoring from postgres");
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	862 force_restore_from_postgres();
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	863 wasCreated = false;
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	864 }
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	865 }
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	866
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	867 public void force_restore_from_postgres()
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	868 throws IOException, LuanException, SQLException
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	869 {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	870 luanLogger.warn("start restore_from_postgres");
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	871 if( postgresBackup==null )
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	872 throw new NullPointerException();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	873 if( writeLock.isHeldByCurrentThread() )
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	874 throw new RuntimeException();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	875 writeLock.lock();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	876 boolean ok = false;
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	877 try {
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	878 writer.deleteAll();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	879 long nextId = postgresBackup.maxId() + 1;
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	880 postgresBackup.restoreLucene(this);
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	881 id = idLim = nextId;
1395 9dfff82dfc59 finish postgres work Franklin Schmidt <fschmidt@gmail.com> parents: 1393 diff changeset	882 saveNextId(nextId);
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	883 ok = true;
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	884 writer.commit();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	885 } finally {
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	886 if( !ok ) {
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	887 writer.rollback();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	888 reopen();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	889 }
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	890 wrote();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	891 writeLock.unlock();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	892 }
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	893 luanLogger.warn("end restore_from_postgres");
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	894 }
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	895
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	896 void restore(LuanTable doc)
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	897 throws LuanException, IOException
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	898 {
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	899 writer.addDocument(toLucene(doc,null));
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	900 }
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	901
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	902 public void check(LuanFunction completer) throws IOException, SQLException, LuanException {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	903 luanLogger.info("start check");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	904 CheckIndex.Status status = new CheckIndex(fsDir).checkIndex();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	905 if( !status.clean )
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	906 luanLogger.error("index not clean");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	907 if( postgresBackup != null )
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	908 checkPostgres(completer);
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	909 luanLogger.info("end check");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	910 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	911
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	912 private void checkPostgres(LuanFunction completer) throws IOException, SQLException, LuanException {
1395 9dfff82dfc59 finish postgres work Franklin Schmidt <fschmidt@gmail.com> parents: 1393 diff changeset	913 luanLogger.info("start postgres check");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	914 final PostgresBackup.Checker postgresChecker;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	915 final IndexSearcher searcher;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	916 writeLock.lock();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	917 try {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	918 postgresChecker = postgresBackup.newChecker();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	919 searcher = openSearcher();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	920 } finally {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	921 writeLock.unlock();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	922 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	923 try {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	924 final List<Long> idsLucene = new ArrayList<Long>();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	925 Query query = new PrefixQuery(new Term("id"));
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	926 MyCollector col = new MyCollector() {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	927 @Override public void collect(int iDoc) throws IOException {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	928 Document doc = searcher.doc( docBase + iDoc );
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	929 Long id = (Long)doc.getField("id").numericValue();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	930 idsLucene.add(id);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	931 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	932 };
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	933 searcher.search(query,col);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	934 Collections.sort(idsLucene);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	935 final List<Long> idsPostgres = postgresChecker.getIds();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	936 final int nLucene = idsLucene.size();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	937 final int nPostgres = idsPostgres.size();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	938 int iLucene = 0;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	939 int iPostgres = 0;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	940 LuanToString lts = new LuanToString();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	941 lts.strict = true;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	942 lts.numberTypes = true;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	943 while( iLucene < nLucene && iPostgres < nPostgres ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	944 long idLucene = idsLucene.get(iLucene);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	945 long idPostgres = idsPostgres.get(iPostgres);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	946 if( idLucene < idPostgres ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	947 iLucene++;
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	948 luanLogger.error("id "+idLucene+" found in lucene but not postgres");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	949 } else if( idLucene > idPostgres ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	950 iPostgres++;
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	951 luanLogger.error("id "+idPostgres+" found in postgres but not lucene");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	952 } else { // ==
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	953 LuanTable docPostgres = postgresChecker.getDoc(idPostgres);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	954 TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	955 if( td.totalHits != 1 ) throw new RuntimeException();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	956 Document doc = searcher.doc( td.scoreDocs[0].doc );
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	957 LuanTable docLucene = toTable(completer.luan(),doc);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	958 docLucene = (LuanTable)completer.call(docLucene);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	959 if( !equal(docPostgres,docLucene) ) {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	960 luanLogger.error("id "+idLucene+" not equal");
cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	961 luanLogger.error("lucene = "+lts.toString(docLucene));
cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	962 luanLogger.error("postgres = "+lts.toString(docPostgres));
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	963 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	964 iLucene++;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	965 iPostgres++;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	966 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	967 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	968 while( iLucene < nLucene ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	969 long idLucene = idsLucene.get(iLucene++);
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	970 luanLogger.error("id "+idLucene+" found in lucene but not postgres");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	971 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	972 while( iPostgres < nPostgres ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	973 long idPostgres = idsPostgres.get(iPostgres++);
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	974 luanLogger.error("id "+idPostgres+" found in postgres but not lucene");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	975 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	976 } finally {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	977 close(searcher);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	978 postgresChecker.close();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	979 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	980 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	981
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	982 private boolean equal(LuanTable t1,LuanTable t2) throws LuanException {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	983 return t1.asMap().equals(t2.asMap());
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	984 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	985
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	986 }

Mercurial Hosting > luan

annotate src/luan/modules/lucene/LuceneIndex.java @ 1397:0dc9837c16be