Mercurial Hosting > luan

annotate src/luan/modules/lucene/LuceneIndex.java @ 1540:79f67662d3e7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
delete next_id
author Franklin Schmidt <fschmidt@gmail.com>
date Sun, 13 Sep 2020 15:32:51 -0600
parents c27dc6af87ca
children dc23c96f5021
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
1 package luan.modules.lucene;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
2
527
Franklin Schmidt <fschmidt@gmail.com>
parents: 524
diff changeset
3 import java.io.Closeable;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
4 import java.io.File;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
5 import java.io.FileOutputStream;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
6 import java.io.FileInputStream;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
7 import java.io.IOException;
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
8 import java.lang.ref.Reference;
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
9 import java.lang.ref.WeakReference;
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
10 import java.sql.SQLException;
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
11 import java.util.Arrays;
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
12 import java.util.Iterator;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
13 import java.util.Map;
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
14 import java.util.HashMap;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
15 import java.util.List;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
16 import java.util.ArrayList;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
17 import java.util.Set;
618
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
18 import java.util.HashSet;
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
19 import java.util.Collections;
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
20 import java.util.concurrent.atomic.AtomicInteger;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
21 import java.util.concurrent.locks.Lock;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
22 import java.util.concurrent.locks.ReentrantLock;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
23 import java.util.zip.ZipOutputStream;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
24 import java.util.zip.ZipEntry;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
25 import org.apache.lucene.analysis.Analyzer;
1341
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
26 import org.apache.lucene.analysis.TokenStream;
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
27 import org.apache.lucene.analysis.core.KeywordAnalyzer;
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
28 import org.apache.lucene.analysis.en.EnglishAnalyzer;
233
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
29 import org.apache.lucene.document.Document;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
30 import org.apache.lucene.document.Field;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
31 import org.apache.lucene.document.StoredField;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
32 import org.apache.lucene.document.StringField;
599
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
33 import org.apache.lucene.document.TextField;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
34 import org.apache.lucene.document.IntField;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
35 import org.apache.lucene.document.LongField;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
36 import org.apache.lucene.document.DoubleField;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
37 import org.apache.lucene.index.IndexableField;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
38 import org.apache.lucene.index.IndexWriter;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
39 import org.apache.lucene.index.IndexWriterConfig;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
40 import org.apache.lucene.index.DirectoryReader;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
41 import org.apache.lucene.index.Term;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
42 import org.apache.lucene.index.SnapshotDeletionPolicy;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
43 import org.apache.lucene.index.IndexCommit;
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
44 import org.apache.lucene.index.AtomicReaderContext;
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
45 import org.apache.lucene.index.CheckIndex;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
46 import org.apache.lucene.store.Directory;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
47 import org.apache.lucene.store.FSDirectory;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
48 import org.apache.lucene.util.Version;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
49 import org.apache.lucene.util.BytesRef;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
50 import org.apache.lucene.util.NumericUtils;
312
d34be4588556 add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 303
diff changeset
51 import org.apache.lucene.search.Query;
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
52 import org.apache.lucene.search.PrefixQuery;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
53 import org.apache.lucene.search.TermQuery;
1539
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
54 import org.apache.lucene.search.MatchAllDocsQuery;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
55 import org.apache.lucene.search.TopDocs;
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
56 import org.apache.lucene.search.Sort;
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
57 import org.apache.lucene.search.SortField;
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
58 import org.apache.lucene.search.IndexSearcher;
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
59 import org.apache.lucene.search.TotalHitCountCollector;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
60 import org.apache.lucene.search.ScoreDoc;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
61 import org.apache.lucene.search.Collector;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
62 import org.apache.lucene.search.Scorer;
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
63 import org.apache.lucene.search.Explanation;
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
64 import org.apache.lucene.search.highlight.Formatter;
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
65 import org.apache.lucene.search.highlight.Highlighter;
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
66 import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
1248
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
67 import org.apache.lucene.search.highlight.Fragmenter;
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
68 import org.apache.lucene.search.highlight.NullFragmenter;
1248
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
69 import org.apache.lucene.search.highlight.SimpleSpanFragmenter;
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
70 import org.apache.lucene.search.highlight.QueryScorer;
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
71 import org.apache.lucene.search.highlight.TokenGroup;
1474
13cbce740e1e LowercaseAnalyzer
Franklin Schmidt <fschmidt@gmail.com>
parents: 1459
diff changeset
72 import goodjava.lucene.analysis.LowercaseAnalyzer;
1459
b04b8fc5f4f4 GoodQueryParser
Franklin Schmidt <fschmidt@gmail.com>
parents: 1458
diff changeset
73 import goodjava.lucene.queryparser.GoodQueryParser;
1458
6b6c11c9164e goodjava.lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1454
diff changeset
74 import goodjava.lucene.queryparser.FieldParser;
6b6c11c9164e goodjava.lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1454
diff changeset
75 import goodjava.lucene.queryparser.MultiFieldParser;
6b6c11c9164e goodjava.lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1454
diff changeset
76 import goodjava.lucene.queryparser.StringFieldParser;
6b6c11c9164e goodjava.lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1454
diff changeset
77 import goodjava.lucene.queryparser.NumberFieldParser;
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
78 import goodjava.lucene.api.GoodIndexWriter;
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
79 import goodjava.lucene.api.LuceneIndexWriter;
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
80 import goodjava.lucene.api.GoodIndexWriterConfig;
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
81 import goodjava.lucene.api.LuceneUtils;
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
82 import goodjava.lucene.logging.LoggingIndexWriter;
1402
27efb1fcbcb5 move luan.lib to goodjava
Franklin Schmidt <fschmidt@gmail.com>
parents: 1399
diff changeset
83 import goodjava.parser.ParseException;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
84 import luan.modules.Utils;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
85 import luan.Luan;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
86 import luan.LuanTable;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
87 import luan.LuanFunction;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
88 import luan.LuanException;
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
89 import luan.LuanRuntimeException;
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
90 import luan.modules.parsers.LuanToString;
1402
27efb1fcbcb5 move luan.lib to goodjava
Franklin Schmidt <fschmidt@gmail.com>
parents: 1399
diff changeset
91 import goodjava.logging.Logger;
27efb1fcbcb5 move luan.lib to goodjava
Franklin Schmidt <fschmidt@gmail.com>
parents: 1399
diff changeset
92 import goodjava.logging.LoggerFactory;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
93
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
94
1398
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
95 public final class LuceneIndex {
1454
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
96 private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class);
521
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
97
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
98 private static Map<String,Reference<LuceneIndex>> indexes = new HashMap<String,Reference<LuceneIndex>>();
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
99
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
100 public static LuceneIndex getLuceneIndex(Luan luan,File indexDir,LuanTable options)
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
101 throws LuanException, IOException, ClassNotFoundException, SQLException
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
102 {
1369
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1347
diff changeset
103 String key = indexDir.getCanonicalPath();
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
104 synchronized(indexes) {
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
105 Reference<LuceneIndex> ref = indexes.get(key);
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
106 if( ref != null ) {
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
107 LuceneIndex li = ref.get();
1398
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
108 if( li != null ) {
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
109 Object version = options.get("version");
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
110 if( version==null || version.equals(li.version) )
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
111 return li;
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
112 li.closeWriter();
1398
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
113 }
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
114 }
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
115 LuceneIndex li = new LuceneIndex(luan,indexDir,options);
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
116 indexes.put(key, new WeakReference<LuceneIndex>(li));
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
117 return li;
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
118 }
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
119 }
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
120
1398
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
121 private static final Version luceneVersion = Version.LUCENE_4_9;
1539
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
122 // private static final String FLD_NEXT_ID = "nextId";
599
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
123 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer());
1532
060ff7695317 Lucene.type.lowercase
Franklin Schmidt <fschmidt@gmail.com>
parents: 1530
diff changeset
124 public static final StringFieldParser LOWERCASE_FIELD_PARSER = new StringFieldParser(new LowercaseAnalyzer(luceneVersion));
1398
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
125 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(luceneVersion));
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
126 private static final SortField ID_SORT = new SortField("id",SortField.Type.LONG);
1539
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
127 private static final SortField ID_DESC_SORT = new SortField("id",SortField.Type.LONG,true);
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
128
1398
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
129 private final Object version;
1393
cc0dbca576dc better logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1392
diff changeset
130
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
131 private final ReentrantLock writeLock = new ReentrantLock();
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
132 private final File indexDir;
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
133 private GoodIndexWriter writer;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
134 private DirectoryReader reader;
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
135 private IndexSearcher searcher;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
136 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>();
599
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
137 private final MultiFieldParser mfp;
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
138 private final Analyzer analyzer; // ???
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
139
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
140 private FSDirectory fsDir;
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
141 private int writeCount;
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
142 private AtomicInteger writeCounter = new AtomicInteger();
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
143 private final GoodIndexWriterConfig config;
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
144
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
145 private final PostgresBackup postgresBackup;
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
146 private boolean wasCreated;
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
147 private final File logDir;
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
148
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
149 private LuceneIndex(Luan luan,File indexDir,LuanTable options)
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
150 throws LuanException, IOException, ClassNotFoundException, SQLException
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
151 {
1420
225808b90cee options handling
Franklin Schmidt <fschmidt@gmail.com>
parents: 1410
diff changeset
152 options = new LuanTable(options);
225808b90cee options handling
Franklin Schmidt <fschmidt@gmail.com>
parents: 1410
diff changeset
153 this.version = options.remove("version");
225808b90cee options handling
Franklin Schmidt <fschmidt@gmail.com>
parents: 1410
diff changeset
154 FieldParser defaultFieldParser = (FieldParser)options.remove("default_type");
225808b90cee options handling
Franklin Schmidt <fschmidt@gmail.com>
parents: 1410
diff changeset
155 LuanTable defaultFieldsTbl = Utils.removeTable(options,"default_fields");
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
156 String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]);
1420
225808b90cee options handling
Franklin Schmidt <fschmidt@gmail.com>
parents: 1410
diff changeset
157 LuanTable postgresSpec = Utils.removeTable(options,"postgres_spec");
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
158 LuanFunction supplementer = Utils.removeFunction(options,"supplementer");
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
159 logDir = (File)options.remove("log_dir");
1420
225808b90cee options handling
Franklin Schmidt <fschmidt@gmail.com>
parents: 1410
diff changeset
160 Utils.checkEmpty(options);
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
161
599
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
162 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
163 mfp.fields.put( "type", STRING_FIELD_PARSER );
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
164 mfp.fields.put( "id", NumberFieldParser.LONG );
233
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
165 this.indexDir = indexDir;
599
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
166 Analyzer analyzer = STRING_FIELD_PARSER.analyzer;
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
167 if( defaultFieldParser instanceof StringFieldParser ) {
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
168 StringFieldParser sfp = (StringFieldParser)defaultFieldParser;
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
169 analyzer = sfp.analyzer;
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
170 }
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
171 this.analyzer = analyzer;
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
172 this.config = new SupplementingConfig(luceneVersion,mfp,supplementer);
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
173 wasCreated = reopen();
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
174 if( postgresSpec == null ) {
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
175 postgresBackup = null;
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
176 } else {
1420
225808b90cee options handling
Franklin Schmidt <fschmidt@gmail.com>
parents: 1410
diff changeset
177 postgresBackup = new PostgresBackup(luan,postgresSpec);
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
178 if( !wasCreated && postgresBackup.wasCreated ) {
1454
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
179 logger.error("rebuilding postgres backup");
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
180 rebuild_postgres_backup(luan);
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
181 /*
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
182 } else if( wasCreated && !postgresBackup.wasCreated ) {
1454
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
183 logger.error("restoring from postgres");
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
184 restore_from_postgres();
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
185 */
1390
179c4882c6b6 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1388
diff changeset
186 }
179c4882c6b6 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1388
diff changeset
187 }
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
188 }
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
189
1390
179c4882c6b6 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1388
diff changeset
190 public boolean reopen() throws IOException {
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
191 fsDir = FSDirectory.open(indexDir);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
192 boolean wasCreated = !fsDir.getDirectory().exists();
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
193 writer = new LuceneIndexWriter(fsDir,config);
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
194 if( logDir != null )
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
195 writer = new LoggingIndexWriter((LuceneIndexWriter)writer,logDir);
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
196 reader = DirectoryReader.open(fsDir);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
197 searcher = new IndexSearcher(reader);
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
198 initId();
1390
179c4882c6b6 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1388
diff changeset
199 return wasCreated;
233
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
200 }
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
201
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
202 private void wrote() {
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
203 writeCounter.incrementAndGet();
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
204 }
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
205
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
206 public void delete_all() throws IOException, SQLException {
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
207 boolean commit = !writeLock.isHeldByCurrentThread();
252
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
208 writeLock.lock();
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
209 try {
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
210 writer.deleteAll();
1539
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
211 id = 0;
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
212 if( postgresBackup != null )
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
213 postgresBackup.deleteAll();
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
214 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
215 } finally {
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
216 wrote();
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
217 writeLock.unlock();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
218 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
219 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
220
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
221 private static Term term(String key,long value) {
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
222 BytesRef br = new BytesRef();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
223 NumericUtils.longToPrefixCoded(value,0,br);
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
224 return new Term(key,br);
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
225 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
226
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
227 private void backupDelete(Query query)
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
228 throws IOException, SQLException, LuanException
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
229 {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
230 if( postgresBackup != null ) {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
231 final List<Long> ids = new ArrayList<Long>();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
232 IndexSearcher searcher = openSearcher();
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
233 try {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
234 MyCollector col = new MyCollector() {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
235 @Override public void collect(int iDoc) throws IOException {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
236 Document doc = searcher.doc( docBase + iDoc );
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
237 Long id = (Long)doc.getField("id").numericValue();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
238 ids.add(id);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
239 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
240 };
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
241 searcher.search(query,col);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
242 } finally {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
243 close(searcher);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
244 }
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
245 postgresBackup.begin();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
246 for( Long id : ids ) {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
247 postgresBackup.delete(id);
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
248 }
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
249 postgresBackup.commit();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
250 }
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
251 }
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
252
1346
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
253 public void delete(String queryStr)
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
254 throws IOException, ParseException, SQLException, LuanException
1341
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
255 {
1459
b04b8fc5f4f4 GoodQueryParser
Franklin Schmidt <fschmidt@gmail.com>
parents: 1458
diff changeset
256 Query query = GoodQueryParser.parseQuery(mfp,queryStr);
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
257
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
258 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
259 writeLock.lock();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
260 try {
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
261 backupDelete(query);
547
0be287ab0309 add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents: 546
diff changeset
262 writer.deleteDocuments(query);
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
263 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
264 } finally {
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
265 wrote();
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
266 writeLock.unlock();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
267 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
268 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
269
1535
e73b72a510b4 Lucene reindex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
270 public void reindex(String queryStr)
e73b72a510b4 Lucene reindex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
271 throws IOException, ParseException
e73b72a510b4 Lucene reindex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
272 {
e73b72a510b4 Lucene reindex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
273 Query query = GoodQueryParser.parseQuery(mfp,queryStr);
e73b72a510b4 Lucene reindex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
274
e73b72a510b4 Lucene reindex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
275 boolean commit = !writeLock.isHeldByCurrentThread();
e73b72a510b4 Lucene reindex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
276 writeLock.lock();
e73b72a510b4 Lucene reindex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
277 try {
e73b72a510b4 Lucene reindex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
278 writer.reindexDocuments("id",query);
e73b72a510b4 Lucene reindex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
279 if(commit) writer.commit();
e73b72a510b4 Lucene reindex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
280 } finally {
e73b72a510b4 Lucene reindex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
281 wrote();
e73b72a510b4 Lucene reindex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
282 writeLock.unlock();
e73b72a510b4 Lucene reindex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
283 }
e73b72a510b4 Lucene reindex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
284 }
e73b72a510b4 Lucene reindex
Franklin Schmidt <fschmidt@gmail.com>
parents: 1533
diff changeset
285
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
286 public void save(LuanTable doc)
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
287 throws LuanException, IOException, SQLException
1341
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
288 {
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
289 Object obj = doc.get("id");
601
d36027b41570 better error message
Franklin Schmidt <fschmidt@gmail.com>
parents: 599
diff changeset
290 Long id;
d36027b41570 better error message
Franklin Schmidt <fschmidt@gmail.com>
parents: 599
diff changeset
291 try {
d36027b41570 better error message
Franklin Schmidt <fschmidt@gmail.com>
parents: 599
diff changeset
292 id = (Long)obj;
d36027b41570 better error message
Franklin Schmidt <fschmidt@gmail.com>
parents: 599
diff changeset
293 } catch(ClassCastException e) {
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
294 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName());
601
d36027b41570 better error message
Franklin Schmidt <fschmidt@gmail.com>
parents: 599
diff changeset
295 }
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
296
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
297 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
298 writeLock.lock();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
299 try {
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
300 if( id == null ) {
1539
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
301 id = ++this.id;
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
302 doc.put("id",id);
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
303 if( postgresBackup != null )
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
304 postgresBackup.add(doc);
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
305 writer.addDocument(toLucene(doc));
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
306 } else {
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
307 if( postgresBackup != null )
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
308 postgresBackup.update(doc);
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
309 writer.updateDocument( "id", toLucene(doc) );
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
310 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
311 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
312 } finally {
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
313 wrote();
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
314 writeLock.unlock();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
315 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
316 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
317
1533
28f06ac6ce39 add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1532
diff changeset
318 public boolean is_in_transaction() {
28f06ac6ce39 add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1532
diff changeset
319 return writeLock.isHeldByCurrentThread();
28f06ac6ce39 add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1532
diff changeset
320 }
28f06ac6ce39 add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1532
diff changeset
321
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
322 public Object run_in_transaction(LuanFunction fn)
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
323 throws IOException, LuanException, SQLException
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
324 {
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
325 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
326 writeLock.lock();
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
327 boolean ok = false;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
328 try {
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
329 if( commit && postgresBackup != null )
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
330 postgresBackup.begin();
1379
87a3738d7cc5 run_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1369
diff changeset
331 Object rtn = fn.call();
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
332 ok = true;
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
333 if(commit) {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
334 if( postgresBackup != null )
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
335 postgresBackup.commit();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
336 writer.commit();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
337 }
1379
87a3738d7cc5 run_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1369
diff changeset
338 return rtn;
252
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
339 } finally {
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
340 if( !ok && commit ) {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
341 if( postgresBackup != null )
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
342 postgresBackup.rollback();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
343 writer.rollback();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
344 reopen();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
345 }
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
346 wrote();
252
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
347 writeLock.unlock();
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
348 }
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
349 }
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
350
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
351 // ???
1379
87a3738d7cc5 run_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1369
diff changeset
352 public Object run_in_lock(LuanFunction fn) throws IOException, LuanException {
756
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
353 if( writeLock.isHeldByCurrentThread() )
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
354 throw new RuntimeException();
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
355 writeLock.lock();
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
356 try {
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
357 synchronized(this) {
1379
87a3738d7cc5 run_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1369
diff changeset
358 return fn.call();
756
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
359 }
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
360 } finally {
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
361 wrote();
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
362 writeLock.unlock();
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
363 }
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
364 }
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
365
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
366
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
367 private long id;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
368
1388
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
369 private void initId() throws IOException {
1539
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
370 // TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1);
1540
79f67662d3e7 delete next_id
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
371 writer.deleteDocuments(new TermQuery(new Term("type","next_id")));
79f67662d3e7 delete next_id
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
372 writer.commit();
79f67662d3e7 delete next_id
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
373
1539
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
374 TopDocs td = searcher.search(new MatchAllDocsQuery(),1,new Sort(ID_DESC_SORT));
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
375 switch(td.scoreDocs.length) {
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
376 case 0:
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
377 id = 0;
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
378 break;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
379 case 1:
1539
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
380 id = (Long)searcher.doc(td.scoreDocs[0].doc).getField("id").numericValue();
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
381 break;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
382 default:
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
383 throw new RuntimeException();
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
384 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
385 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
386
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
387 /*
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
388 public void backup(String zipFile) throws LuanException, IOException {
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
389 if( !zipFile.endsWith(".zip") )
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
390 throw new LuanException("file "+zipFile+" doesn't end with '.zip'");
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
391 IndexCommit ic = snapshotDeletionPolicy.snapshot();
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
392 try {
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
393 ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile));
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
394 for( String fileName : ic.getFileNames() ) {
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
395 out.putNextEntry(new ZipEntry(fileName));
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
396 FileInputStream in = new FileInputStream(new File(indexDir,fileName));
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
397 Utils.copyAll(in,out);
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
398 in.close();
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
399 out.closeEntry();
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
400 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
401 out.close();
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
402 } finally {
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
403 snapshotDeletionPolicy.release(ic);
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
404 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
405 }
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
406 */
1129
3234a14bb1f8 minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents: 1111
diff changeset
407 public SnapshotDeletionPolicy snapshotDeletionPolicy() {
1539
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
408 return (SnapshotDeletionPolicy)writer.getLuceneIndexWriter().getConfig().getIndexDeletionPolicy();
1129
3234a14bb1f8 minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents: 1111
diff changeset
409 }
3234a14bb1f8 minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents: 1111
diff changeset
410
1346
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
411 public Object snapshot(LuanFunction fn) throws LuanException, IOException {
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
412 SnapshotDeletionPolicy snapshotDeletionPolicy = snapshotDeletionPolicy();
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
413 IndexCommit ic = snapshotDeletionPolicy.snapshot();
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
414 try {
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
415 String dir = fsDir.getDirectory().toString();
1346
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
416 LuanTable fileNames = new LuanTable(fn.luan(),new ArrayList(ic.getFileNames()));
1335
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
417 return fn.call(dir,fileNames);
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
418 } finally {
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
419 snapshotDeletionPolicy.release(ic);
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
420 }
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
421 }
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
422
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
423
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
424
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
425 public String to_string() {
1539
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
426 return writer.getLuceneIndexWriter().getDirectory().toString();
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
427 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
428
1398
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
429 protected void finalize() throws Throwable {
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
430 close();
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
431 super.finalize();
521
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
432 }
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
433
1398
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
434 public void close() throws IOException, SQLException {
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
435 closeWriter();
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
436 reader.close();
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
437 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
438
1398
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
439 private void closeWriter() throws IOException, SQLException {
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
440 writeLock.lock();
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
441 try {
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
442 writer.close();
1398
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
443 if( postgresBackup != null )
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
444 postgresBackup.close();
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
445 } finally {
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
446 writeLock.unlock();
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
447 }
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
448 }
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
449
312
d34be4588556 add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 303
diff changeset
450
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
451 private static class DocFn extends LuanFunction {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
452 final IndexSearcher searcher;
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
453 final Query query;
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
454 int docID;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
455
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
456 DocFn(Luan luan,IndexSearcher searcher,Query query) {
1335
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
457 super(luan);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
458 this.searcher = searcher;
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
459 this.query = query;
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
460 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
461
1335
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
462 @Override public Object call(Object[] args) throws LuanException {
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
463 try {
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
464 LuanTable doc = toTable(luan(),searcher.doc(docID));
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
465 if( args.length > 0 && "explain".equals(args[0]) ) {
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
466 Explanation explanation = searcher.explain(query,docID);
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
467 return new Object[]{doc,explanation};
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
468 } else {
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
469 return doc;
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
470 }
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
471 } catch(IOException e) {
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
472 throw new LuanException(e);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
473 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
474 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
475 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
476
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
477 private static abstract class MyCollector extends Collector {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
478 int docBase;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
479 int i = 0;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
480
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
481 @Override public void setScorer(Scorer scorer) {}
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
482 @Override public void setNextReader(AtomicReaderContext context) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
483 this.docBase = context.docBase;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
484 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
485 @Override public boolean acceptsDocsOutOfOrder() {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
486 return true;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
487 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
488 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
489
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
490 private synchronized IndexSearcher openSearcher() throws IOException {
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
491 int gwc = writeCounter.get();
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
492 if( writeCount != gwc ) {
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
493 writeCount = gwc;
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
494 DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
1539
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
495 // DirectoryReader newReader = DirectoryReader.openIfChanged(reader,writer.getLuceneIndexWriter(),true);
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
496 if( newReader != null ) {
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
497 reader.decRef();
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
498 reader = newReader;
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
499 searcher = new IndexSearcher(reader);
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
500 }
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
501 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
502 reader.incRef();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
503 return searcher;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
504 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
505
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
506 // call in finally block
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
507 private static void close(IndexSearcher searcher) throws IOException {
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
508 searcher.getIndexReader().decRef();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
509 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
510
591
790d5de23042 add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents: 578
diff changeset
511 public void ensure_open() throws IOException {
790d5de23042 add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents: 578
diff changeset
512 close(openSearcher());
790d5de23042 add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents: 578
diff changeset
513 }
790d5de23042 add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents: 578
diff changeset
514
1346
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
515 public int advanced_search( String queryStr, LuanFunction fn, Integer n, String sortStr )
1341
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
516 throws LuanException, IOException, ParseException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
517 {
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
518 Utils.checkNotNull(queryStr);
1459
b04b8fc5f4f4 GoodQueryParser
Franklin Schmidt <fschmidt@gmail.com>
parents: 1458
diff changeset
519 Query query = GoodQueryParser.parseQuery(mfp,queryStr);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
520 IndexSearcher searcher = threadLocalSearcher.get();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
521 boolean inTransaction = searcher != null;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
522 if( !inTransaction )
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
523 searcher = openSearcher();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
524 try {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
525 if( fn!=null && n==null ) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
526 if( sortStr != null )
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
527 throw new LuanException("sort must be nil when n is nil");
1346
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
528 final DocFn docFn = new DocFn(fn.luan(),searcher,query);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
529 MyCollector col = new MyCollector() {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
530 @Override public void collect(int doc) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
531 try {
547
0be287ab0309 add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents: 546
diff changeset
532 docFn.docID = docBase + doc;
1335
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
533 fn.call(++i,docFn);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
534 } catch(LuanException e) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
535 throw new LuanRuntimeException(e);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
536 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
537 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
538 };
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
539 try {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
540 searcher.search(query,col);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
541 } catch(LuanRuntimeException e) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
542 throw (LuanException)e.getCause();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
543 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
544 return col.i;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
545 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
546 if( fn==null || n==0 ) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
547 TotalHitCountCollector thcc = new TotalHitCountCollector();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
548 searcher.search(query,thcc);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
549 return thcc.getTotalHits();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
550 }
1459
b04b8fc5f4f4 GoodQueryParser
Franklin Schmidt <fschmidt@gmail.com>
parents: 1458
diff changeset
551 Sort sort = sortStr==null ? null : GoodQueryParser.parseSort(mfp,sortStr);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
552 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
553 final ScoreDoc[] scoreDocs = td.scoreDocs;
1346
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
554 DocFn docFn = new DocFn(fn.luan(),searcher,query);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
555 for( int i=0; i<scoreDocs.length; i++ ) {
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
556 ScoreDoc scoreDoc = scoreDocs[i];
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
557 docFn.docID = scoreDoc.doc;
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
558 fn.call(i+1,docFn,scoreDoc.score);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
559 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
560 return td.totalHits;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
561 } finally {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
562 if( !inTransaction )
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
563 close(searcher);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
564 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
565 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
566
1335
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
567 public Object search_in_transaction(LuanFunction fn) throws LuanException, IOException {
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
568 if( threadLocalSearcher.get() != null )
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
569 throw new LuanException("can't nest search_in_transaction calls");
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
570 IndexSearcher searcher = openSearcher();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
571 threadLocalSearcher.set(searcher);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
572 try {
1335
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
573 return fn.call();
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
574 } finally {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
575 threadLocalSearcher.set(null);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
576 close(searcher);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
577 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
578 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
579
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
580
796
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
581 public FieldParser getIndexedFieldParser(String field) {
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
582 return mfp.fields.get(field);
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
583 }
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
584
796
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
585 public void setIndexedFieldParser(String field,FieldParser fp) {
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
586 if( fp==null ) { // delete
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
587 mfp.fields.remove(field);
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
588 return;
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
589 }
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
590 mfp.fields.put( field, fp );
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
591 }
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
592
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
593 static Map<String,Object> toLucene(LuanTable table) throws LuanException {
1529
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
594 return SupplementingConfig.toLucene(table);
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
595 }
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
596
1333
25746915a241 merge Luan and LuanState
Franklin Schmidt <fschmidt@gmail.com>
parents: 1271
diff changeset
597 private static LuanTable toTable(Luan luan,Document doc) throws LuanException {
1529
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
598 return doc==null ? null : SupplementingConfig.toTable(luan,LuceneUtils.toMap(doc));
312
d34be4588556 add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 303
diff changeset
599 }
d34be4588556 add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 303
diff changeset
600
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
601
1248
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
602 private static final Formatter nullFormatter = new Formatter() {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
603 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
604 return originalText;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
605 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
606 };
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
607
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
608 public LuanFunction highlighter(String queryStr,final LuanFunction formatter,final Integer fragmentSize,String dotdotdot)
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
609 throws ParseException
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
610 {
1459
b04b8fc5f4f4 GoodQueryParser
Franklin Schmidt <fschmidt@gmail.com>
parents: 1458
diff changeset
611 Query query = GoodQueryParser.parseQuery(mfp,queryStr);
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
612 Formatter fmt = new Formatter() {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
613 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
1435
Franklin Schmidt <fschmidt@gmail.com>
parents: 1430
diff changeset
614 if( tokenGroup.getTotalScore() <= 0 )
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
615 return originalText;
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
616 try {
1335
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
617 return (String)Luan.first(formatter.call(originalText));
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
618 } catch(LuanException e) {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
619 throw new LuanRuntimeException(e);
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
620 }
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
621 }
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
622 };
1248
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
623 QueryScorer queryScorer = new QueryScorer(query);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
624 final Highlighter chooser = fragmentSize==null ? null : new Highlighter(nullFormatter,queryScorer);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
625 if( chooser != null )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
626 chooser.setTextFragmenter( new SimpleSpanFragmenter(queryScorer,fragmentSize) );
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
627 final Highlighter hl = new Highlighter(fmt,queryScorer);
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
628 hl.setTextFragmenter( new NullFragmenter() );
1335
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
629 return new LuanFunction(false) { // ???
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
630 @Override public String call(Object[] args) throws LuanException {
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
631 String text = (String)args[0];
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
632 try {
1248
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
633 if( chooser != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
634 String s = chooser.getBestFragment(analyzer,null,text);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
635 if( s != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
636 if( dotdotdot != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
637 boolean atStart = text.startsWith(s);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
638 boolean atEnd = text.endsWith(s);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
639 if( !atStart )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
640 s = dotdotdot + s;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
641 if( !atEnd )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
642 s = s + dotdotdot;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
643 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
644 text = s;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
645 } else if( text.length() > fragmentSize ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
646 text = text.substring(0,fragmentSize);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
647 if( dotdotdot != null )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
648 text += "...";
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
649 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
650 }
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
651 String s = hl.getBestFragment(analyzer,null,text);
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
652 return s!=null ? s : text;
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
653 } catch(LuanRuntimeException e) {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
654 throw (LuanException)e.getCause();
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
655 } catch(IOException e) {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
656 throw new RuntimeException(e);
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
657 } catch(InvalidTokenOffsetsException e) {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
658 throw new RuntimeException(e);
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
659 }
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
660 }
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
661 };
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
662 }
1341
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
663
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
664 public int count_tokens(String text)
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
665 throws IOException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
666 {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
667 int n = 0;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
668 TokenStream ts = analyzer.tokenStream(null,text);
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
669 ts.reset();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
670 while( ts.incrementToken() ) {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
671 n++;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
672 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
673 ts.close();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
674 return n;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
675 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
676
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
677
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
678
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
679 public boolean hasPostgresBackup() {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
680 return postgresBackup != null;
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
681 }
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
682
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
683 public void rebuild_postgres_backup(Luan luan)
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
684 throws IOException, LuanException, SQLException
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
685 {
1454
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
686 logger.info("start rebuild_postgres_backup");
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
687 writeLock.lock();
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
688 IndexSearcher searcher = openSearcher();
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
689 boolean ok = false;
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
690 try {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
691 postgresBackup.begin();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
692 postgresBackup.deleteAll();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
693 Query query = new PrefixQuery(new Term("id"));
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
694 MyCollector col = new MyCollector() {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
695 @Override public void collect(int iDoc) throws IOException {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
696 try {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
697 Document doc = searcher.doc( docBase + iDoc );
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
698 LuanTable tbl = toTable(luan,doc);
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
699 postgresBackup.add(tbl);
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
700 } catch(LuanException e) {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
701 throw new LuanRuntimeException(e);
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
702 } catch(SQLException e) {
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
703 throw new RuntimeException(e);
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
704 }
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
705 }
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
706 };
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
707 try {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
708 searcher.search(query,col);
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
709 } catch(LuanRuntimeException e) {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
710 throw (LuanException)e.getCause();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
711 }
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
712 ok = true;
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
713 postgresBackup.commit();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
714 } finally {
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
715 close(searcher);
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
716 if( !ok )
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
717 postgresBackup.rollback();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
718 writeLock.unlock();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
719 }
1454
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
720 logger.info("end rebuild_postgres_backup");
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
721 }
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
722
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
723 public void restore_from_postgres()
1447
851b9a48cc44 Luan.parse
Franklin Schmidt <fschmidt@gmail.com>
parents: 1435
diff changeset
724 throws IOException, LuanException, SQLException, ParseException
1388
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
725 {
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
726 if( postgresBackup!=null && wasCreated && !postgresBackup.wasCreated ) {
1454
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
727 logger.error("restoring from postgres");
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
728 force_restore_from_postgres();
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
729 }
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
730 }
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
731
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
732 public void force_restore_from_postgres()
1447
851b9a48cc44 Luan.parse
Franklin Schmidt <fschmidt@gmail.com>
parents: 1435
diff changeset
733 throws IOException, LuanException, SQLException, ParseException
1397
0dc9837c16be fix lucene issues
Franklin Schmidt <fschmidt@gmail.com>
parents: 1395
diff changeset
734 {
1454
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
735 logger.warn("start restore_from_postgres");
1388
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
736 if( postgresBackup==null )
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
737 throw new NullPointerException();
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
738 if( writeLock.isHeldByCurrentThread() )
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
739 throw new RuntimeException();
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
740 writeLock.lock();
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
741 boolean ok = false;
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
742 try {
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
743 writer.deleteAll();
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
744 postgresBackup.restoreLucene(this);
1388
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
745 ok = true;
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
746 writer.commit();
1539
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
747 wrote();
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
748 ensure_open(); // refresh searcher
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
749 initId();
1398
67c0e47b5be3 more lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 1397
diff changeset
750 wasCreated = false;
1388
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
751 } finally {
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
752 if( !ok ) {
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
753 writer.rollback();
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
754 reopen();
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
755 }
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
756 wrote();
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
757 writeLock.unlock();
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
758 }
1454
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
759 logger.warn("end restore_from_postgres");
1388
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
760 }
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
761
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
762 void restore(LuanTable doc)
1388
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
763 throws LuanException, IOException
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
764 {
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1526
diff changeset
765 writer.addDocument(toLucene(doc));
1388
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
766 }
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
767
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
768 public void check(Luan luan) throws IOException, SQLException, LuanException, ParseException {
1406
Franklin Schmidt <fschmidt@gmail.com>
parents: 1402
diff changeset
769 boolean hasPostgres = postgresBackup != null;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1402
diff changeset
770 String msg = "start check";
Franklin Schmidt <fschmidt@gmail.com>
parents: 1402
diff changeset
771 if( hasPostgres )
Franklin Schmidt <fschmidt@gmail.com>
parents: 1402
diff changeset
772 msg += " with postgres";
1454
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
773 logger.info(msg);
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
774 CheckIndex.Status status = new CheckIndex(fsDir).checkIndex();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
775 if( !status.clean )
1454
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
776 logger.error("index not clean");
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
777 if( writer instanceof LoggingIndexWriter ) {
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
778 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer;
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
779 logger.info("log check");
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
780 boolean ok = loggingWriter.check(ID_SORT);
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
781 }
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
782 if( hasPostgres ) {
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
783 logger.info("postgres check");
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
784 checkPostgres(luan);
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
785 }
1454
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
786 logger.info("end check");
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
787 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
788
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
789 public void rebuild_log() throws IOException {
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
790 logger.info("start rebuild_log");
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
791 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer;
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
792 loggingWriter.newLogs();
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
793 logger.info("end rebuild_log");
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
794 }
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1535
diff changeset
795
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
796 private void checkPostgres(Luan luan)
1447
851b9a48cc44 Luan.parse
Franklin Schmidt <fschmidt@gmail.com>
parents: 1435
diff changeset
797 throws IOException, SQLException, LuanException, ParseException
1430
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
798 {
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
799 final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker();
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
800 final IndexSearcher searcher = openSearcher();
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
801 try {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
802 final List<Long> idsLucene = new ArrayList<Long>();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
803 Query query = new PrefixQuery(new Term("id"));
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
804 MyCollector col = new MyCollector() {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
805 @Override public void collect(int iDoc) throws IOException {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
806 Document doc = searcher.doc( docBase + iDoc );
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
807 Long id = (Long)doc.getField("id").numericValue();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
808 idsLucene.add(id);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
809 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
810 };
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
811 searcher.search(query,col);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
812 Collections.sort(idsLucene);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
813 final List<Long> idsPostgres = postgresChecker.getIds();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
814 final int nLucene = idsLucene.size();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
815 final int nPostgres = idsPostgres.size();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
816 int iLucene = 0;
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
817 int iPostgres = 0;
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
818 LuanToString lts = new LuanToString();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
819 lts.strict = true;
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
820 lts.numberTypes = true;
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
821 while( iLucene < nLucene && iPostgres < nPostgres ) {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
822 long idLucene = idsLucene.get(iLucene);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
823 long idPostgres = idsPostgres.get(iPostgres);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
824 if( idLucene < idPostgres ) {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
825 iLucene++;
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
826 checkPostgres(luan,postgresChecker,lts,idLucene);
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
827 } else if( idLucene > idPostgres ) {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
828 iPostgres++;
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
829 checkPostgres(luan,postgresChecker,lts,idPostgres);
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
830 } else { // ==
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
831 LuanTable docPostgres = postgresChecker.getDoc(idPostgres);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
832 TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
833 if( td.totalHits != 1 ) throw new RuntimeException();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
834 Document doc = searcher.doc( td.scoreDocs[0].doc );
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
835 LuanTable docLucene = toTable(luan,doc);
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
836 if( !equal(docPostgres,docLucene) ) {
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
837 checkPostgres(luan,postgresChecker,lts,idPostgres);
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
838 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
839 iLucene++;
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
840 iPostgres++;
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
841 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
842 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
843 while( iLucene < nLucene ) {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
844 long idLucene = idsLucene.get(iLucene++);
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
845 checkPostgres(luan,postgresChecker,lts,idLucene);
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
846 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
847 while( iPostgres < nPostgres ) {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
848 long idPostgres = idsPostgres.get(iPostgres++);
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
849 checkPostgres(luan,postgresChecker,lts,idPostgres);
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
850 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
851 } finally {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
852 close(searcher);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
853 postgresChecker.close();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
854 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
855 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
856
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
857 private void checkPostgres(Luan luan,PostgresBackup.Checker postgresChecker,LuanToString lts,long id)
1447
851b9a48cc44 Luan.parse
Franklin Schmidt <fschmidt@gmail.com>
parents: 1435
diff changeset
858 throws IOException, SQLException, LuanException, ParseException
1430
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
859 {
1454
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
860 //logger.info("check id "+id);
1430
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
861 writeLock.lock();
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
862 try {
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
863 final IndexSearcher searcher = openSearcher();
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
864 try {
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
865 LuanTable docPostgres = postgresChecker.getDoc(id);
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
866 TopDocs td = searcher.search(new TermQuery(term("id",id)),1);
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
867 LuanTable docLucene;
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
868 if( td.totalHits == 0 ) {
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
869 docLucene = null;
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
870 } else if( td.totalHits == 1 ) {
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
871 Document doc = searcher.doc( td.scoreDocs[0].doc );
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
872 docLucene = toTable(luan,doc);
1430
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
873 } else
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
874 throw new RuntimeException();
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
875 if( docPostgres == null ) {
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
876 if( docLucene != null )
1454
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
877 logger.error("id "+id+" found in lucene but not postgres");
1430
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
878 return;
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
879 }
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
880 if( docLucene == null ) {
1454
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
881 logger.error("id "+id+" found in postgres but not lucene");
1430
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
882 return;
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
883 }
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
884 if( !equal(docPostgres,docLucene) ) {
1454
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
885 logger.error("id "+id+" not equal");
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
886 logger.error("lucene = "+lts.toString(docLucene));
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
887 logger.error("postgres = "+lts.toString(docPostgres));
1430
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
888 }
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
889 } finally {
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
890 close(searcher);
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
891 }
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
892 } finally {
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
893 writeLock.unlock();
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
894 }
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
895 }
103d0ce70385 fix postgres check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1420
diff changeset
896
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
897 private static boolean equal(LuanTable t1,LuanTable t2) throws LuanException {
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
898 return t1!=null && t2!=null && toJava(t1).equals(toJava(t2));
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
899 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
900
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
901 private static Map toJava(LuanTable t) throws LuanException {
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
902 Map map = t.asMap();
1526
efbc3720d3f3 postgres backup fix
Franklin Schmidt <fschmidt@gmail.com>
parents: 1482
diff changeset
903 for( Iterator iter = map.entrySet().iterator(); iter.hasNext(); ) {
efbc3720d3f3 postgres backup fix
Franklin Schmidt <fschmidt@gmail.com>
parents: 1482
diff changeset
904 Map.Entry entry = (Map.Entry)iter.next();
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
905 Object value = entry.getValue();
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
906 if( value instanceof LuanTable ) {
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
907 LuanTable v = (LuanTable)value;
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
908 if( !v.isList() )
1454
219f2b937f2b remove log4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1453
diff changeset
909 logger.error("not list");
1530
447b7ef9197f minor fix
Franklin Schmidt <fschmidt@gmail.com>
parents: 1529
diff changeset
910 List list = v.asList();
447b7ef9197f minor fix
Franklin Schmidt <fschmidt@gmail.com>
parents: 1529
diff changeset
911 if( list.isEmpty() ) {
1526
efbc3720d3f3 postgres backup fix
Franklin Schmidt <fschmidt@gmail.com>
parents: 1482
diff changeset
912 iter.remove();
1530
447b7ef9197f minor fix
Franklin Schmidt <fschmidt@gmail.com>
parents: 1529
diff changeset
913 } else if( list.size() == 1 ) {
447b7ef9197f minor fix
Franklin Schmidt <fschmidt@gmail.com>
parents: 1529
diff changeset
914 entry.setValue(list.get(0));
1526
efbc3720d3f3 postgres backup fix
Franklin Schmidt <fschmidt@gmail.com>
parents: 1482
diff changeset
915 } else {
1530
447b7ef9197f minor fix
Franklin Schmidt <fschmidt@gmail.com>
parents: 1529
diff changeset
916 entry.setValue(list);
1526
efbc3720d3f3 postgres backup fix
Franklin Schmidt <fschmidt@gmail.com>
parents: 1482
diff changeset
917 }
1453
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
918 }
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
919 }
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
920 return map;
928be2a4d565 fix postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1447
diff changeset
921 }
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
922 }