Mercurial Hosting > luan
comparison src/luan/modules/lucene/LuceneIndex.java @ 1562:b89212fd04b5
remove table.luan
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Sun, 08 Nov 2020 16:50:59 -0700 |
parents | 52241b69c339 |
children | 8fbcc4747091 |
comparison
equal
deleted
inserted
replaced
1561:e1a13e707bf3 | 1562:b89212fd04b5 |
---|---|
105 synchronized(indexes) { | 105 synchronized(indexes) { |
106 Reference<LuceneIndex> ref = indexes.get(key); | 106 Reference<LuceneIndex> ref = indexes.get(key); |
107 if( ref != null ) { | 107 if( ref != null ) { |
108 LuceneIndex li = ref.get(); | 108 LuceneIndex li = ref.get(); |
109 if( li != null ) { | 109 if( li != null ) { |
110 Object version = options.get("version"); | 110 Object version = options.get(luan,"version"); |
111 if( version==null || version.equals(li.version) ) | 111 if( version==null || version.equals(li.version) ) |
112 return li; | 112 return li; |
113 li.closeWriter(); | 113 li.closeWriter(); |
114 } | 114 } |
115 } | 115 } |
285 wrote(); | 285 wrote(); |
286 writeLock.unlock(); | 286 writeLock.unlock(); |
287 } | 287 } |
288 } | 288 } |
289 | 289 |
290 public void save(LuanTable doc) | 290 public void save(Luan luan,LuanTable doc) |
291 throws LuanException, IOException, SQLException | 291 throws LuanException, IOException, SQLException |
292 { | 292 { |
293 Object obj = doc.get("id"); | 293 Object obj = doc.get(luan,"id"); |
294 Long id; | 294 Long id; |
295 try { | 295 try { |
296 id = (Long)obj; | 296 id = (Long)obj; |
297 } catch(ClassCastException e) { | 297 } catch(ClassCastException e) { |
298 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName()); | 298 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName()); |
301 boolean commit = !writeLock.isHeldByCurrentThread(); | 301 boolean commit = !writeLock.isHeldByCurrentThread(); |
302 writeLock.lock(); | 302 writeLock.lock(); |
303 try { | 303 try { |
304 if( id == null ) { | 304 if( id == null ) { |
305 id = ++this.id; | 305 id = ++this.id; |
306 doc.put("id",id); | 306 doc.put(luan,"id",id); |
307 if( postgresBackup != null ) | 307 if( postgresBackup != null ) |
308 postgresBackup.add(doc); | 308 postgresBackup.add(luan,doc); |
309 writer.addDocument(toLucene(doc)); | 309 writer.addDocument(toLucene(doc)); |
310 } else { | 310 } else { |
311 if( postgresBackup != null ) | 311 if( postgresBackup != null ) |
312 postgresBackup.update(doc); | 312 postgresBackup.update(luan,doc); |
313 writer.updateDocument( "id", toLucene(doc) ); | 313 writer.updateDocument( "id", toLucene(doc) ); |
314 } | 314 } |
315 if(commit) writer.commit(); | 315 if(commit) writer.commit(); |
316 } finally { | 316 } finally { |
317 wrote(); | 317 wrote(); |
411 public Object snapshot(LuanFunction fn) throws LuanException, IOException { | 411 public Object snapshot(LuanFunction fn) throws LuanException, IOException { |
412 SnapshotDeletionPolicy snapshotDeletionPolicy = snapshotDeletionPolicy(); | 412 SnapshotDeletionPolicy snapshotDeletionPolicy = snapshotDeletionPolicy(); |
413 IndexCommit ic = snapshotDeletionPolicy.snapshot(); | 413 IndexCommit ic = snapshotDeletionPolicy.snapshot(); |
414 try { | 414 try { |
415 String dir = fsDir.getDirectory().toString(); | 415 String dir = fsDir.getDirectory().toString(); |
416 LuanTable fileNames = new LuanTable(fn.luan(),new ArrayList(ic.getFileNames())); | 416 LuanTable fileNames = new LuanTable(new ArrayList(ic.getFileNames())); |
417 return fn.call(dir,fileNames); | 417 return fn.call(dir,fileNames); |
418 } finally { | 418 } finally { |
419 snapshotDeletionPolicy.release(ic); | 419 snapshotDeletionPolicy.release(ic); |
420 } | 420 } |
421 } | 421 } |
470 this.query = query; | 470 this.query = query; |
471 } | 471 } |
472 | 472 |
473 @Override public Object call(Object[] args) throws LuanException { | 473 @Override public Object call(Object[] args) throws LuanException { |
474 try { | 474 try { |
475 LuanTable doc = toTable(luan(),searcher.doc(docID)); | 475 LuanTable doc = toTable(searcher.doc(docID)); |
476 if( args.length > 0 && "explain".equals(args[0]) ) { | 476 if( args.length > 0 && "explain".equals(args[0]) ) { |
477 Explanation explanation = searcher.explain(query,docID); | 477 Explanation explanation = searcher.explain(query,docID); |
478 return new Object[]{doc,explanation}; | 478 return new Object[]{doc,explanation}; |
479 } else { | 479 } else { |
480 return doc; | 480 return doc; |
603 | 603 |
604 static Map<String,Object> toLucene(LuanTable table) throws LuanException { | 604 static Map<String,Object> toLucene(LuanTable table) throws LuanException { |
605 return SupplementingConfig.toLucene(table); | 605 return SupplementingConfig.toLucene(table); |
606 } | 606 } |
607 | 607 |
608 private static LuanTable toTable(Luan luan,Document doc) throws LuanException { | 608 private static LuanTable toTable(Document doc) throws LuanException { |
609 return doc==null ? null : SupplementingConfig.toTable(luan,LuceneUtils.toMap(doc)); | 609 return doc==null ? null : SupplementingConfig.toTable(LuceneUtils.toMap(doc)); |
610 } | 610 } |
611 | 611 |
612 | 612 |
613 private static final Formatter nullFormatter = new Formatter() { | 613 private static final Formatter nullFormatter = new Formatter() { |
614 public String highlightTerm(String originalText,TokenGroup tokenGroup) { | 614 public String highlightTerm(String originalText,TokenGroup tokenGroup) { |
704 Query query = new PrefixQuery(new Term("id")); | 704 Query query = new PrefixQuery(new Term("id")); |
705 MyCollector col = new MyCollector() { | 705 MyCollector col = new MyCollector() { |
706 @Override public void collect(int iDoc) throws IOException { | 706 @Override public void collect(int iDoc) throws IOException { |
707 try { | 707 try { |
708 Document doc = searcher.doc( docBase + iDoc ); | 708 Document doc = searcher.doc( docBase + iDoc ); |
709 LuanTable tbl = toTable(luan,doc); | 709 LuanTable tbl = toTable(doc); |
710 postgresBackup.add(tbl); | 710 postgresBackup.add(luan,tbl); |
711 } catch(LuanException e) { | 711 } catch(LuanException e) { |
712 throw new LuanRuntimeException(e); | 712 throw new LuanRuntimeException(e); |
713 } catch(SQLException e) { | 713 } catch(SQLException e) { |
714 throw new RuntimeException(e); | 714 throw new RuntimeException(e); |
715 } | 715 } |
870 final List<Long> idsPostgres = postgresChecker.getIds(); | 870 final List<Long> idsPostgres = postgresChecker.getIds(); |
871 final int nLucene = idsLucene.size(); | 871 final int nLucene = idsLucene.size(); |
872 final int nPostgres = idsPostgres.size(); | 872 final int nPostgres = idsPostgres.size(); |
873 int iLucene = 0; | 873 int iLucene = 0; |
874 int iPostgres = 0; | 874 int iPostgres = 0; |
875 LuanToString lts = new LuanToString(null,null); | 875 LuanToString lts = new LuanToString(luan,null,null); |
876 lts.settingsInit.strict = true; | 876 lts.settingsInit.strict = true; |
877 lts.settingsInit.numberTypes = true; | 877 lts.settingsInit.numberTypes = true; |
878 while( iLucene < nLucene && iPostgres < nPostgres ) { | 878 while( iLucene < nLucene && iPostgres < nPostgres ) { |
879 long idLucene = idsLucene.get(iLucene); | 879 long idLucene = idsLucene.get(iLucene); |
880 long idPostgres = idsPostgres.get(iPostgres); | 880 long idPostgres = idsPostgres.get(iPostgres); |
881 if( idLucene < idPostgres ) { | 881 if( idLucene < idPostgres ) { |
882 iLucene++; | 882 iLucene++; |
883 checkPostgres(luan,postgresChecker,lts,idLucene); | 883 checkPostgres(postgresChecker,lts,idLucene); |
884 } else if( idLucene > idPostgres ) { | 884 } else if( idLucene > idPostgres ) { |
885 iPostgres++; | 885 iPostgres++; |
886 checkPostgres(luan,postgresChecker,lts,idPostgres); | 886 checkPostgres(postgresChecker,lts,idPostgres); |
887 } else { // == | 887 } else { // == |
888 LuanTable docPostgres = postgresChecker.getDoc(idPostgres); | 888 LuanTable docPostgres = postgresChecker.getDoc(idPostgres); |
889 TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1); | 889 TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1); |
890 if( td.totalHits != 1 ) throw new RuntimeException(); | 890 if( td.totalHits != 1 ) throw new RuntimeException(); |
891 Document doc = searcher.doc( td.scoreDocs[0].doc ); | 891 Document doc = searcher.doc( td.scoreDocs[0].doc ); |
892 LuanTable docLucene = toTable(luan,doc); | 892 LuanTable docLucene = toTable(doc); |
893 if( !equal(docPostgres,docLucene) ) { | 893 if( !equal(docPostgres,docLucene) ) { |
894 checkPostgres(luan,postgresChecker,lts,idPostgres); | 894 checkPostgres(postgresChecker,lts,idPostgres); |
895 } | 895 } |
896 iLucene++; | 896 iLucene++; |
897 iPostgres++; | 897 iPostgres++; |
898 } | 898 } |
899 } | 899 } |
900 while( iLucene < nLucene ) { | 900 while( iLucene < nLucene ) { |
901 long idLucene = idsLucene.get(iLucene++); | 901 long idLucene = idsLucene.get(iLucene++); |
902 checkPostgres(luan,postgresChecker,lts,idLucene); | 902 checkPostgres(postgresChecker,lts,idLucene); |
903 } | 903 } |
904 while( iPostgres < nPostgres ) { | 904 while( iPostgres < nPostgres ) { |
905 long idPostgres = idsPostgres.get(iPostgres++); | 905 long idPostgres = idsPostgres.get(iPostgres++); |
906 checkPostgres(luan,postgresChecker,lts,idPostgres); | 906 checkPostgres(postgresChecker,lts,idPostgres); |
907 } | 907 } |
908 } finally { | 908 } finally { |
909 close(searcher); | 909 close(searcher); |
910 postgresChecker.close(); | 910 postgresChecker.close(); |
911 } | 911 } |
912 } | 912 } |
913 | 913 |
914 private void checkPostgres(Luan luan,PostgresBackup.Checker postgresChecker,LuanToString lts,long id) | 914 private void checkPostgres(PostgresBackup.Checker postgresChecker,LuanToString lts,long id) |
915 throws IOException, SQLException, LuanException, ParseException | 915 throws IOException, SQLException, LuanException, ParseException |
916 { | 916 { |
917 //logger.info("check id "+id); | 917 //logger.info("check id "+id); |
918 writeLock.lock(); | 918 writeLock.lock(); |
919 try { | 919 try { |
924 LuanTable docLucene; | 924 LuanTable docLucene; |
925 if( td.totalHits == 0 ) { | 925 if( td.totalHits == 0 ) { |
926 docLucene = null; | 926 docLucene = null; |
927 } else if( td.totalHits == 1 ) { | 927 } else if( td.totalHits == 1 ) { |
928 Document doc = searcher.doc( td.scoreDocs[0].doc ); | 928 Document doc = searcher.doc( td.scoreDocs[0].doc ); |
929 docLucene = toTable(luan,doc); | 929 docLucene = toTable(doc); |
930 } else | 930 } else |
931 throw new RuntimeException(); | 931 throw new RuntimeException(); |
932 if( docPostgres == null ) { | 932 if( docPostgres == null ) { |
933 if( docLucene != null ) | 933 if( docLucene != null ) |
934 logger.error("id "+id+" found in lucene but not postgres"); | 934 logger.error("id "+id+" found in lucene but not postgres"); |