Mercurial Hosting > luan
comparison src/luan/modules/lucene/LuceneIndex.java @ 1563:8fbcc4747091
remove LuanFunction.luan
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Mon, 09 Nov 2020 01:37:57 -0700 |
parents | b89212fd04b5 |
children | 582384548a69 |
comparison
equal
deleted
inserted
replaced
1562:b89212fd04b5 | 1563:8fbcc4747091 |
---|---|
169 if( defaultFieldParser instanceof StringFieldParser ) { | 169 if( defaultFieldParser instanceof StringFieldParser ) { |
170 StringFieldParser sfp = (StringFieldParser)defaultFieldParser; | 170 StringFieldParser sfp = (StringFieldParser)defaultFieldParser; |
171 analyzer = sfp.analyzer; | 171 analyzer = sfp.analyzer; |
172 } | 172 } |
173 this.analyzer = analyzer; | 173 this.analyzer = analyzer; |
174 this.config = new SupplementingConfig(luceneVersion,mfp,supplementer); | 174 this.config = new SupplementingConfig(luceneVersion,mfp,luan,supplementer); |
175 wasCreated = reopen(); | 175 wasCreated = reopen(); |
176 if( postgresSpec == null ) { | 176 if( postgresSpec == null ) { |
177 postgresBackup = null; | 177 postgresBackup = null; |
178 } else { | 178 } else { |
179 postgresBackup = new PostgresBackup(luan,postgresSpec); | 179 postgresBackup = new PostgresBackup(postgresSpec); |
180 if( !wasCreated && postgresBackup.wasCreated ) { | 180 if( !wasCreated && postgresBackup.wasCreated ) { |
181 logger.error("rebuilding postgres backup"); | 181 logger.error("rebuilding postgres backup"); |
182 rebuild_postgres_backup(luan); | 182 rebuild_postgres_backup(luan); |
183 /* | 183 /* |
184 } else if( wasCreated && !postgresBackup.wasCreated ) { | 184 } else if( wasCreated && !postgresBackup.wasCreated ) { |
321 | 321 |
322 public boolean is_in_transaction() { | 322 public boolean is_in_transaction() { |
323 return writeLock.isHeldByCurrentThread(); | 323 return writeLock.isHeldByCurrentThread(); |
324 } | 324 } |
325 | 325 |
326 public Object run_in_transaction(LuanFunction fn) | 326 public Object run_in_transaction(Luan luan,LuanFunction fn) |
327 throws IOException, LuanException, SQLException | 327 throws IOException, LuanException, SQLException |
328 { | 328 { |
329 boolean commit = !writeLock.isHeldByCurrentThread(); | 329 boolean commit = !writeLock.isHeldByCurrentThread(); |
330 writeLock.lock(); | 330 writeLock.lock(); |
331 boolean ok = false; | 331 boolean ok = false; |
332 try { | 332 try { |
333 if( commit && postgresBackup != null ) | 333 if( commit && postgresBackup != null ) |
334 postgresBackup.begin(); | 334 postgresBackup.begin(); |
335 Object rtn = fn.call(); | 335 Object rtn = fn.call(luan); |
336 ok = true; | 336 ok = true; |
337 if(commit) { | 337 if(commit) { |
338 if( postgresBackup != null ) | 338 if( postgresBackup != null ) |
339 postgresBackup.commit(); | 339 postgresBackup.commit(); |
340 writer.commit(); | 340 writer.commit(); |
351 writeLock.unlock(); | 351 writeLock.unlock(); |
352 } | 352 } |
353 } | 353 } |
354 | 354 |
355 // ??? | 355 // ??? |
356 public Object run_in_lock(LuanFunction fn) throws IOException, LuanException { | 356 public Object run_in_lock(Luan luan,LuanFunction fn) throws IOException, LuanException { |
357 if( writeLock.isHeldByCurrentThread() ) | 357 if( writeLock.isHeldByCurrentThread() ) |
358 throw new RuntimeException(); | 358 throw new RuntimeException(); |
359 writeLock.lock(); | 359 writeLock.lock(); |
360 try { | 360 try { |
361 synchronized(this) { | 361 synchronized(this) { |
362 return fn.call(); | 362 return fn.call(luan); |
363 } | 363 } |
364 } finally { | 364 } finally { |
365 wrote(); | 365 wrote(); |
366 writeLock.unlock(); | 366 writeLock.unlock(); |
367 } | 367 } |
406 */ | 406 */ |
407 public SnapshotDeletionPolicy snapshotDeletionPolicy() { | 407 public SnapshotDeletionPolicy snapshotDeletionPolicy() { |
408 return (SnapshotDeletionPolicy)writer.getLuceneIndexWriter().getConfig().getIndexDeletionPolicy(); | 408 return (SnapshotDeletionPolicy)writer.getLuceneIndexWriter().getConfig().getIndexDeletionPolicy(); |
409 } | 409 } |
410 | 410 |
411 public Object snapshot(LuanFunction fn) throws LuanException, IOException { | 411 public Object snapshot(Luan luan,LuanFunction fn) throws LuanException, IOException { |
412 SnapshotDeletionPolicy snapshotDeletionPolicy = snapshotDeletionPolicy(); | 412 SnapshotDeletionPolicy snapshotDeletionPolicy = snapshotDeletionPolicy(); |
413 IndexCommit ic = snapshotDeletionPolicy.snapshot(); | 413 IndexCommit ic = snapshotDeletionPolicy.snapshot(); |
414 try { | 414 try { |
415 String dir = fsDir.getDirectory().toString(); | 415 String dir = fsDir.getDirectory().toString(); |
416 LuanTable fileNames = new LuanTable(new ArrayList(ic.getFileNames())); | 416 LuanTable fileNames = new LuanTable(new ArrayList(ic.getFileNames())); |
417 return fn.call(dir,fileNames); | 417 return fn.call(luan,dir,fileNames); |
418 } finally { | 418 } finally { |
419 snapshotDeletionPolicy.release(ic); | 419 snapshotDeletionPolicy.release(ic); |
420 } | 420 } |
421 } | 421 } |
422 | 422 |
462 private static class DocFn extends LuanFunction { | 462 private static class DocFn extends LuanFunction { |
463 final IndexSearcher searcher; | 463 final IndexSearcher searcher; |
464 final Query query; | 464 final Query query; |
465 int docID; | 465 int docID; |
466 | 466 |
467 DocFn(Luan luan,IndexSearcher searcher,Query query) { | 467 DocFn(IndexSearcher searcher,Query query) { |
468 super(luan); | |
469 this.searcher = searcher; | 468 this.searcher = searcher; |
470 this.query = query; | 469 this.query = query; |
471 } | 470 } |
472 | 471 |
473 @Override public Object call(Object[] args) throws LuanException { | 472 @Override public Object call(Luan luan,Object[] args) throws LuanException { |
474 try { | 473 try { |
475 LuanTable doc = toTable(searcher.doc(docID)); | 474 LuanTable doc = toTable(searcher.doc(docID)); |
476 if( args.length > 0 && "explain".equals(args[0]) ) { | 475 if( args.length > 0 && "explain".equals(args[0]) ) { |
477 Explanation explanation = searcher.explain(query,docID); | 476 Explanation explanation = searcher.explain(query,docID); |
478 return new Object[]{doc,explanation}; | 477 return new Object[]{doc,explanation}; |
521 | 520 |
522 public void ensure_open() throws IOException { | 521 public void ensure_open() throws IOException { |
523 close(openSearcher()); | 522 close(openSearcher()); |
524 } | 523 } |
525 | 524 |
526 public int advanced_search( String queryStr, LuanFunction fn, Integer n, String sortStr ) | 525 public int advanced_search( final Luan luan, String queryStr, LuanFunction fn, Integer n, String sortStr ) |
527 throws LuanException, IOException, ParseException | 526 throws LuanException, IOException, ParseException |
528 { | 527 { |
529 Utils.checkNotNull(queryStr); | 528 Utils.checkNotNull(queryStr); |
530 Query query = GoodQueryParser.parseQuery(mfp,queryStr); | 529 Query query = GoodQueryParser.parseQuery(mfp,queryStr); |
531 IndexSearcher searcher = threadLocalSearcher.get(); | 530 IndexSearcher searcher = threadLocalSearcher.get(); |
534 searcher = openSearcher(); | 533 searcher = openSearcher(); |
535 try { | 534 try { |
536 if( fn!=null && n==null ) { | 535 if( fn!=null && n==null ) { |
537 if( sortStr != null ) | 536 if( sortStr != null ) |
538 throw new LuanException("sort must be nil when n is nil"); | 537 throw new LuanException("sort must be nil when n is nil"); |
539 final DocFn docFn = new DocFn(fn.luan(),searcher,query); | 538 final DocFn docFn = new DocFn(searcher,query); |
540 MyCollector col = new MyCollector() { | 539 MyCollector col = new MyCollector() { |
541 @Override public void collect(int doc) { | 540 @Override public void collect(int doc) { |
542 try { | 541 try { |
543 docFn.docID = docBase + doc; | 542 docFn.docID = docBase + doc; |
544 fn.call(++i,docFn); | 543 fn.call(luan,++i,docFn); |
545 } catch(LuanException e) { | 544 } catch(LuanException e) { |
546 throw new LuanRuntimeException(e); | 545 throw new LuanRuntimeException(e); |
547 } | 546 } |
548 } | 547 } |
549 }; | 548 }; |
560 return thcc.getTotalHits(); | 559 return thcc.getTotalHits(); |
561 } | 560 } |
562 Sort sort = sortStr==null ? null : GoodQueryParser.parseSort(mfp,sortStr); | 561 Sort sort = sortStr==null ? null : GoodQueryParser.parseSort(mfp,sortStr); |
563 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort); | 562 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort); |
564 final ScoreDoc[] scoreDocs = td.scoreDocs; | 563 final ScoreDoc[] scoreDocs = td.scoreDocs; |
565 DocFn docFn = new DocFn(fn.luan(),searcher,query); | 564 DocFn docFn = new DocFn(searcher,query); |
566 for( int i=0; i<scoreDocs.length; i++ ) { | 565 for( int i=0; i<scoreDocs.length; i++ ) { |
567 ScoreDoc scoreDoc = scoreDocs[i]; | 566 ScoreDoc scoreDoc = scoreDocs[i]; |
568 docFn.docID = scoreDoc.doc; | 567 docFn.docID = scoreDoc.doc; |
569 fn.call(i+1,docFn,scoreDoc.score); | 568 fn.call(luan,i+1,docFn,scoreDoc.score); |
570 } | 569 } |
571 return td.totalHits; | 570 return td.totalHits; |
572 } finally { | 571 } finally { |
573 if( !inTransaction ) | 572 if( !inTransaction ) |
574 close(searcher); | 573 close(searcher); |
575 } | 574 } |
576 } | 575 } |
577 | 576 |
578 public Object search_in_transaction(LuanFunction fn) throws LuanException, IOException { | 577 public Object search_in_transaction(Luan luan,LuanFunction fn) throws LuanException, IOException { |
579 if( threadLocalSearcher.get() != null ) | 578 if( threadLocalSearcher.get() != null ) |
580 throw new LuanException("can't nest search_in_transaction calls"); | 579 throw new LuanException("can't nest search_in_transaction calls"); |
581 IndexSearcher searcher = openSearcher(); | 580 IndexSearcher searcher = openSearcher(); |
582 threadLocalSearcher.set(searcher); | 581 threadLocalSearcher.set(searcher); |
583 try { | 582 try { |
584 return fn.call(); | 583 return fn.call(luan); |
585 } finally { | 584 } finally { |
586 threadLocalSearcher.set(null); | 585 threadLocalSearcher.set(null); |
587 close(searcher); | 586 close(searcher); |
588 } | 587 } |
589 } | 588 } |
614 public String highlightTerm(String originalText,TokenGroup tokenGroup) { | 613 public String highlightTerm(String originalText,TokenGroup tokenGroup) { |
615 return originalText; | 614 return originalText; |
616 } | 615 } |
617 }; | 616 }; |
618 | 617 |
619 public LuanFunction highlighter(String queryStr,final LuanFunction formatter,final Integer fragmentSize,String dotdotdot) | 618 public LuanFunction highlighter(final Luan luan,String queryStr,final LuanFunction formatter,final Integer fragmentSize,String dotdotdot) |
620 throws ParseException | 619 throws ParseException |
621 { | 620 { |
622 Query query = GoodQueryParser.parseQuery(mfp,queryStr); | 621 Query query = GoodQueryParser.parseQuery(mfp,queryStr); |
623 Formatter fmt = new Formatter() { | 622 Formatter fmt = new Formatter() { |
624 public String highlightTerm(String originalText,TokenGroup tokenGroup) { | 623 public String highlightTerm(String originalText,TokenGroup tokenGroup) { |
625 if( tokenGroup.getTotalScore() <= 0 ) | 624 if( tokenGroup.getTotalScore() <= 0 ) |
626 return originalText; | 625 return originalText; |
627 try { | 626 try { |
628 return (String)Luan.first(formatter.call(originalText)); | 627 return (String)Luan.first(formatter.call(luan,originalText)); |
629 } catch(LuanException e) { | 628 } catch(LuanException e) { |
630 throw new LuanRuntimeException(e); | 629 throw new LuanRuntimeException(e); |
631 } | 630 } |
632 } | 631 } |
633 }; | 632 }; |
635 final Highlighter chooser = fragmentSize==null ? null : new Highlighter(nullFormatter,queryScorer); | 634 final Highlighter chooser = fragmentSize==null ? null : new Highlighter(nullFormatter,queryScorer); |
636 if( chooser != null ) | 635 if( chooser != null ) |
637 chooser.setTextFragmenter( new SimpleSpanFragmenter(queryScorer,fragmentSize) ); | 636 chooser.setTextFragmenter( new SimpleSpanFragmenter(queryScorer,fragmentSize) ); |
638 final Highlighter hl = new Highlighter(fmt,queryScorer); | 637 final Highlighter hl = new Highlighter(fmt,queryScorer); |
639 hl.setTextFragmenter( new NullFragmenter() ); | 638 hl.setTextFragmenter( new NullFragmenter() ); |
640 return new LuanFunction(false) { // ??? | 639 return new LuanFunction() { |
641 @Override public String call(Object[] args) throws LuanException { | 640 @Override public String call(Luan luan,Object[] args) throws LuanException { |
642 String text = (String)args[0]; | 641 String text = (String)args[0]; |
643 try { | 642 try { |
644 if( chooser != null ) { | 643 if( chooser != null ) { |
645 String s = chooser.getBestFragment(analyzer,null,text); | 644 String s = chooser.getBestFragment(analyzer,null,text); |
646 if( s != null ) { | 645 if( s != null ) { |
789 writeLock.unlock(); | 788 writeLock.unlock(); |
790 } | 789 } |
791 logger.info("end relog"); | 790 logger.info("end relog"); |
792 } | 791 } |
793 | 792 |
794 public void restore_from_log(LuanFunction handler) | 793 public void restore_from_log(Luan luan,LuanFunction handler) |
795 throws IOException, LuanException, SQLException, ParseException | 794 throws IOException, LuanException, SQLException, ParseException |
796 { | 795 { |
797 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer; | 796 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer; |
798 if( wasCreated && !loggingWriter.wasCreated ) { | 797 if( wasCreated && !loggingWriter.wasCreated ) { |
799 logger.error("restoring from log"); | 798 logger.error("restoring from log"); |
800 force_restore_from_log(handler); | 799 force_restore_from_log(luan,handler); |
801 } | 800 } |
802 } | 801 } |
803 | 802 |
804 public void force_restore_from_log(LuanFunction handler) | 803 public void force_restore_from_log(Luan luan,LuanFunction handler) |
805 throws IOException | 804 throws IOException |
806 { | 805 { |
807 logger.warn("start force_restore_from_log"); | 806 logger.warn("start force_restore_from_log"); |
808 if( writeLock.isHeldByCurrentThread() ) | 807 if( writeLock.isHeldByCurrentThread() ) |
809 throw new RuntimeException(); | 808 throw new RuntimeException(); |
810 OpDoer opDoer = handler==null ? null : new LuanOpDoer(writer,handler); | 809 OpDoer opDoer = handler==null ? null : new LuanOpDoer(writer,luan,handler); |
811 writeLock.lock(); | 810 writeLock.lock(); |
812 boolean ok = false; | 811 boolean ok = false; |
813 try { | 812 try { |
814 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer; | 813 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer; |
815 loggingWriter.playLogs(opDoer); | 814 loggingWriter.playLogs(opDoer); |
827 writeLock.unlock(); | 826 writeLock.unlock(); |
828 } | 827 } |
829 logger.warn("end force_restore_from_log"); | 828 logger.warn("end force_restore_from_log"); |
830 } | 829 } |
831 | 830 |
832 public void check(Luan luan) throws IOException, SQLException, LuanException, ParseException { | 831 public void check() throws IOException, SQLException, LuanException, ParseException { |
833 boolean hasPostgres = postgresBackup != null; | 832 boolean hasPostgres = postgresBackup != null; |
834 String msg = "start check"; | 833 String msg = "start check"; |
835 if( hasPostgres ) | 834 if( hasPostgres ) |
836 msg += " with postgres"; | 835 msg += " with postgres"; |
837 logger.info(msg); | 836 logger.info(msg); |
843 logger.info("log check"); | 842 logger.info("log check"); |
844 boolean ok = loggingWriter.check(ID_SORT); | 843 boolean ok = loggingWriter.check(ID_SORT); |
845 } | 844 } |
846 if( hasPostgres ) { | 845 if( hasPostgres ) { |
847 logger.info("postgres check"); | 846 logger.info("postgres check"); |
848 checkPostgres(luan); | 847 checkPostgres(); |
849 } | 848 } |
850 logger.info("end check"); | 849 logger.info("end check"); |
851 } | 850 } |
852 | 851 |
853 private void checkPostgres(Luan luan) | 852 private void checkPostgres() |
854 throws IOException, SQLException, LuanException, ParseException | 853 throws IOException, SQLException, LuanException, ParseException |
855 { | 854 { |
856 final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker(); | 855 final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker(); |
857 final IndexSearcher searcher = openSearcher(); | 856 final IndexSearcher searcher = openSearcher(); |
858 try { | 857 try { |
870 final List<Long> idsPostgres = postgresChecker.getIds(); | 869 final List<Long> idsPostgres = postgresChecker.getIds(); |
871 final int nLucene = idsLucene.size(); | 870 final int nLucene = idsLucene.size(); |
872 final int nPostgres = idsPostgres.size(); | 871 final int nPostgres = idsPostgres.size(); |
873 int iLucene = 0; | 872 int iLucene = 0; |
874 int iPostgres = 0; | 873 int iPostgres = 0; |
875 LuanToString lts = new LuanToString(luan,null,null); | 874 LuanToString lts = new LuanToString(null,null); |
876 lts.settingsInit.strict = true; | 875 lts.settingsInit.strict = true; |
877 lts.settingsInit.numberTypes = true; | 876 lts.settingsInit.numberTypes = true; |
878 while( iLucene < nLucene && iPostgres < nPostgres ) { | 877 while( iLucene < nLucene && iPostgres < nPostgres ) { |
879 long idLucene = idsLucene.get(iLucene); | 878 long idLucene = idsLucene.get(iLucene); |
880 long idPostgres = idsPostgres.get(iPostgres); | 879 long idPostgres = idsPostgres.get(iPostgres); |