comparison src/luan/modules/lucene/LuceneIndex.java @ 1563:8fbcc4747091

remove LuanFunction.luan
author Franklin Schmidt <fschmidt@gmail.com>
date Mon, 09 Nov 2020 01:37:57 -0700
parents b89212fd04b5
children 582384548a69
comparison
equal deleted inserted replaced
1562:b89212fd04b5 1563:8fbcc4747091
169 if( defaultFieldParser instanceof StringFieldParser ) { 169 if( defaultFieldParser instanceof StringFieldParser ) {
170 StringFieldParser sfp = (StringFieldParser)defaultFieldParser; 170 StringFieldParser sfp = (StringFieldParser)defaultFieldParser;
171 analyzer = sfp.analyzer; 171 analyzer = sfp.analyzer;
172 } 172 }
173 this.analyzer = analyzer; 173 this.analyzer = analyzer;
174 this.config = new SupplementingConfig(luceneVersion,mfp,supplementer); 174 this.config = new SupplementingConfig(luceneVersion,mfp,luan,supplementer);
175 wasCreated = reopen(); 175 wasCreated = reopen();
176 if( postgresSpec == null ) { 176 if( postgresSpec == null ) {
177 postgresBackup = null; 177 postgresBackup = null;
178 } else { 178 } else {
179 postgresBackup = new PostgresBackup(luan,postgresSpec); 179 postgresBackup = new PostgresBackup(postgresSpec);
180 if( !wasCreated && postgresBackup.wasCreated ) { 180 if( !wasCreated && postgresBackup.wasCreated ) {
181 logger.error("rebuilding postgres backup"); 181 logger.error("rebuilding postgres backup");
182 rebuild_postgres_backup(luan); 182 rebuild_postgres_backup(luan);
183 /* 183 /*
184 } else if( wasCreated && !postgresBackup.wasCreated ) { 184 } else if( wasCreated && !postgresBackup.wasCreated ) {
321 321
322 public boolean is_in_transaction() { 322 public boolean is_in_transaction() {
323 return writeLock.isHeldByCurrentThread(); 323 return writeLock.isHeldByCurrentThread();
324 } 324 }
325 325
326 public Object run_in_transaction(LuanFunction fn) 326 public Object run_in_transaction(Luan luan,LuanFunction fn)
327 throws IOException, LuanException, SQLException 327 throws IOException, LuanException, SQLException
328 { 328 {
329 boolean commit = !writeLock.isHeldByCurrentThread(); 329 boolean commit = !writeLock.isHeldByCurrentThread();
330 writeLock.lock(); 330 writeLock.lock();
331 boolean ok = false; 331 boolean ok = false;
332 try { 332 try {
333 if( commit && postgresBackup != null ) 333 if( commit && postgresBackup != null )
334 postgresBackup.begin(); 334 postgresBackup.begin();
335 Object rtn = fn.call(); 335 Object rtn = fn.call(luan);
336 ok = true; 336 ok = true;
337 if(commit) { 337 if(commit) {
338 if( postgresBackup != null ) 338 if( postgresBackup != null )
339 postgresBackup.commit(); 339 postgresBackup.commit();
340 writer.commit(); 340 writer.commit();
351 writeLock.unlock(); 351 writeLock.unlock();
352 } 352 }
353 } 353 }
354 354
355 // ??? 355 // ???
356 public Object run_in_lock(LuanFunction fn) throws IOException, LuanException { 356 public Object run_in_lock(Luan luan,LuanFunction fn) throws IOException, LuanException {
357 if( writeLock.isHeldByCurrentThread() ) 357 if( writeLock.isHeldByCurrentThread() )
358 throw new RuntimeException(); 358 throw new RuntimeException();
359 writeLock.lock(); 359 writeLock.lock();
360 try { 360 try {
361 synchronized(this) { 361 synchronized(this) {
362 return fn.call(); 362 return fn.call(luan);
363 } 363 }
364 } finally { 364 } finally {
365 wrote(); 365 wrote();
366 writeLock.unlock(); 366 writeLock.unlock();
367 } 367 }
406 */ 406 */
407 public SnapshotDeletionPolicy snapshotDeletionPolicy() { 407 public SnapshotDeletionPolicy snapshotDeletionPolicy() {
408 return (SnapshotDeletionPolicy)writer.getLuceneIndexWriter().getConfig().getIndexDeletionPolicy(); 408 return (SnapshotDeletionPolicy)writer.getLuceneIndexWriter().getConfig().getIndexDeletionPolicy();
409 } 409 }
410 410
411 public Object snapshot(LuanFunction fn) throws LuanException, IOException { 411 public Object snapshot(Luan luan,LuanFunction fn) throws LuanException, IOException {
412 SnapshotDeletionPolicy snapshotDeletionPolicy = snapshotDeletionPolicy(); 412 SnapshotDeletionPolicy snapshotDeletionPolicy = snapshotDeletionPolicy();
413 IndexCommit ic = snapshotDeletionPolicy.snapshot(); 413 IndexCommit ic = snapshotDeletionPolicy.snapshot();
414 try { 414 try {
415 String dir = fsDir.getDirectory().toString(); 415 String dir = fsDir.getDirectory().toString();
416 LuanTable fileNames = new LuanTable(new ArrayList(ic.getFileNames())); 416 LuanTable fileNames = new LuanTable(new ArrayList(ic.getFileNames()));
417 return fn.call(dir,fileNames); 417 return fn.call(luan,dir,fileNames);
418 } finally { 418 } finally {
419 snapshotDeletionPolicy.release(ic); 419 snapshotDeletionPolicy.release(ic);
420 } 420 }
421 } 421 }
422 422
462 private static class DocFn extends LuanFunction { 462 private static class DocFn extends LuanFunction {
463 final IndexSearcher searcher; 463 final IndexSearcher searcher;
464 final Query query; 464 final Query query;
465 int docID; 465 int docID;
466 466
467 DocFn(Luan luan,IndexSearcher searcher,Query query) { 467 DocFn(IndexSearcher searcher,Query query) {
468 super(luan);
469 this.searcher = searcher; 468 this.searcher = searcher;
470 this.query = query; 469 this.query = query;
471 } 470 }
472 471
473 @Override public Object call(Object[] args) throws LuanException { 472 @Override public Object call(Luan luan,Object[] args) throws LuanException {
474 try { 473 try {
475 LuanTable doc = toTable(searcher.doc(docID)); 474 LuanTable doc = toTable(searcher.doc(docID));
476 if( args.length > 0 && "explain".equals(args[0]) ) { 475 if( args.length > 0 && "explain".equals(args[0]) ) {
477 Explanation explanation = searcher.explain(query,docID); 476 Explanation explanation = searcher.explain(query,docID);
478 return new Object[]{doc,explanation}; 477 return new Object[]{doc,explanation};
521 520
522 public void ensure_open() throws IOException { 521 public void ensure_open() throws IOException {
523 close(openSearcher()); 522 close(openSearcher());
524 } 523 }
525 524
526 public int advanced_search( String queryStr, LuanFunction fn, Integer n, String sortStr ) 525 public int advanced_search( final Luan luan, String queryStr, LuanFunction fn, Integer n, String sortStr )
527 throws LuanException, IOException, ParseException 526 throws LuanException, IOException, ParseException
528 { 527 {
529 Utils.checkNotNull(queryStr); 528 Utils.checkNotNull(queryStr);
530 Query query = GoodQueryParser.parseQuery(mfp,queryStr); 529 Query query = GoodQueryParser.parseQuery(mfp,queryStr);
531 IndexSearcher searcher = threadLocalSearcher.get(); 530 IndexSearcher searcher = threadLocalSearcher.get();
534 searcher = openSearcher(); 533 searcher = openSearcher();
535 try { 534 try {
536 if( fn!=null && n==null ) { 535 if( fn!=null && n==null ) {
537 if( sortStr != null ) 536 if( sortStr != null )
538 throw new LuanException("sort must be nil when n is nil"); 537 throw new LuanException("sort must be nil when n is nil");
539 final DocFn docFn = new DocFn(fn.luan(),searcher,query); 538 final DocFn docFn = new DocFn(searcher,query);
540 MyCollector col = new MyCollector() { 539 MyCollector col = new MyCollector() {
541 @Override public void collect(int doc) { 540 @Override public void collect(int doc) {
542 try { 541 try {
543 docFn.docID = docBase + doc; 542 docFn.docID = docBase + doc;
544 fn.call(++i,docFn); 543 fn.call(luan,++i,docFn);
545 } catch(LuanException e) { 544 } catch(LuanException e) {
546 throw new LuanRuntimeException(e); 545 throw new LuanRuntimeException(e);
547 } 546 }
548 } 547 }
549 }; 548 };
560 return thcc.getTotalHits(); 559 return thcc.getTotalHits();
561 } 560 }
562 Sort sort = sortStr==null ? null : GoodQueryParser.parseSort(mfp,sortStr); 561 Sort sort = sortStr==null ? null : GoodQueryParser.parseSort(mfp,sortStr);
563 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort); 562 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort);
564 final ScoreDoc[] scoreDocs = td.scoreDocs; 563 final ScoreDoc[] scoreDocs = td.scoreDocs;
565 DocFn docFn = new DocFn(fn.luan(),searcher,query); 564 DocFn docFn = new DocFn(searcher,query);
566 for( int i=0; i<scoreDocs.length; i++ ) { 565 for( int i=0; i<scoreDocs.length; i++ ) {
567 ScoreDoc scoreDoc = scoreDocs[i]; 566 ScoreDoc scoreDoc = scoreDocs[i];
568 docFn.docID = scoreDoc.doc; 567 docFn.docID = scoreDoc.doc;
569 fn.call(i+1,docFn,scoreDoc.score); 568 fn.call(luan,i+1,docFn,scoreDoc.score);
570 } 569 }
571 return td.totalHits; 570 return td.totalHits;
572 } finally { 571 } finally {
573 if( !inTransaction ) 572 if( !inTransaction )
574 close(searcher); 573 close(searcher);
575 } 574 }
576 } 575 }
577 576
578 public Object search_in_transaction(LuanFunction fn) throws LuanException, IOException { 577 public Object search_in_transaction(Luan luan,LuanFunction fn) throws LuanException, IOException {
579 if( threadLocalSearcher.get() != null ) 578 if( threadLocalSearcher.get() != null )
580 throw new LuanException("can't nest search_in_transaction calls"); 579 throw new LuanException("can't nest search_in_transaction calls");
581 IndexSearcher searcher = openSearcher(); 580 IndexSearcher searcher = openSearcher();
582 threadLocalSearcher.set(searcher); 581 threadLocalSearcher.set(searcher);
583 try { 582 try {
584 return fn.call(); 583 return fn.call(luan);
585 } finally { 584 } finally {
586 threadLocalSearcher.set(null); 585 threadLocalSearcher.set(null);
587 close(searcher); 586 close(searcher);
588 } 587 }
589 } 588 }
614 public String highlightTerm(String originalText,TokenGroup tokenGroup) { 613 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
615 return originalText; 614 return originalText;
616 } 615 }
617 }; 616 };
618 617
619 public LuanFunction highlighter(String queryStr,final LuanFunction formatter,final Integer fragmentSize,String dotdotdot) 618 public LuanFunction highlighter(final Luan luan,String queryStr,final LuanFunction formatter,final Integer fragmentSize,String dotdotdot)
620 throws ParseException 619 throws ParseException
621 { 620 {
622 Query query = GoodQueryParser.parseQuery(mfp,queryStr); 621 Query query = GoodQueryParser.parseQuery(mfp,queryStr);
623 Formatter fmt = new Formatter() { 622 Formatter fmt = new Formatter() {
624 public String highlightTerm(String originalText,TokenGroup tokenGroup) { 623 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
625 if( tokenGroup.getTotalScore() <= 0 ) 624 if( tokenGroup.getTotalScore() <= 0 )
626 return originalText; 625 return originalText;
627 try { 626 try {
628 return (String)Luan.first(formatter.call(originalText)); 627 return (String)Luan.first(formatter.call(luan,originalText));
629 } catch(LuanException e) { 628 } catch(LuanException e) {
630 throw new LuanRuntimeException(e); 629 throw new LuanRuntimeException(e);
631 } 630 }
632 } 631 }
633 }; 632 };
635 final Highlighter chooser = fragmentSize==null ? null : new Highlighter(nullFormatter,queryScorer); 634 final Highlighter chooser = fragmentSize==null ? null : new Highlighter(nullFormatter,queryScorer);
636 if( chooser != null ) 635 if( chooser != null )
637 chooser.setTextFragmenter( new SimpleSpanFragmenter(queryScorer,fragmentSize) ); 636 chooser.setTextFragmenter( new SimpleSpanFragmenter(queryScorer,fragmentSize) );
638 final Highlighter hl = new Highlighter(fmt,queryScorer); 637 final Highlighter hl = new Highlighter(fmt,queryScorer);
639 hl.setTextFragmenter( new NullFragmenter() ); 638 hl.setTextFragmenter( new NullFragmenter() );
640 return new LuanFunction(false) { // ??? 639 return new LuanFunction() {
641 @Override public String call(Object[] args) throws LuanException { 640 @Override public String call(Luan luan,Object[] args) throws LuanException {
642 String text = (String)args[0]; 641 String text = (String)args[0];
643 try { 642 try {
644 if( chooser != null ) { 643 if( chooser != null ) {
645 String s = chooser.getBestFragment(analyzer,null,text); 644 String s = chooser.getBestFragment(analyzer,null,text);
646 if( s != null ) { 645 if( s != null ) {
789 writeLock.unlock(); 788 writeLock.unlock();
790 } 789 }
791 logger.info("end relog"); 790 logger.info("end relog");
792 } 791 }
793 792
794 public void restore_from_log(LuanFunction handler) 793 public void restore_from_log(Luan luan,LuanFunction handler)
795 throws IOException, LuanException, SQLException, ParseException 794 throws IOException, LuanException, SQLException, ParseException
796 { 795 {
797 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer; 796 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer;
798 if( wasCreated && !loggingWriter.wasCreated ) { 797 if( wasCreated && !loggingWriter.wasCreated ) {
799 logger.error("restoring from log"); 798 logger.error("restoring from log");
800 force_restore_from_log(handler); 799 force_restore_from_log(luan,handler);
801 } 800 }
802 } 801 }
803 802
804 public void force_restore_from_log(LuanFunction handler) 803 public void force_restore_from_log(Luan luan,LuanFunction handler)
805 throws IOException 804 throws IOException
806 { 805 {
807 logger.warn("start force_restore_from_log"); 806 logger.warn("start force_restore_from_log");
808 if( writeLock.isHeldByCurrentThread() ) 807 if( writeLock.isHeldByCurrentThread() )
809 throw new RuntimeException(); 808 throw new RuntimeException();
810 OpDoer opDoer = handler==null ? null : new LuanOpDoer(writer,handler); 809 OpDoer opDoer = handler==null ? null : new LuanOpDoer(writer,luan,handler);
811 writeLock.lock(); 810 writeLock.lock();
812 boolean ok = false; 811 boolean ok = false;
813 try { 812 try {
814 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer; 813 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer;
815 loggingWriter.playLogs(opDoer); 814 loggingWriter.playLogs(opDoer);
827 writeLock.unlock(); 826 writeLock.unlock();
828 } 827 }
829 logger.warn("end force_restore_from_log"); 828 logger.warn("end force_restore_from_log");
830 } 829 }
831 830
832 public void check(Luan luan) throws IOException, SQLException, LuanException, ParseException { 831 public void check() throws IOException, SQLException, LuanException, ParseException {
833 boolean hasPostgres = postgresBackup != null; 832 boolean hasPostgres = postgresBackup != null;
834 String msg = "start check"; 833 String msg = "start check";
835 if( hasPostgres ) 834 if( hasPostgres )
836 msg += " with postgres"; 835 msg += " with postgres";
837 logger.info(msg); 836 logger.info(msg);
843 logger.info("log check"); 842 logger.info("log check");
844 boolean ok = loggingWriter.check(ID_SORT); 843 boolean ok = loggingWriter.check(ID_SORT);
845 } 844 }
846 if( hasPostgres ) { 845 if( hasPostgres ) {
847 logger.info("postgres check"); 846 logger.info("postgres check");
848 checkPostgres(luan); 847 checkPostgres();
849 } 848 }
850 logger.info("end check"); 849 logger.info("end check");
851 } 850 }
852 851
853 private void checkPostgres(Luan luan) 852 private void checkPostgres()
854 throws IOException, SQLException, LuanException, ParseException 853 throws IOException, SQLException, LuanException, ParseException
855 { 854 {
856 final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker(); 855 final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker();
857 final IndexSearcher searcher = openSearcher(); 856 final IndexSearcher searcher = openSearcher();
858 try { 857 try {
870 final List<Long> idsPostgres = postgresChecker.getIds(); 869 final List<Long> idsPostgres = postgresChecker.getIds();
871 final int nLucene = idsLucene.size(); 870 final int nLucene = idsLucene.size();
872 final int nPostgres = idsPostgres.size(); 871 final int nPostgres = idsPostgres.size();
873 int iLucene = 0; 872 int iLucene = 0;
874 int iPostgres = 0; 873 int iPostgres = 0;
875 LuanToString lts = new LuanToString(luan,null,null); 874 LuanToString lts = new LuanToString(null,null);
876 lts.settingsInit.strict = true; 875 lts.settingsInit.strict = true;
877 lts.settingsInit.numberTypes = true; 876 lts.settingsInit.numberTypes = true;
878 while( iLucene < nLucene && iPostgres < nPostgres ) { 877 while( iLucene < nLucene && iPostgres < nPostgres ) {
879 long idLucene = idsLucene.get(iLucene); 878 long idLucene = idsLucene.get(iLucene);
880 long idPostgres = idsPostgres.get(iPostgres); 879 long idPostgres = idsPostgres.get(iPostgres);