001/* 002 * File : $Source$ 003 * Date : $Date$ 004 * Version: $Revision$ 005 * 006 * This library is part of OpenCms - 007 * the Open Source Content Management System 008 * 009 * Copyright (C) 2002 - 2009 Alkacon Software (http://www.alkacon.com) 010 * 011 * This library is free software; you can redistribute it and/or 012 * modify it under the terms of the GNU Lesser General Public 013 * License as published by the Free Software Foundation; either 014 * version 2.1 of the License, or (at your option) any later version. 015 * 016 * This library is distributed in the hope that it will be useful, 017 * but WITHOUT ANY WARRANTY; without even the implied warranty of 018 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 019 * Lesser General Public License for more details. 020 * 021 * For further information about Alkacon Software, please see the 022 * company website: http://www.alkacon.com 023 * 024 * For further information about OpenCms, please see the 025 * project website: http://www.opencms.org 026 * 027 * You should have received a copy of the GNU Lesser General Public 028 * License along with this library; if not, write to the Free Software 029 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 030 */ 031 032package org.opencms.search.solr; 033 034import org.opencms.configuration.CmsConfigurationException; 035import org.opencms.configuration.CmsParameterConfiguration; 036import org.opencms.file.CmsFile; 037import org.opencms.file.CmsObject; 038import org.opencms.file.CmsProject; 039import org.opencms.file.CmsPropertyDefinition; 040import org.opencms.file.CmsResource; 041import org.opencms.file.CmsResourceFilter; 042import org.opencms.i18n.CmsEncoder; 043import org.opencms.i18n.CmsLocaleManager; 044import org.opencms.main.CmsException; 045import org.opencms.main.CmsIllegalArgumentException; 046import org.opencms.main.CmsLog; 047import org.opencms.main.OpenCms; 048import org.opencms.report.I_CmsReport; 049import org.opencms.search.CmsSearchException; 050import org.opencms.search.CmsSearchIndex; 051import org.opencms.search.CmsSearchManager; 052import org.opencms.search.CmsSearchParameters; 053import org.opencms.search.CmsSearchResource; 054import org.opencms.search.CmsSearchResultList; 055import org.opencms.search.I_CmsIndexWriter; 056import org.opencms.search.I_CmsSearchDocument; 057import org.opencms.search.fields.CmsSearchField; 058import org.opencms.search.galleries.CmsGallerySearchParameters; 059import org.opencms.search.galleries.CmsGallerySearchResult; 060import org.opencms.search.galleries.CmsGallerySearchResultList; 061import org.opencms.security.CmsRole; 062import org.opencms.security.CmsRoleViolationException; 063import org.opencms.util.CmsFileUtil; 064import org.opencms.util.CmsRequestUtil; 065import org.opencms.util.CmsStringUtil; 066 067import java.io.IOException; 068import java.io.OutputStreamWriter; 069import java.io.UnsupportedEncodingException; 070import java.io.Writer; 071import java.nio.charset.Charset; 072import java.util.ArrayList; 073import java.util.Arrays; 074import java.util.Collections; 075import java.util.HashSet; 076import java.util.List; 077import java.util.Locale; 078import java.util.Optional; 079import java.util.Set; 080import java.util.stream.Stream; 081 082import javax.servlet.ServletResponse; 083 084import org.apache.commons.logging.Log; 085import org.apache.solr.client.solrj.SolrClient; 086import org.apache.solr.client.solrj.SolrQuery; 087import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; 088import org.apache.solr.client.solrj.response.QueryResponse; 089import org.apache.solr.common.SolrDocument; 090import org.apache.solr.common.SolrDocumentList; 091import org.apache.solr.common.SolrInputDocument; 092import org.apache.solr.common.util.ContentStreamBase; 093import org.apache.solr.common.util.FastWriter; 094import org.apache.solr.common.util.NamedList; 095import org.apache.solr.common.util.SimpleOrderedMap; 096import org.apache.solr.core.CoreContainer; 097import org.apache.solr.core.SolrCore; 098import org.apache.solr.handler.ReplicationHandler; 099import org.apache.solr.request.LocalSolrQueryRequest; 100import org.apache.solr.request.SolrQueryRequest; 101import org.apache.solr.request.SolrRequestHandler; 102import org.apache.solr.response.BinaryQueryResponseWriter; 103import org.apache.solr.response.QueryResponseWriter; 104import org.apache.solr.response.SolrQueryResponse; 105 106import com.google.common.base.Objects; 107 108/** 109 * Implements the search within an Solr index.<p> 110 * 111 * @since 8.5.0 112 */ 113public class CmsSolrIndex extends CmsSearchIndex { 114 115 /** The serial version id. */ 116 private static final long serialVersionUID = -1570077792574476721L; 117 118 /** The name of the default Solr Offline index. */ 119 public static final String DEFAULT_INDEX_NAME_OFFLINE = "Solr Offline"; 120 121 /** The name of the default Solr Online index. */ 122 public static final String DEFAULT_INDEX_NAME_ONLINE = "Solr Online"; 123 124 /** Constant for additional parameter to set the post processor class name. */ 125 public static final String POST_PROCESSOR = "search.solr.postProcessor"; 126 127 /** 128 * Constant for additional parameter to set the maximally processed results (start + rows) for searches with this index. 129 * It overwrites the global configuration from {@link CmsSolrConfiguration#getMaxProcessedResults()} for this index. 130 **/ 131 public static final String SOLR_SEARCH_MAX_PROCESSED_RESULTS = "search.solr.maxProcessedResults"; 132 133 /** Constant for additional parameter to set the fields the select handler should return at maximum. */ 134 public static final String SOLR_HANDLER_ALLOWED_FIELDS = "handle.solr.allowedFields"; 135 136 /** Constant for additional parameter to set the number results the select handler should return at maxium per request. */ 137 public static final String SOLR_HANDLER_MAX_ALLOWED_RESULTS_PER_PAGE = "handle.solr.maxAllowedResultsPerPage"; 138 139 /** Constant for additional parameter to set the maximal number of a result, the select handler should return. */ 140 public static final String SOLR_HANDLER_MAX_ALLOWED_RESULTS_AT_ALL = "handle.solr.maxAllowedResultsAtAll"; 141 142 /** Constant for additional parameter to disable the select handler (except for debug mode). */ 143 private static final String SOLR_HANDLER_DISABLE_SELECT = "handle.solr.disableSelectHandler"; 144 145 /** Constant for additional parameter to set the VFS path to the file holding the debug secret. */ 146 private static final String SOLR_HANDLER_DEBUG_SECRET_FILE = "handle.solr.debugSecretFile"; 147 148 /** Constant for additional parameter to disable the spell handler (except for debug mode). */ 149 private static final String SOLR_HANDLER_DISABLE_SPELL = "handle.solr.disableSpellHandler"; 150 151 /** Constant for additional parameter to configure an external solr server specifically for the index. */ 152 private static final String SOLR_SERVER_URL = "server.url"; 153 154 /** The solr exclude property. */ 155 public static final String PROPERTY_SEARCH_EXCLUDE_VALUE_SOLR = "solr"; 156 157 /** Indicates the maximum number of documents from the complete result set to return. */ 158 public static final int ROWS_MAX = 50; 159 160 /** The constant for an unlimited maximum number of results to return in a Solr search. */ 161 public static final int MAX_RESULTS_UNLIMITED = -1; 162 163 /** The constant for an unlimited maximum number of results to return in a Solr search. */ 164 public static final int MAX_RESULTS_GALLERY = 10000; 165 166 /** A constant for debug formatting output. */ 167 protected static final int DEBUG_PADDING_RIGHT = 50; 168 169 /** The name for the parameters key of the response header. */ 170 private static final String HEADER_PARAMS_NAME = "params"; 171 172 /** The log object for this class. */ 173 private static final Log LOG = CmsLog.getLog(CmsSolrIndex.class); 174 175 /** Pseudo resource used for not permission checked indexes. */ 176 private static final CmsResource PSEUDO_RES = new CmsResource( 177 null, 178 null, 179 null, 180 0, 181 false, 182 0, 183 null, 184 null, 185 0L, 186 null, 187 0L, 188 null, 189 0L, 190 0L, 191 0, 192 0, 193 0L, 194 0); 195 196 /** The name of the key that is used for the result documents inside the Solr query response. */ 197 private static final String QUERY_RESPONSE_NAME = "response"; 198 199 /** The name of the key that is used for the query time. */ 200 private static final String QUERY_TIME_NAME = "QTime"; 201 202 /** The name of the key that is used for the query time. */ 203 private static final String QUERY_HIGHLIGHTING_NAME = "highlighting"; 204 205 /** A constant for UTF-8 charset. */ 206 private static final Charset UTF8 = Charset.forName("UTF-8"); 207 208 /** The name of the request parameter holding the debug secret. */ 209 private static final String REQUEST_PARAM_DEBUG_SECRET = "_debug"; 210 211 /** The name of the query parameter enabling spell checking. */ 212 private static final String QUERY_SPELLCHECK_NAME = "spellcheck"; 213 214 /** The name of the query parameter sorting. */ 215 private static final String QUERY_SORT_NAME = "sort"; 216 217 /** The name of the query parameter expand. */ 218 private static final String QUERY_PARAM_EXPAND = "expand"; 219 220 /** The embedded Solr client for this index. */ 221 transient SolrClient m_solr; 222 223 /** The post document manipulator. */ 224 private transient I_CmsSolrPostSearchProcessor m_postProcessor; 225 226 /** The core name for the index. */ 227 private transient String m_coreName; 228 229 /** The list of allowed fields to return. */ 230 private String[] m_handlerAllowedFields; 231 232 /** The number of maximally allowed results per page when using the handler. */ 233 private int m_handlerMaxAllowedResultsPerPage = -1; 234 235 /** The number of maximally allowed results at all when using the handler. */ 236 private int m_handlerMaxAllowedResultsAtAll = -1; 237 238 /** Flag, indicating if the handler only works in debug mode. */ 239 private boolean m_handlerSelectDisabled; 240 241 /** Path to the secret file. Must be under /system/.../ or /shared/.../ and readable by all users that should be able to debug. */ 242 private String m_handlerDebugSecretFile; 243 244 /** Flag, indicating if the spellcheck handler is disabled for the index. */ 245 private boolean m_handlerSpellDisabled; 246 247 /** The maximal number of results to process for search queries. */ 248 int m_maxProcessedResults = -2; // special value for not initialized. 249 250 /** Server URL to use specific for the index. If set, it overwrites all other server settings. */ 251 private String m_serverUrl; 252 253 /** 254 * Default constructor.<p> 255 */ 256 public CmsSolrIndex() { 257 258 super(); 259 } 260 261 /** 262 * Public constructor to create a Solr index.<p> 263 * 264 * @param name the name for this index.<p> 265 * 266 * @throws CmsIllegalArgumentException if something goes wrong 267 */ 268 public CmsSolrIndex(String name) 269 throws CmsIllegalArgumentException { 270 271 super(name); 272 } 273 274 /** 275 * Returns the resource type for the given root path.<p> 276 * 277 * @param cms the current CMS context 278 * @param rootPath the root path of the resource to get the type for 279 * 280 * @return the resource type for the given root path 281 */ 282 public static final String getType(CmsObject cms, String rootPath) { 283 284 String type = null; 285 CmsSolrIndex index = CmsSearchManager.getIndexSolr(cms, null); 286 if (index != null) { 287 I_CmsSearchDocument doc = index.getDocument(CmsSearchField.FIELD_PATH, rootPath); 288 if (doc != null) { 289 type = doc.getFieldValueAsString(CmsSearchField.FIELD_TYPE); 290 } 291 } 292 return type; 293 } 294 295 /** 296 * @see org.opencms.search.CmsSearchIndex#addConfigurationParameter(java.lang.String, java.lang.String) 297 */ 298 @Override 299 public void addConfigurationParameter(String key, String value) { 300 301 switch (key) { 302 case POST_PROCESSOR: 303 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 304 try { 305 setPostProcessor((I_CmsSolrPostSearchProcessor)Class.forName(value).newInstance()); 306 } catch (Exception e) { 307 CmsException ex = new CmsException( 308 Messages.get().container(Messages.LOG_SOLR_ERR_POST_PROCESSOR_NOT_EXIST_1, value), 309 e); 310 LOG.error(ex.getMessage(), ex); 311 } 312 } 313 break; 314 case SOLR_HANDLER_ALLOWED_FIELDS: 315 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 316 m_handlerAllowedFields = Stream.of(value.split(",")).map(v -> v.trim()).toArray(String[]::new); 317 } 318 break; 319 case SOLR_HANDLER_MAX_ALLOWED_RESULTS_PER_PAGE: 320 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 321 try { 322 m_handlerMaxAllowedResultsPerPage = Integer.parseInt(value); 323 } catch (NumberFormatException e) { 324 LOG.warn( 325 "Could not parse parameter \"" 326 + SOLR_HANDLER_MAX_ALLOWED_RESULTS_PER_PAGE 327 + "\" for index \"" 328 + getName() 329 + "\". Results per page will not be restricted."); 330 } 331 } 332 break; 333 case SOLR_HANDLER_MAX_ALLOWED_RESULTS_AT_ALL: 334 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 335 try { 336 m_handlerMaxAllowedResultsAtAll = Integer.parseInt(value); 337 } catch (NumberFormatException e) { 338 LOG.warn( 339 "Could not parse parameter \"" 340 + SOLR_HANDLER_MAX_ALLOWED_RESULTS_AT_ALL 341 + "\" for index \"" 342 + getName() 343 + "\". Results per page will not be restricted."); 344 } 345 } 346 break; 347 case SOLR_HANDLER_DISABLE_SELECT: 348 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 349 m_handlerSelectDisabled = value.trim().toLowerCase().equals("true"); 350 } 351 break; 352 case SOLR_HANDLER_DEBUG_SECRET_FILE: 353 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 354 m_handlerDebugSecretFile = value.trim(); 355 } 356 break; 357 case SOLR_HANDLER_DISABLE_SPELL: 358 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 359 m_handlerSpellDisabled = value.trim().toLowerCase().equals("true"); 360 } 361 break; 362 case SOLR_SEARCH_MAX_PROCESSED_RESULTS: 363 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 364 try { 365 m_maxProcessedResults = Integer.parseInt(value); 366 } catch (NumberFormatException e) { 367 LOG.warn( 368 "Could not parse parameter \"" 369 + SOLR_SEARCH_MAX_PROCESSED_RESULTS 370 + "\" for index \"" 371 + getName() 372 + "\". The global configuration will be used instead."); 373 } 374 } 375 break; 376 case SOLR_SERVER_URL: 377 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 378 m_serverUrl = value.trim(); 379 } 380 break; 381 default: 382 super.addConfigurationParameter(key, value); 383 break; 384 } 385 } 386 387 /** 388 * @see org.opencms.search.CmsSearchIndex#createEmptyDocument(org.opencms.file.CmsResource) 389 */ 390 @Override 391 public I_CmsSearchDocument createEmptyDocument(CmsResource resource) { 392 393 CmsSolrDocument doc = new CmsSolrDocument(new SolrInputDocument()); 394 doc.setId(resource.getStructureId()); 395 return doc; 396 } 397 398 /** 399 * @see org.opencms.search.CmsSearchIndex#createIndexWriter(boolean, org.opencms.report.I_CmsReport) 400 */ 401 @Override 402 public I_CmsIndexWriter createIndexWriter(boolean create, I_CmsReport report) { 403 404 return new CmsSolrIndexWriter(m_solr, this); 405 } 406 407 /** 408 * @see org.opencms.search.CmsSearchIndex#excludeFromIndex(CmsObject, CmsResource) 409 */ 410 @Override 411 public boolean excludeFromIndex(CmsObject cms, CmsResource resource) { 412 413 if (resource.isFolder() || resource.isTemporaryFile()) { 414 // don't index folders or temporary files for galleries, but pretty much everything else 415 return true; 416 } 417 // If this is the default offline index than it is used for gallery search that needs all resources indexed. 418 if (this.getName().equals(DEFAULT_INDEX_NAME_OFFLINE)) { 419 return false; 420 } 421 422 boolean isOnlineIndex = getProject().equals(CmsProject.ONLINE_PROJECT_NAME); 423 if (isOnlineIndex && (resource.getDateExpired() <= System.currentTimeMillis())) { 424 return true; 425 } 426 427 try { 428 // do property lookup with folder search 429 String propValue = cms.readPropertyObject( 430 resource, 431 CmsPropertyDefinition.PROPERTY_SEARCH_EXCLUDE, 432 true).getValue(); 433 if (propValue != null) { 434 if (!("false".equalsIgnoreCase(propValue.trim()))) { 435 return true; 436 } 437 } 438 } catch (CmsException e) { 439 if (LOG.isDebugEnabled()) { 440 LOG.debug( 441 org.opencms.search.Messages.get().getBundle().key( 442 org.opencms.search.Messages.LOG_UNABLE_TO_READ_PROPERTY_1, 443 resource.getRootPath())); 444 } 445 } 446 if (!USE_ALL_LOCALE.equalsIgnoreCase(getLocale().getLanguage())) { 447 // check if any resource default locale has a match with the index locale, if not skip resource 448 List<Locale> locales = OpenCms.getLocaleManager().getDefaultLocales(cms, resource); 449 Locale match = OpenCms.getLocaleManager().getFirstMatchingLocale( 450 Collections.singletonList(getLocale()), 451 locales); 452 return (match == null); 453 } 454 return false; 455 456 } 457 458 /** 459 * Performs a search with according to the gallery search parameters.<p> 460 * 461 * @param cms the cms context 462 * @param params the search parameters 463 * 464 * @return the search result 465 */ 466 public CmsGallerySearchResultList gallerySearch(CmsObject cms, CmsGallerySearchParameters params) { 467 468 CmsGallerySearchResultList resultList = new CmsGallerySearchResultList(); 469 if (params.isForceEmptyResult()) { 470 return resultList; 471 } 472 473 try { 474 CmsSolrResultList list = search( 475 cms, 476 params.getQuery(cms), 477 false, 478 null, 479 true, 480 CmsResourceFilter.ONLY_VISIBLE_NO_DELETED, 481 MAX_RESULTS_GALLERY); // ignore the maximally searched number of contents. 482 483 if (null == list) { 484 return null; 485 } 486 487 resultList.setHitCount(Long.valueOf(list.getNumFound()).intValue()); 488 for (CmsSearchResource resource : list) { 489 I_CmsSearchDocument document = resource.getDocument(); 490 Locale locale = CmsLocaleManager.getLocale(params.getLocale()); 491 492 CmsGallerySearchResult result = new CmsGallerySearchResult( 493 document, 494 cms, 495 (int)document.getScore(), 496 locale); 497 498 resultList.add(result); 499 } 500 } catch (CmsSearchException e) { 501 LOG.error(e.getMessage(), e); 502 } 503 return resultList; 504 } 505 506 /** 507 * @see org.opencms.search.CmsSearchIndex#getConfiguration() 508 */ 509 @Override 510 public CmsParameterConfiguration getConfiguration() { 511 512 CmsParameterConfiguration result = super.getConfiguration(); 513 if (getPostProcessor() != null) { 514 result.put(POST_PROCESSOR, getPostProcessor().getClass().getName()); 515 } 516 return result; 517 } 518 519 /** 520 * Returns the name of the core of the index. 521 * NOTE: Index and core name differ since OpenCms 10.5 due to new naming rules for cores in SOLR. 522 * 523 * @return the name of the core of the index. 524 */ 525 public String getCoreName() { 526 527 return m_coreName; 528 } 529 530 /** 531 * @see org.opencms.search.CmsSearchIndex#getDocument(java.lang.String, java.lang.String) 532 */ 533 @Override 534 public synchronized I_CmsSearchDocument getDocument(String fieldname, String term) { 535 536 return getDocument(fieldname, term, null); 537 } 538 539 /** 540 * Version of {@link org.opencms.search.CmsSearchIndex#getDocument(java.lang.String, java.lang.String)} where 541 * the returned fields can be restricted. 542 * 543 * @param fieldname the field to query in 544 * @param term the query 545 * @param fls the returned fields. 546 * @return the document. 547 */ 548 public synchronized I_CmsSearchDocument getDocument(String fieldname, String term, String[] fls) { 549 550 try { 551 SolrQuery query = new SolrQuery(); 552 if (CmsSearchField.FIELD_PATH.equals(fieldname)) { 553 query.setQuery(fieldname + ":\"" + term + "\""); 554 } else { 555 query.setQuery(fieldname + ":" + term); 556 } 557 // We could have more than one document due to serial dates. We only want one arbitrary document per id/path 558 query.setRows(Integer.valueOf(1)); 559 if (null != fls) { 560 query.setFields(fls); 561 } 562 QueryResponse res = m_solr.query(getCoreName(), query); 563 if (res != null) { 564 SolrDocumentList sdl = m_solr.query(getCoreName(), query).getResults(); 565 if ((sdl.getNumFound() > 0L) && (sdl.get(0) != null)) { 566 return new CmsSolrDocument(sdl.get(0)); 567 } 568 } 569 } catch (Exception e) { 570 // ignore and assume that the document could not be found 571 LOG.error(e.getMessage(), e); 572 } 573 return null; 574 } 575 576 /** 577 * Returns the language locale for the given resource in this index.<p> 578 * 579 * @param cms the current OpenCms user context 580 * @param resource the resource to check 581 * @param availableLocales a list of locales supported by the resource 582 * 583 * @return the language locale for the given resource in this index 584 */ 585 @Override 586 public Locale getLocaleForResource(CmsObject cms, CmsResource resource, List<Locale> availableLocales) { 587 588 Locale result = null; 589 List<Locale> defaultLocales = OpenCms.getLocaleManager().getDefaultLocales(cms, resource); 590 if ((availableLocales != null) && (availableLocales.size() > 0)) { 591 result = OpenCms.getLocaleManager().getBestMatchingLocale( 592 defaultLocales.get(0), 593 defaultLocales, 594 availableLocales); 595 } 596 if (result == null) { 597 result = ((availableLocales != null) && availableLocales.isEmpty()) 598 ? availableLocales.get(0) 599 : defaultLocales.get(0); 600 } 601 return result; 602 } 603 604 /** 605 * Returns the maximal number of results (start + rows) that are processed for each search query unless another 606 * maximum is explicitly specified in {@link #search(CmsObject, CmsSolrQuery, boolean, ServletResponse, boolean, CmsResourceFilter, int)}. 607 * 608 * @return the maximal number of results (start + rows) that are processed for a search query. 609 */ 610 public int getMaxProcessedResults() { 611 612 return m_maxProcessedResults; 613 } 614 615 /** 616 * Returns the search post processor.<p> 617 * 618 * @return the post processor to use 619 */ 620 public I_CmsSolrPostSearchProcessor getPostProcessor() { 621 622 return m_postProcessor; 623 } 624 625 /** 626 * Returns the Solr server URL to connect to for this specific index, or <code>null</code> if no specific URL is configured. 627 * @return the Solr server URL to connect to for this specific index, or <code>null</code> if no specific URL is configured. 628 */ 629 public String getServerUrl() { 630 631 return m_serverUrl; 632 } 633 634 /** 635 * @see org.opencms.search.CmsSearchIndex#initialize() 636 */ 637 @Override 638 public void initialize() throws CmsSearchException { 639 640 super.initialize(); 641 if (m_maxProcessedResults == -2) { 642 m_maxProcessedResults = OpenCms.getSearchManager().getSolrServerConfiguration().getMaxProcessedResults(); 643 } 644 try { 645 OpenCms.getSearchManager().registerSolrIndex(this); 646 } catch (CmsConfigurationException ex) { 647 LOG.error(ex.getMessage(), ex); 648 setEnabled(false); 649 } 650 } 651 652 /** Returns a flag, indicating if the Solr server is not yet set. 653 * @return a flag, indicating if the Solr server is not yet set. 654 */ 655 public boolean isNoSolrServerSet() { 656 657 return null == m_solr; 658 } 659 660 /** 661 * Not yet implemented for Solr.<p> 662 * 663 * <code> 664 * #################<br> 665 * ### DON'T USE ###<br> 666 * #################<br> 667 * </code> 668 * 669 * @deprecated Use {@link #search(CmsObject, SolrQuery)} or {@link #search(CmsObject, String)} instead 670 */ 671 @Override 672 @Deprecated 673 public synchronized CmsSearchResultList search(CmsObject cms, CmsSearchParameters params) { 674 675 throw new UnsupportedOperationException(); 676 } 677 678 /** 679 * Default search method.<p> 680 * 681 * @param cms the current CMS object 682 * @param query the query 683 * 684 * @return the results 685 * 686 * @throws CmsSearchException if something goes wrong 687 * 688 * @see #search(CmsObject, String) 689 */ 690 public CmsSolrResultList search(CmsObject cms, CmsSolrQuery query) throws CmsSearchException { 691 692 return search(cms, query, false); 693 } 694 695 /** 696 * Performs a search.<p> 697 * 698 * Returns a list of 'OpenCms resource documents' 699 * ({@link CmsSearchResource}) encapsulated within the class {@link CmsSolrResultList}. 700 * This list can be accessed exactly like an {@link List} which entries are 701 * {@link CmsSearchResource} that extend {@link CmsResource} and holds the Solr 702 * implementation of {@link I_CmsSearchDocument} as member. <b>This enables you to deal 703 * with the resulting list as you do with well known {@link List} and work on it's entries 704 * like you do on {@link CmsResource}.</b> 705 * 706 * <h4>What will be done with the Solr search result?</h4> 707 * <ul> 708 * <li>Although it can happen, that there are less results returned than rows were requested 709 * (imagine an index containing less documents than requested rows) we try to guarantee 710 * the requested amount of search results and to provide a working pagination with 711 * security check.</li> 712 * 713 * <li>To be sure we get enough documents left even the permission check reduces the amount 714 * of found documents, the rows are multiplied by <code>'5'</code> and the current page 715 * additionally the offset is added. The count of documents we don't have enough 716 * permissions for grows with increasing page number, that's why we also multiply 717 * the rows by the current page count.</li> 718 * 719 * <li>Also make sure we perform the permission check for all found documents, so start with 720 * the first found doc.</li> 721 * </ul> 722 * 723 * <b>NOTE:</b> If latter pages than the current one are containing protected documents the 724 * total hit count will be incorrect, because the permission check ends if we have 725 * enough results found for the page to display. With other words latter pages than 726 * the current can contain documents that will first be checked if those pages are 727 * requested to be displayed, what causes a incorrect hit count.<p> 728 * 729 * @param cms the current OpenCms context 730 * @param ignoreMaxRows <code>true</code> to return all all requested rows, <code>false</code> to use max rows 731 * @param query the OpenCms Solr query 732 * 733 * @return the list of found documents 734 * 735 * @throws CmsSearchException if something goes wrong 736 * 737 * @see org.opencms.search.solr.CmsSolrResultList 738 * @see org.opencms.search.CmsSearchResource 739 * @see org.opencms.search.I_CmsSearchDocument 740 * @see org.opencms.search.solr.CmsSolrQuery 741 */ 742 public CmsSolrResultList search(CmsObject cms, final CmsSolrQuery query, boolean ignoreMaxRows) 743 throws CmsSearchException { 744 745 return search(cms, query, ignoreMaxRows, null, false, null); 746 } 747 748 /** 749 * Like {@link #search(CmsObject, CmsSolrQuery, boolean)}, but additionally a resource filter can be specified. 750 * By default, the filter depends on the index. 751 * 752 * @param cms the current OpenCms context 753 * @param ignoreMaxRows <code>true</code> to return all all requested rows, <code>false</code> to use max rows 754 * @param query the OpenCms Solr query 755 * @param filter the resource filter to use for post-processing. 756 * 757 * @return the list of documents found. 758 * 759 * @throws CmsSearchException if something goes wrong 760 */ 761 public CmsSolrResultList search( 762 CmsObject cms, 763 final CmsSolrQuery query, 764 boolean ignoreMaxRows, 765 final CmsResourceFilter filter) 766 throws CmsSearchException { 767 768 return search(cms, query, ignoreMaxRows, null, false, filter); 769 } 770 771 /** 772 * Performs the actual search.<p> 773 * 774 * @param cms the current OpenCms context 775 * @param query the OpenCms Solr query 776 * @param ignoreMaxRows <code>true</code> to return all all requested rows, <code>false</code> to use max rows 777 * @param response the servlet response to write the query result to, may also be <code>null</code> 778 * @param ignoreSearchExclude if set to false, only contents with search_exclude unset or "false" will be found - typical for the the non-gallery case 779 * @param filter the resource filter to use 780 * 781 * @return the found documents 782 * 783 * @throws CmsSearchException if something goes wrong 784 * 785 * @see #search(CmsObject, CmsSolrQuery, boolean) 786 */ 787 public CmsSolrResultList search( 788 CmsObject cms, 789 final CmsSolrQuery query, 790 boolean ignoreMaxRows, 791 ServletResponse response, 792 boolean ignoreSearchExclude, 793 CmsResourceFilter filter) 794 throws CmsSearchException { 795 796 return search(cms, query, ignoreMaxRows, response, ignoreSearchExclude, filter, getMaxProcessedResults()); 797 } 798 799 /** 800 * Performs the actual search.<p> 801 * 802 * To provide for correct permissions two queries are performed and the response is fused from that queries: 803 * <ol> 804 * <li>a query for permission checking, where fl, start and rows is adjusted. From this query result we take for the response: 805 * <ul> 806 * <li>facets</li> 807 * <li>spellcheck</li> 808 * <li>suggester</li> 809 * <li>morelikethis</li> 810 * <li>clusters</li> 811 * </ul> 812 * </li> 813 * <li>a query that collects only the resources determined by the first query and performs highlighting. From this query we take for the response: 814 * <li>result</li> 815 * <li>highlighting</li> 816 * </li> 817 *</ol> 818 * 819 * Currently not or only partly supported Solr features are: 820 * <ul> 821 * <li>groups</li> 822 * <li>collapse - representatives of the collapsed group might be filtered by the permission check</li> 823 * <li>expand is disabled</li> 824 * </ul> 825 * 826 * @param cms the current OpenCms context 827 * @param query the OpenCms Solr query 828 * @param ignoreMaxRows <code>true</code> to return all requested rows, <code>false</code> to use max rows 829 * @param response the servlet response to write the query result to, may also be <code>null</code> 830 * @param ignoreSearchExclude if set to false, only contents with search_exclude unset or "false" will be found - typical for the the non-gallery case 831 * @param filter the resource filter to use 832 * @param maxNumResults the maximal number of results to search for 833 * 834 * @return the found documents 835 * 836 * @throws CmsSearchException if something goes wrong 837 * 838 * @see #search(CmsObject, CmsSolrQuery, boolean) 839 */ 840 @SuppressWarnings("unchecked") 841 public CmsSolrResultList search( 842 CmsObject cms, 843 final CmsSolrQuery query, 844 boolean ignoreMaxRows, 845 ServletResponse response, 846 boolean ignoreSearchExclude, 847 CmsResourceFilter filter, 848 int maxNumResults) 849 throws CmsSearchException { 850 851 CmsSolrResultList result = null; 852 long startTime = System.currentTimeMillis(); 853 854 // TODO: 855 // - fall back to "last found results" if none are present at the "last page"? 856 // - deal with cursorMarks? 857 // - deal with groups? 858 // - deal with result clustering? 859 // - remove max score calculation? 860 861 if (LOG.isDebugEnabled()) { 862 LOG.debug(Messages.get().getBundle().key(Messages.LOG_SOLR_DEBUG_ORIGINAL_QUERY_2, query, getName())); 863 } 864 865 // change thread priority in order to reduce search impact on overall system performance 866 int previousPriority = Thread.currentThread().getPriority(); 867 if (getPriority() > 0) { 868 Thread.currentThread().setPriority(getPriority()); 869 } 870 871 // check if the user is allowed to access this index 872 checkOfflineAccess(cms); 873 874 if (!ignoreSearchExclude) { 875 if (LOG.isInfoEnabled()) { 876 LOG.info( 877 Messages.get().getBundle().key( 878 Messages.LOG_SOLR_INFO_ADDING_SEARCH_EXCLUDE_FILTER_FOR_QUERY_2, 879 query, 880 getName())); 881 } 882 query.addFilterQuery(CmsSearchField.FIELD_SEARCH_EXCLUDE + ":\"false\""); 883 } 884 885 // get start parameter from the request 886 int start = null == query.getStart() ? 0 : query.getStart().intValue(); 887 888 // correct negative start values to 0. 889 if (start < 0) { 890 query.setStart(Integer.valueOf(0)); 891 start = 0; 892 } 893 894 // Adjust the maximal number of results to process in case it is unlimited. 895 if (maxNumResults < 0) { 896 maxNumResults = Integer.MAX_VALUE; 897 if (LOG.isInfoEnabled()) { 898 LOG.info( 899 Messages.get().getBundle().key( 900 Messages.LOG_SOLR_INFO_LIMITING_MAX_PROCESSED_RESULTS_3, 901 query, 902 getName(), 903 Integer.valueOf(maxNumResults))); 904 } 905 } 906 907 // Correct the rows parameter 908 // Set the default rows, if rows are not set in the original query. 909 int rows = null == query.getRows() ? CmsSolrQuery.DEFAULT_ROWS.intValue() : query.getRows().intValue(); 910 911 // Restrict the rows, such that the maximal number of queryable results is not exceeded. 912 if ((((rows + start) > maxNumResults) || ((rows + start) < 0))) { 913 rows = maxNumResults - start; 914 } 915 // Restrict the rows to the maximally allowed number, if they should be restricted. 916 if (!ignoreMaxRows && (rows > ROWS_MAX)) { 917 if (LOG.isInfoEnabled()) { 918 LOG.info( 919 Messages.get().getBundle().key( 920 Messages.LOG_SOLR_INFO_LIMITING_MAX_ROWS_4, 921 new Object[] {query, getName(), Integer.valueOf(rows), Integer.valueOf(ROWS_MAX)})); 922 } 923 rows = ROWS_MAX; 924 } 925 // If start is higher than maxNumResults, the rows could be negative here - correct this. 926 if (rows < 0) { 927 if (LOG.isInfoEnabled()) { 928 LOG.info( 929 Messages.get().getBundle().key( 930 Messages.LOG_SOLR_INFO_CORRECTING_ROWS_4, 931 new Object[] {query, getName(), Integer.valueOf(rows), Integer.valueOf(0)})); 932 } 933 rows = 0; 934 } 935 // Set the corrected rows for the query. 936 query.setRows(Integer.valueOf(rows)); 937 938 // remove potentially set expand parameter 939 if (null != query.getParams(QUERY_PARAM_EXPAND)) { 940 LOG.info(Messages.get().getBundle().key(Messages.LOG_SOLR_INFO_REMOVING_EXPAND_2, query, getName())); 941 query.remove("expand"); 942 } 943 944 float maxScore = 0; 945 946 LocalSolrQueryRequest solrQueryRequest = null; 947 SolrCore core = null; 948 String[] sortParamValues = query.getParams(QUERY_SORT_NAME); 949 boolean sortByScoreDesc = (null == sortParamValues) 950 || (sortParamValues.length == 0) 951 || Objects.equal(sortParamValues[0], "score desc"); 952 953 try { 954 955 // initialize the search context 956 CmsObject searchCms = OpenCms.initCmsObject(cms); 957 958 //////////////////////////////////////////////////////////////////////////////////////////////////////////////// 959 //////////////////////// QUERY FOR PERMISSION CHECK, FACETS, SPELLCHECK, SUGGESTIONS /////////////////////////// 960 //////////////////////////////////////////////////////////////////////////////////////////////////////////////// 961 962 // Clone the query and keep the original one 963 CmsSolrQuery checkQuery = query.clone(); 964 // Initialize rows, offset, end and the current page. 965 int end = start + rows; 966 int itemsToCheck = 0 == end ? 0 : Math.max(10, end + (end / 5)); // request 20 percent more, but at least 10 results if permissions are filtered 967 // use a set to prevent double entries if multiple check queries are performed. 968 Set<String> resultSolrIds = new HashSet<>(rows); // rows are set before definitely. 969 970 // counter for the documents found and accessible 971 int cnt = 0; 972 long hitCount = 0; 973 long visibleHitCount = 0; 974 int processedResults = 0; 975 long solrPermissionTime = 0; 976 // disable highlighting - it's done in the next query. 977 checkQuery.setHighlight(false); 978 // adjust rows and start for the permission check. 979 checkQuery.setRows(Integer.valueOf(Math.min(maxNumResults - processedResults, itemsToCheck))); 980 checkQuery.setStart(Integer.valueOf(processedResults)); 981 // return only the fields required for the permission check and for scoring 982 checkQuery.setFields(CmsSearchField.FIELD_TYPE, CmsSearchField.FIELD_SOLR_ID, CmsSearchField.FIELD_PATH); 983 List<String> originalFields = Arrays.asList(query.getFields().split(",")); 984 if (originalFields.contains(CmsSearchField.FIELD_SCORE)) { 985 checkQuery.addField(CmsSearchField.FIELD_SCORE); 986 } 987 if (LOG.isDebugEnabled()) { 988 LOG.debug(Messages.get().getBundle().key(Messages.LOG_SOLR_DEBUG_CHECK_QUERY_2, checkQuery, getName())); 989 } 990 // perform the permission check Solr query and remember the response and time Solr took. 991 long solrCheckTime = System.currentTimeMillis(); 992 QueryResponse checkQueryResponse = m_solr.query(getCoreName(), checkQuery); 993 solrCheckTime = System.currentTimeMillis() - solrCheckTime; 994 solrPermissionTime += solrCheckTime; 995 996 // initialize the counts 997 hitCount = checkQueryResponse.getResults().getNumFound(); 998 int maxToProcess = Long.valueOf(Math.min(hitCount, maxNumResults)).intValue(); 999 visibleHitCount = hitCount; 1000 1001 // process found documents 1002 for (SolrDocument doc : checkQueryResponse.getResults()) { 1003 try { 1004 CmsSolrDocument searchDoc = new CmsSolrDocument(doc); 1005 if (needsPermissionCheck(searchDoc) && !hasPermissions(searchCms, searchDoc, filter)) { 1006 visibleHitCount--; 1007 } else { 1008 if (cnt >= start) { 1009 resultSolrIds.add(searchDoc.getFieldValueAsString(CmsSearchField.FIELD_SOLR_ID)); 1010 } 1011 if (sortByScoreDesc && (searchDoc.getScore() > maxScore)) { 1012 maxScore = searchDoc.getScore(); 1013 } 1014 if (++cnt >= end) { 1015 break; 1016 } 1017 } 1018 } catch (Exception e) { 1019 // should not happen, but if it does we want to go on with the next result nevertheless 1020 visibleHitCount--; 1021 LOG.warn(Messages.get().getBundle().key(Messages.LOG_SOLR_ERR_RESULT_ITERATION_FAILED_0), e); 1022 } 1023 } 1024 processedResults += checkQueryResponse.getResults().size(); 1025 1026 if ((resultSolrIds.size() < rows) && (processedResults < maxToProcess)) { 1027 CmsSolrQuery secondCheckQuery = checkQuery.clone(); 1028 // disable all features not necessary, since results are present from the first check query. 1029 secondCheckQuery.setFacet(false); 1030 secondCheckQuery.setMoreLikeThis(false); 1031 secondCheckQuery.set(QUERY_SPELLCHECK_NAME, false); 1032 do { 1033 // query directly more under certain conditions to reduce number of queries 1034 itemsToCheck = itemsToCheck < 3000 ? itemsToCheck * 4 : itemsToCheck; 1035 // adjust rows and start for the permission check. 1036 secondCheckQuery.setRows( 1037 Integer.valueOf( 1038 Long.valueOf(Math.min(maxToProcess - processedResults, itemsToCheck)).intValue())); 1039 secondCheckQuery.setStart(Integer.valueOf(processedResults)); 1040 1041 if (LOG.isDebugEnabled()) { 1042 LOG.debug( 1043 Messages.get().getBundle().key( 1044 Messages.LOG_SOLR_DEBUG_SECONDCHECK_QUERY_2, 1045 secondCheckQuery, 1046 getName())); 1047 } 1048 1049 long solrSecondCheckTime = System.currentTimeMillis(); 1050 QueryResponse secondCheckQueryResponse = m_solr.query(getCoreName(), secondCheckQuery); 1051 processedResults += secondCheckQueryResponse.getResults().size(); 1052 solrSecondCheckTime = System.currentTimeMillis() - solrSecondCheckTime; 1053 solrPermissionTime += solrCheckTime; 1054 1055 // process found documents 1056 for (SolrDocument doc : secondCheckQueryResponse.getResults()) { 1057 try { 1058 CmsSolrDocument searchDoc = new CmsSolrDocument(doc); 1059 String docSolrId = searchDoc.getFieldValueAsString(CmsSearchField.FIELD_SOLR_ID); 1060 if ((needsPermissionCheck(searchDoc) && !hasPermissions(searchCms, searchDoc, filter)) 1061 || resultSolrIds.contains(docSolrId)) { 1062 visibleHitCount--; 1063 } else { 1064 if (cnt >= start) { 1065 resultSolrIds.add(docSolrId); 1066 } 1067 if (sortByScoreDesc && (searchDoc.getScore() > maxScore)) { 1068 maxScore = searchDoc.getScore(); 1069 } 1070 if (++cnt >= end) { 1071 break; 1072 } 1073 } 1074 } catch (Exception e) { 1075 // should not happen, but if it does we want to go on with the next result nevertheless 1076 visibleHitCount--; 1077 LOG.warn( 1078 Messages.get().getBundle().key(Messages.LOG_SOLR_ERR_RESULT_ITERATION_FAILED_0), 1079 e); 1080 } 1081 } 1082 1083 } while ((resultSolrIds.size() < rows) && (processedResults < maxToProcess)); 1084 } 1085 1086 //////////////////////////////////////////////////////////////////////////////////////////////////////////////// 1087 //////////////////////// QUERY FOR RESULTS AND HIGHLIGHTING //////////////////////////////////////////////////// 1088 //////////////////////////////////////////////////////////////////////////////////////////////////////////////// 1089 1090 // the lists storing the found documents that will be returned 1091 List<CmsSearchResource> resourceDocumentList = new ArrayList<CmsSearchResource>(resultSolrIds.size()); 1092 SolrDocumentList solrDocumentList = new SolrDocumentList(); 1093 1094 long solrResultTime = 0; 1095 1096 // If we're using a post-processor, (re-)initialize it before using it 1097 if (m_postProcessor != null) { 1098 m_postProcessor.init(); 1099 } 1100 1101 // build the query for getting the results 1102 SolrQuery queryForResults = query.clone(); 1103 // we add an additional filter, such that we can only find the documents we want to retrieve, as we figured out in the check query. 1104 if (!resultSolrIds.isEmpty()) { 1105 Optional<String> queryFilterString = resultSolrIds.stream().map(a -> '"' + a + '"').reduce( 1106 (a, b) -> a + " OR " + b); 1107 queryForResults.addFilterQuery(CmsSearchField.FIELD_SOLR_ID + ":(" + queryFilterString.get() + ")"); 1108 } 1109 queryForResults.setRows(Integer.valueOf(resultSolrIds.size())); 1110 queryForResults.setStart(Integer.valueOf(0)); 1111 1112 if (LOG.isDebugEnabled()) { 1113 LOG.debug( 1114 Messages.get().getBundle().key(Messages.LOG_SOLR_DEBUG_RESULT_QUERY_2, queryForResults, getName())); 1115 } 1116 // perform the result query. 1117 solrResultTime = System.currentTimeMillis(); 1118 QueryResponse resultQueryResponse = m_solr.query(getCoreName(), queryForResults); 1119 solrResultTime = System.currentTimeMillis() - solrResultTime; 1120 1121 // List containing solr ids of filtered contents for which highlighting has to be removed. 1122 // Since we checked permissions just a few milliseconds ago, this should typically stay empty. 1123 List<String> filteredResultIds = new ArrayList<>(5); 1124 1125 for (SolrDocument doc : resultQueryResponse.getResults()) { 1126 try { 1127 CmsSolrDocument searchDoc = new CmsSolrDocument(doc); 1128 if (needsPermissionCheck(searchDoc)) { 1129 CmsResource resource = filter == null 1130 ? getResource(searchCms, searchDoc) 1131 : getResource(searchCms, searchDoc, filter); 1132 if (null != resource) { 1133 if (m_postProcessor != null) { 1134 doc = m_postProcessor.process( 1135 searchCms, 1136 resource, 1137 (SolrInputDocument)searchDoc.getDocument()); 1138 } 1139 resourceDocumentList.add(new CmsSearchResource(resource, searchDoc)); 1140 solrDocumentList.add(doc); 1141 } else { 1142 filteredResultIds.add(searchDoc.getFieldValueAsString(CmsSearchField.FIELD_SOLR_ID)); 1143 } 1144 } else { // should not happen unless the index has changed since the first query. 1145 resourceDocumentList.add(new CmsSearchResource(PSEUDO_RES, searchDoc)); 1146 solrDocumentList.add(doc); 1147 visibleHitCount--; 1148 } 1149 } catch (Exception e) { 1150 // should not happen, but if it does we want to go on with the next result nevertheless 1151 visibleHitCount--; 1152 LOG.warn(Messages.get().getBundle().key(Messages.LOG_SOLR_ERR_RESULT_ITERATION_FAILED_0), e); 1153 } 1154 } 1155 1156 long processTime = System.currentTimeMillis() - startTime - solrPermissionTime - solrResultTime; 1157 1158 //////////////////////////////////////////////////////////////////////////////////////////////////////////// 1159 //////////////////////// CREATE THE FINAL RESPONSE ///////////////////////////////////////////////////////// 1160 //////////////////////////////////////////////////////////////////////////////////////////////////////////// 1161 1162 // we are manipulating the checkQueryResponse to set up the final response, we want to deliver. 1163 1164 // adjust start, max score and hit count displayed in the result list. 1165 solrDocumentList.setStart(start); 1166 Float finalMaxScore = sortByScoreDesc ? new Float(maxScore) : checkQueryResponse.getResults().getMaxScore(); 1167 solrDocumentList.setMaxScore(finalMaxScore); 1168 solrDocumentList.setNumFound(visibleHitCount); 1169 1170 // Exchange the search parameters in the response header by the ones from the (adjusted) original query. 1171 NamedList<Object> params = ((NamedList<Object>)(checkQueryResponse.getHeader().get(HEADER_PARAMS_NAME))); 1172 params.clear(); 1173 for (String paramName : query.getParameterNames()) { 1174 params.add(paramName, query.get(paramName)); 1175 } 1176 1177 // Fill in the documents to return. 1178 checkQueryResponse.getResponse().setVal( 1179 checkQueryResponse.getResponse().indexOf(QUERY_RESPONSE_NAME, 0), 1180 solrDocumentList); 1181 1182 // Fill in the time, the overall query took, including processing and permission check. 1183 checkQueryResponse.getResponseHeader().setVal( 1184 checkQueryResponse.getResponseHeader().indexOf(QUERY_TIME_NAME, 0), 1185 new Integer(new Long(System.currentTimeMillis() - startTime).intValue())); 1186 1187 // Fill in the highlighting information from the result query. 1188 if (query.getHighlight()) { 1189 NamedList<Object> highlighting = (NamedList<Object>)resultQueryResponse.getResponse().get( 1190 QUERY_HIGHLIGHTING_NAME); 1191 // filter out highlighting for documents where access is not permitted. 1192 for (String filteredId : filteredResultIds) { 1193 highlighting.remove(filteredId); 1194 } 1195 NamedList<Object> completeResponse = new SimpleOrderedMap<Object>(1); 1196 completeResponse.addAll(checkQueryResponse.getResponse()); 1197 completeResponse.add(QUERY_HIGHLIGHTING_NAME, highlighting); 1198 checkQueryResponse.setResponse(completeResponse); 1199 } 1200 1201 // build the result 1202 result = new CmsSolrResultList( 1203 query, 1204 checkQueryResponse, 1205 solrDocumentList, 1206 resourceDocumentList, 1207 start, 1208 new Integer(rows), 1209 Math.min(end, (start + solrDocumentList.size())), 1210 rows > 0 ? (start / rows) + 1 : 0, //page - but matches only in case of equally sized pages and is zero for rows=0 (because this was this way before!?!) 1211 visibleHitCount, 1212 finalMaxScore, 1213 startTime, 1214 System.currentTimeMillis()); 1215 if (LOG.isDebugEnabled()) { 1216 Object[] logParams = new Object[] { 1217 new Long(System.currentTimeMillis() - startTime), 1218 new Long(result.getNumFound()), 1219 new Long(solrPermissionTime + solrResultTime), 1220 new Long(processTime), 1221 new Long(result.getHighlightEndTime() != 0 ? result.getHighlightEndTime() - startTime : 0)}; 1222 LOG.debug( 1223 query.toString() 1224 + "\n" 1225 + Messages.get().getBundle().key(Messages.LOG_SOLR_SEARCH_EXECUTED_5, logParams)); 1226 } 1227 // write the response for the handler 1228 if (response != null) { 1229 // create and return the result 1230 core = m_solr instanceof EmbeddedSolrServer 1231 ? ((EmbeddedSolrServer)m_solr).getCoreContainer().getCore(getCoreName()) 1232 : null; 1233 1234 solrQueryRequest = new LocalSolrQueryRequest(core, query); 1235 SolrQueryResponse solrQueryResponse = new SolrQueryResponse(); 1236 solrQueryResponse.setAllValues(checkQueryResponse.getResponse()); 1237 writeResp(response, solrQueryRequest, solrQueryResponse); 1238 } 1239 } catch ( 1240 1241 Exception e) { 1242 throw new CmsSearchException( 1243 Messages.get().container( 1244 Messages.LOG_SOLR_ERR_SEARCH_EXECUTION_FAILD_1, 1245 CmsEncoder.decode(query.toString()), 1246 e), 1247 e); 1248 } finally { 1249 if (solrQueryRequest != null) { 1250 solrQueryRequest.close(); 1251 } 1252 if (null != core) { 1253 core.close(); 1254 } 1255 // re-set thread to previous priority 1256 Thread.currentThread().setPriority(previousPriority); 1257 } 1258 return result; 1259 } 1260 1261 /** 1262 * Default search method.<p> 1263 * 1264 * @param cms the current CMS object 1265 * @param query the query 1266 * 1267 * @return the results 1268 * 1269 * @throws CmsSearchException if something goes wrong 1270 * 1271 * @see #search(CmsObject, String) 1272 */ 1273 public CmsSolrResultList search(CmsObject cms, SolrQuery query) throws CmsSearchException { 1274 1275 return search(cms, CmsEncoder.decode(query.toString())); 1276 } 1277 1278 /** 1279 * Performs a search.<p> 1280 * 1281 * @param cms the cms object 1282 * @param solrQuery the Solr query 1283 * 1284 * @return a list of documents 1285 * 1286 * @throws CmsSearchException if something goes wrong 1287 * 1288 * @see #search(CmsObject, CmsSolrQuery, boolean) 1289 */ 1290 public CmsSolrResultList search(CmsObject cms, String solrQuery) throws CmsSearchException { 1291 1292 return search(cms, new CmsSolrQuery(null, CmsRequestUtil.createParameterMap(solrQuery)), false); 1293 } 1294 1295 /** 1296 * Writes the response into the writer.<p> 1297 * 1298 * NOTE: Currently not available for HTTP server.<p> 1299 * 1300 * @param response the servlet response 1301 * @param cms the CMS object to use for search 1302 * @param query the Solr query 1303 * @param ignoreMaxRows if to return unlimited results 1304 * 1305 * @throws Exception if there is no embedded server 1306 */ 1307 public void select(ServletResponse response, CmsObject cms, CmsSolrQuery query, boolean ignoreMaxRows) 1308 throws Exception { 1309 1310 throwExceptionIfSafetyRestrictionsAreViolated(cms, query, false); 1311 boolean isOnline = cms.getRequestContext().getCurrentProject().isOnlineProject(); 1312 CmsResourceFilter filter = isOnline ? null : CmsResourceFilter.IGNORE_EXPIRATION; 1313 1314 search(cms, query, ignoreMaxRows, response, false, filter); 1315 } 1316 1317 /** 1318 * Sets the logical key/name of this search index.<p> 1319 * 1320 * @param name the logical key/name of this search index 1321 * 1322 * @throws CmsIllegalArgumentException if the given name is null, empty or already taken by another search index 1323 */ 1324 @Override 1325 public void setName(String name) throws CmsIllegalArgumentException { 1326 1327 super.setName(name); 1328 updateCoreName(); 1329 } 1330 1331 /** 1332 * Sets the search post processor.<p> 1333 * 1334 * @param postProcessor the search post processor to set 1335 */ 1336 public void setPostProcessor(I_CmsSolrPostSearchProcessor postProcessor) { 1337 1338 m_postProcessor = postProcessor; 1339 } 1340 1341 /** 1342 * Sets the Solr server used by this index.<p> 1343 * 1344 * @param client the server to set 1345 */ 1346 public void setSolrServer(SolrClient client) { 1347 1348 m_solr = client; 1349 } 1350 1351 /** 1352 * Executes a spell checking Solr query and returns the Solr query response.<p> 1353 * 1354 * @param res the servlet response 1355 * @param cms the CMS object 1356 * @param q the query 1357 * 1358 * @throws CmsSearchException if something goes wrong 1359 */ 1360 public void spellCheck(ServletResponse res, CmsObject cms, CmsSolrQuery q) throws CmsSearchException { 1361 1362 throwExceptionIfSafetyRestrictionsAreViolated(cms, q, true); 1363 SolrCore core = null; 1364 LocalSolrQueryRequest solrQueryRequest = null; 1365 try { 1366 q.setRequestHandler("/spell"); 1367 q.setRows(Integer.valueOf(0)); 1368 1369 QueryResponse queryResponse = m_solr.query(getCoreName(), q); 1370 1371 List<CmsSearchResource> resourceDocumentList = new ArrayList<CmsSearchResource>(); 1372 SolrDocumentList solrDocumentList = new SolrDocumentList(); 1373 if (m_postProcessor != null) { 1374 for (int i = 0; (i < queryResponse.getResults().size()); i++) { 1375 try { 1376 SolrDocument doc = queryResponse.getResults().get(i); 1377 CmsSolrDocument searchDoc = new CmsSolrDocument(doc); 1378 if (needsPermissionCheck(searchDoc)) { 1379 // only if the document is an OpenCms internal resource perform the permission check 1380 CmsResource resource = getResource(cms, searchDoc); 1381 if (resource != null) { 1382 // permission check performed successfully: the user has read permissions! 1383 if (m_postProcessor != null) { 1384 doc = m_postProcessor.process( 1385 cms, 1386 resource, 1387 (SolrInputDocument)searchDoc.getDocument()); 1388 } 1389 resourceDocumentList.add(new CmsSearchResource(resource, searchDoc)); 1390 solrDocumentList.add(doc); 1391 } 1392 } 1393 } catch (Exception e) { 1394 // should not happen, but if it does we want to go on with the next result nevertheless 1395 LOG.warn(Messages.get().getBundle().key(Messages.LOG_SOLR_ERR_RESULT_ITERATION_FAILED_0), e); 1396 } 1397 } 1398 queryResponse.getResponse().setVal( 1399 queryResponse.getResponse().indexOf(QUERY_RESPONSE_NAME, 0), 1400 solrDocumentList); 1401 } 1402 1403 // create and return the result 1404 core = m_solr instanceof EmbeddedSolrServer 1405 ? ((EmbeddedSolrServer)m_solr).getCoreContainer().getCore(getCoreName()) 1406 : null; 1407 1408 SolrQueryResponse solrQueryResponse = new SolrQueryResponse(); 1409 solrQueryResponse.setAllValues(queryResponse.getResponse()); 1410 1411 // create and initialize the solr request 1412 solrQueryRequest = new LocalSolrQueryRequest(core, solrQueryResponse.getResponseHeader()); 1413 // set the OpenCms Solr query as parameters to the request 1414 solrQueryRequest.setParams(q); 1415 1416 writeResp(res, solrQueryRequest, solrQueryResponse); 1417 1418 } catch (Exception e) { 1419 throw new CmsSearchException( 1420 Messages.get().container(Messages.LOG_SOLR_ERR_SEARCH_EXECUTION_FAILD_1, q), 1421 e); 1422 } finally { 1423 if (solrQueryRequest != null) { 1424 solrQueryRequest.close(); 1425 } 1426 if (core != null) { 1427 core.close(); 1428 } 1429 } 1430 } 1431 1432 /** 1433 * @see org.opencms.search.CmsSearchIndex#createIndexBackup() 1434 */ 1435 @Override 1436 protected String createIndexBackup() { 1437 1438 if (!isBackupReindexing()) { 1439 // if no backup is generated we don't need to do anything 1440 return null; 1441 } 1442 if (m_solr instanceof EmbeddedSolrServer) { 1443 EmbeddedSolrServer ser = (EmbeddedSolrServer)m_solr; 1444 CoreContainer con = ser.getCoreContainer(); 1445 SolrCore core = con.getCore(getCoreName()); 1446 if (core != null) { 1447 try { 1448 SolrRequestHandler h = core.getRequestHandler("/replication"); 1449 if (h instanceof ReplicationHandler) { 1450 h.handleRequest( 1451 new LocalSolrQueryRequest(core, CmsRequestUtil.createParameterMap("?command=backup")), 1452 new SolrQueryResponse()); 1453 } 1454 } finally { 1455 core.close(); 1456 } 1457 } 1458 } 1459 return null; 1460 } 1461 1462 /** 1463 * Check, if the current user has permissions on the document's resource. 1464 * @param cms the context 1465 * @param doc the solr document (from the search result) 1466 * @param filter the resource filter to use for checking permissions 1467 * @return <code>true</code> iff the resource mirrored by the search result can be read by the current user. 1468 */ 1469 protected boolean hasPermissions(CmsObject cms, CmsSolrDocument doc, CmsResourceFilter filter) { 1470 1471 return null != (filter == null ? getResource(cms, doc) : getResource(cms, doc, filter)); 1472 } 1473 1474 /** 1475 * @see org.opencms.search.CmsSearchIndex#indexSearcherClose() 1476 */ 1477 @SuppressWarnings("sync-override") 1478 @Override 1479 protected void indexSearcherClose() { 1480 1481 // nothing to do here 1482 } 1483 1484 /** 1485 * @see org.opencms.search.CmsSearchIndex#indexSearcherOpen(java.lang.String) 1486 */ 1487 @SuppressWarnings("sync-override") 1488 @Override 1489 protected void indexSearcherOpen(final String path) { 1490 1491 // nothing to do here 1492 } 1493 1494 /** 1495 * @see org.opencms.search.CmsSearchIndex#indexSearcherUpdate() 1496 */ 1497 @SuppressWarnings("sync-override") 1498 @Override 1499 protected void indexSearcherUpdate() { 1500 1501 // nothing to do here 1502 } 1503 1504 /** 1505 * Checks if the current user is allowed to access non-online indexes.<p> 1506 * 1507 * To access non-online indexes the current user must be a workplace user at least.<p> 1508 * 1509 * @param cms the CMS object initialized with the current request context / user 1510 * 1511 * @throws CmsSearchException thrown if the access is not permitted 1512 */ 1513 private void checkOfflineAccess(CmsObject cms) throws CmsSearchException { 1514 1515 // If an offline index is being selected, check permissions 1516 if (!CmsProject.ONLINE_PROJECT_NAME.equals(getProject())) { 1517 // only if the user has the role Workplace user, he is allowed to access the Offline index 1518 try { 1519 OpenCms.getRoleManager().checkRole(cms, CmsRole.ELEMENT_AUTHOR); 1520 } catch (CmsRoleViolationException e) { 1521 throw new CmsSearchException( 1522 Messages.get().container( 1523 Messages.LOG_SOLR_ERR_SEARCH_PERMISSION_VIOLATION_2, 1524 getName(), 1525 cms.getRequestContext().getCurrentUser()), 1526 e); 1527 } 1528 } 1529 } 1530 1531 /** 1532 * Generates a valid core name from the provided name (the index name). 1533 * @param name the index name. 1534 * @return the core name 1535 */ 1536 private String generateCoreName(final String name) { 1537 1538 if (name != null) { 1539 return name.replace(" ", "-"); 1540 } 1541 return null; 1542 } 1543 1544 /** 1545 * Checks if the query should be executed using the debug mode where the security restrictions do not apply. 1546 * @param cms the current context. 1547 * @param query the query to execute. 1548 * @return a flag, indicating, if the query should be performed in debug mode. 1549 */ 1550 private boolean isDebug(CmsObject cms, CmsSolrQuery query) { 1551 1552 String[] debugSecretValues = query.remove(REQUEST_PARAM_DEBUG_SECRET); 1553 String debugSecret = (debugSecretValues == null) || (debugSecretValues.length < 1) 1554 ? null 1555 : debugSecretValues[0]; 1556 if ((null != debugSecret) && !debugSecret.trim().isEmpty() && (null != m_handlerDebugSecretFile)) { 1557 try { 1558 CmsFile secretFile = cms.readFile(m_handlerDebugSecretFile); 1559 String secret = new String(secretFile.getContents(), CmsFileUtil.getEncoding(cms, secretFile)); 1560 return secret.trim().equals(debugSecret.trim()); 1561 } catch (Exception e) { 1562 LOG.info( 1563 "Failed to read secret file for index \"" 1564 + getName() 1565 + "\" at path \"" 1566 + m_handlerDebugSecretFile 1567 + "\"."); 1568 } 1569 } 1570 return false; 1571 } 1572 1573 /** 1574 * Throws an exception if the request can for security reasons not be performed. 1575 * Security restrictions can be set via parameters of the index. 1576 * 1577 * @param cms the current context. 1578 * @param query the query. 1579 * @param isSpell flag, indicating if the spellcheck handler is requested. 1580 * @throws CmsSearchException thrown if the query cannot be executed due to security reasons. 1581 */ 1582 private void throwExceptionIfSafetyRestrictionsAreViolated(CmsObject cms, CmsSolrQuery query, boolean isSpell) 1583 throws CmsSearchException { 1584 1585 if (!isDebug(cms, query)) { 1586 if (isSpell) { 1587 if (m_handlerSpellDisabled) { 1588 throw new CmsSearchException(Messages.get().container(Messages.GUI_HANDLER_REQUEST_NOT_ALLOWED_0)); 1589 } 1590 } else { 1591 if (m_handlerSelectDisabled) { 1592 throw new CmsSearchException(Messages.get().container(Messages.GUI_HANDLER_REQUEST_NOT_ALLOWED_0)); 1593 } 1594 int start = null != query.getStart() ? query.getStart().intValue() : 0; 1595 int rows = null != query.getRows() ? query.getRows().intValue() : CmsSolrQuery.DEFAULT_ROWS.intValue(); 1596 if ((m_handlerMaxAllowedResultsAtAll >= 0) && ((rows + start) > m_handlerMaxAllowedResultsAtAll)) { 1597 throw new CmsSearchException( 1598 Messages.get().container( 1599 Messages.GUI_HANDLER_TOO_MANY_RESULTS_REQUESTED_AT_ALL_2, 1600 Integer.valueOf(m_handlerMaxAllowedResultsAtAll), 1601 Integer.valueOf(rows + start))); 1602 } 1603 if ((m_handlerMaxAllowedResultsPerPage >= 0) && (rows > m_handlerMaxAllowedResultsPerPage)) { 1604 throw new CmsSearchException( 1605 Messages.get().container( 1606 Messages.GUI_HANDLER_TOO_MANY_RESULTS_REQUESTED_PER_PAGE_2, 1607 Integer.valueOf(m_handlerMaxAllowedResultsPerPage), 1608 Integer.valueOf(rows))); 1609 } 1610 if ((null != m_handlerAllowedFields) && (Stream.of(m_handlerAllowedFields).anyMatch(x -> true))) { 1611 if (query.getFields().equals(CmsSolrQuery.ALL_RETURN_FIELDS)) { 1612 query.setFields(m_handlerAllowedFields); 1613 } else { 1614 for (String requestedField : query.getFields().split(",")) { 1615 if (Stream.of(m_handlerAllowedFields).noneMatch( 1616 allowedField -> allowedField.equals(requestedField))) { 1617 throw new CmsSearchException( 1618 Messages.get().container( 1619 Messages.GUI_HANDLER_REQUESTED_FIELD_NOT_ALLOWED_2, 1620 requestedField, 1621 Stream.of(m_handlerAllowedFields).reduce("", (a, b) -> a + "," + b))); 1622 } 1623 } 1624 } 1625 } 1626 } 1627 } 1628 } 1629 1630 /** 1631 * Updates the core name to be in sync with the index name. 1632 */ 1633 private void updateCoreName() { 1634 1635 m_coreName = generateCoreName(getName()); 1636 1637 } 1638 1639 /** 1640 * Writes the Solr response.<p> 1641 * 1642 * @param response the servlet response 1643 * @param queryRequest the Solr request 1644 * @param queryResponse the Solr response to write 1645 * 1646 * @throws IOException if sth. goes wrong 1647 * @throws UnsupportedEncodingException if sth. goes wrong 1648 */ 1649 private void writeResp(ServletResponse response, SolrQueryRequest queryRequest, SolrQueryResponse queryResponse) 1650 throws IOException, UnsupportedEncodingException { 1651 1652 if (m_solr instanceof EmbeddedSolrServer) { 1653 SolrCore core = ((EmbeddedSolrServer)m_solr).getCoreContainer().getCore(getCoreName()); 1654 Writer out = null; 1655 try { 1656 QueryResponseWriter responseWriter = core.getQueryResponseWriter(queryRequest); 1657 1658 final String ct = responseWriter.getContentType(queryRequest, queryResponse); 1659 if (null != ct) { 1660 response.setContentType(ct); 1661 } 1662 1663 if (responseWriter instanceof BinaryQueryResponseWriter) { 1664 BinaryQueryResponseWriter binWriter = (BinaryQueryResponseWriter)responseWriter; 1665 binWriter.write(response.getOutputStream(), queryRequest, queryResponse); 1666 } else { 1667 String charset = ContentStreamBase.getCharsetFromContentType(ct); 1668 out = ((charset == null) || charset.equalsIgnoreCase(UTF8.toString())) 1669 ? new OutputStreamWriter(response.getOutputStream(), UTF8) 1670 : new OutputStreamWriter(response.getOutputStream(), charset); 1671 out = new FastWriter(out); 1672 responseWriter.write(out, queryRequest, queryResponse); 1673 out.flush(); 1674 } 1675 } finally { 1676 core.close(); 1677 if (out != null) { 1678 out.close(); 1679 } 1680 } 1681 } else { 1682 throw new UnsupportedOperationException(); 1683 } 1684 } 1685}