001/* 002 * File : $Source$ 003 * Date : $Date$ 004 * Version: $Revision$ 005 * 006 * This library is part of OpenCms - 007 * the Open Source Content Management System 008 * 009 * Copyright (C) 2002 - 2009 Alkacon Software (http://www.alkacon.com) 010 * 011 * This library is free software; you can redistribute it and/or 012 * modify it under the terms of the GNU Lesser General Public 013 * License as published by the Free Software Foundation; either 014 * version 2.1 of the License, or (at your option) any later version. 015 * 016 * This library is distributed in the hope that it will be useful, 017 * but WITHOUT ANY WARRANTY; without even the implied warranty of 018 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 019 * Lesser General Public License for more details. 020 * 021 * For further information about Alkacon Software, please see the 022 * company website: http://www.alkacon.com 023 * 024 * For further information about OpenCms, please see the 025 * project website: http://www.opencms.org 026 * 027 * You should have received a copy of the GNU Lesser General Public 028 * License along with this library; if not, write to the Free Software 029 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 030 */ 031 032package org.opencms.search.solr; 033 034import org.opencms.configuration.CmsConfigurationException; 035import org.opencms.configuration.CmsParameterConfiguration; 036import org.opencms.file.CmsFile; 037import org.opencms.file.CmsObject; 038import org.opencms.file.CmsProject; 039import org.opencms.file.CmsPropertyDefinition; 040import org.opencms.file.CmsResource; 041import org.opencms.file.CmsResourceFilter; 042import org.opencms.i18n.CmsEncoder; 043import org.opencms.i18n.CmsLocaleManager; 044import org.opencms.main.CmsException; 045import org.opencms.main.CmsIllegalArgumentException; 046import org.opencms.main.CmsLog; 047import org.opencms.main.OpenCms; 048import org.opencms.report.I_CmsReport; 049import org.opencms.search.CmsSearchException; 050import org.opencms.search.CmsSearchIndex; 051import org.opencms.search.CmsSearchManager; 052import org.opencms.search.CmsSearchParameters; 053import org.opencms.search.CmsSearchResource; 054import org.opencms.search.CmsSearchResultList; 055import org.opencms.search.I_CmsIndexWriter; 056import org.opencms.search.I_CmsSearchDocument; 057import org.opencms.search.fields.CmsSearchField; 058import org.opencms.search.galleries.CmsGallerySearchParameters; 059import org.opencms.search.galleries.CmsGallerySearchResult; 060import org.opencms.search.galleries.CmsGallerySearchResultList; 061import org.opencms.security.CmsRole; 062import org.opencms.security.CmsRoleViolationException; 063import org.opencms.util.CmsFileUtil; 064import org.opencms.util.CmsRequestUtil; 065import org.opencms.util.CmsStringUtil; 066 067import java.io.IOException; 068import java.io.OutputStreamWriter; 069import java.io.UnsupportedEncodingException; 070import java.io.Writer; 071import java.nio.charset.Charset; 072import java.util.ArrayList; 073import java.util.Arrays; 074import java.util.Collections; 075import java.util.HashSet; 076import java.util.List; 077import java.util.Locale; 078import java.util.Set; 079import java.util.stream.Collectors; 080import java.util.stream.Stream; 081 082import javax.servlet.ServletResponse; 083 084import org.apache.commons.logging.Log; 085import org.apache.solr.client.solrj.SolrClient; 086import org.apache.solr.client.solrj.SolrQuery; 087import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; 088import org.apache.solr.client.solrj.response.QueryResponse; 089import org.apache.solr.common.SolrDocument; 090import org.apache.solr.common.SolrDocumentList; 091import org.apache.solr.common.SolrInputDocument; 092import org.apache.solr.common.util.ContentStreamBase; 093import org.apache.solr.common.util.FastWriter; 094import org.apache.solr.common.util.NamedList; 095import org.apache.solr.common.util.SimpleOrderedMap; 096import org.apache.solr.core.CoreContainer; 097import org.apache.solr.core.SolrCore; 098import org.apache.solr.handler.ReplicationHandler; 099import org.apache.solr.request.LocalSolrQueryRequest; 100import org.apache.solr.request.SolrQueryRequest; 101import org.apache.solr.request.SolrRequestHandler; 102import org.apache.solr.response.BinaryQueryResponseWriter; 103import org.apache.solr.response.QueryResponseWriter; 104import org.apache.solr.response.SolrQueryResponse; 105 106import com.google.common.base.Objects; 107 108/** 109 * Implements the search within an Solr index.<p> 110 * 111 * @since 8.5.0 112 */ 113public class CmsSolrIndex extends CmsSearchIndex { 114 115 /** The serial version id. */ 116 private static final long serialVersionUID = -1570077792574476721L; 117 118 /** The name of the default Solr Offline index. */ 119 public static final String DEFAULT_INDEX_NAME_OFFLINE = "Solr Offline"; 120 121 /** The name of the default Solr Online index. */ 122 public static final String DEFAULT_INDEX_NAME_ONLINE = "Solr Online"; 123 124 /** Constant for additional parameter to set the post processor class name. */ 125 public static final String POST_PROCESSOR = "search.solr.postProcessor"; 126 127 /** 128 * Constant for additional parameter to set the maximally processed results (start + rows) for searches with this index. 129 * It overwrites the global configuration from {@link CmsSolrConfiguration#getMaxProcessedResults()} for this index. 130 **/ 131 public static final String SOLR_SEARCH_MAX_PROCESSED_RESULTS = "search.solr.maxProcessedResults"; 132 133 /** Constant for additional parameter to set the fields the select handler should return at maximum. */ 134 public static final String SOLR_HANDLER_ALLOWED_FIELDS = "handle.solr.allowedFields"; 135 136 /** Constant for additional parameter to set the number results the select handler should return at maxium per request. */ 137 public static final String SOLR_HANDLER_MAX_ALLOWED_RESULTS_PER_PAGE = "handle.solr.maxAllowedResultsPerPage"; 138 139 /** Constant for additional parameter to set the maximal number of a result, the select handler should return. */ 140 public static final String SOLR_HANDLER_MAX_ALLOWED_RESULTS_AT_ALL = "handle.solr.maxAllowedResultsAtAll"; 141 142 /** Constant for additional parameter to disable the select handler (except for debug mode). */ 143 private static final String SOLR_HANDLER_DISABLE_SELECT = "handle.solr.disableSelectHandler"; 144 145 /** Constant for additional parameter to set the VFS path to the file holding the debug secret. */ 146 private static final String SOLR_HANDLER_DEBUG_SECRET_FILE = "handle.solr.debugSecretFile"; 147 148 /** Constant for additional parameter to disable the spell handler (except for debug mode). */ 149 private static final String SOLR_HANDLER_DISABLE_SPELL = "handle.solr.disableSpellHandler"; 150 151 /** Constant for additional parameter to configure an external solr server specifically for the index. */ 152 private static final String SOLR_SERVER_URL = "server.url"; 153 154 /** The solr exclude property. */ 155 public static final String PROPERTY_SEARCH_EXCLUDE_VALUE_SOLR = "solr"; 156 157 /** Indicates the maximum number of documents from the complete result set to return. */ 158 public static final int ROWS_MAX = 50; 159 160 /** The constant for an unlimited maximum number of results to return in a Solr search. */ 161 public static final int MAX_RESULTS_UNLIMITED = -1; 162 163 /** The constant for an unlimited maximum number of results to return in a Solr search. */ 164 public static final int MAX_RESULTS_GALLERY = 10000; 165 166 /** A constant for debug formatting output. */ 167 protected static final int DEBUG_PADDING_RIGHT = 50; 168 169 /** The name for the parameters key of the response header. */ 170 private static final String HEADER_PARAMS_NAME = "params"; 171 172 /** The log object for this class. */ 173 private static final Log LOG = CmsLog.getLog(CmsSolrIndex.class); 174 175 /** Pseudo resource used for not permission checked indexes. */ 176 private static final CmsResource PSEUDO_RES = new CmsResource( 177 null, 178 null, 179 null, 180 0, 181 false, 182 0, 183 null, 184 null, 185 0L, 186 null, 187 0L, 188 null, 189 0L, 190 0L, 191 0, 192 0, 193 0L, 194 0); 195 196 /** The name of the key that is used for the result documents inside the Solr query response. */ 197 private static final String QUERY_RESPONSE_NAME = "response"; 198 199 /** The name of the key that is used for the query time. */ 200 private static final String QUERY_TIME_NAME = "QTime"; 201 202 /** The name of the key that is used for the query time. */ 203 private static final String QUERY_HIGHLIGHTING_NAME = "highlighting"; 204 205 /** A constant for UTF-8 charset. */ 206 private static final Charset UTF8 = Charset.forName("UTF-8"); 207 208 /** The name of the request parameter holding the debug secret. */ 209 private static final String REQUEST_PARAM_DEBUG_SECRET = "_debug"; 210 211 /** The name of the query parameter enabling spell checking. */ 212 private static final String QUERY_SPELLCHECK_NAME = "spellcheck"; 213 214 /** The name of the query parameter sorting. */ 215 private static final String QUERY_SORT_NAME = "sort"; 216 217 /** The name of the query parameter expand. */ 218 private static final String QUERY_PARAM_EXPAND = "expand"; 219 220 /** The embedded Solr client for this index. */ 221 transient SolrClient m_solr; 222 223 /** The post document manipulator. */ 224 private transient I_CmsSolrPostSearchProcessor m_postProcessor; 225 226 /** The core name for the index. */ 227 private transient String m_coreName; 228 229 /** The list of allowed fields to return. */ 230 private String[] m_handlerAllowedFields; 231 232 /** The number of maximally allowed results per page when using the handler. */ 233 private int m_handlerMaxAllowedResultsPerPage = -1; 234 235 /** The number of maximally allowed results at all when using the handler. */ 236 private int m_handlerMaxAllowedResultsAtAll = -1; 237 238 /** Flag, indicating if the handler only works in debug mode. */ 239 private boolean m_handlerSelectDisabled; 240 241 /** Path to the secret file. Must be under /system/.../ or /shared/.../ and readable by all users that should be able to debug. */ 242 private String m_handlerDebugSecretFile; 243 244 /** Flag, indicating if the spellcheck handler is disabled for the index. */ 245 private boolean m_handlerSpellDisabled; 246 247 /** The maximal number of results to process for search queries. */ 248 int m_maxProcessedResults = -2; // special value for not initialized. 249 250 /** Server URL to use specific for the index. If set, it overwrites all other server settings. */ 251 private String m_serverUrl; 252 253 /** 254 * Default constructor.<p> 255 */ 256 public CmsSolrIndex() { 257 258 super(); 259 } 260 261 /** 262 * Public constructor to create a Solr index.<p> 263 * 264 * @param name the name for this index.<p> 265 * 266 * @throws CmsIllegalArgumentException if something goes wrong 267 */ 268 public CmsSolrIndex(String name) 269 throws CmsIllegalArgumentException { 270 271 super(name); 272 } 273 274 /** 275 * Returns the resource type for the given root path.<p> 276 * 277 * @param cms the current CMS context 278 * @param rootPath the root path of the resource to get the type for 279 * 280 * @return the resource type for the given root path 281 */ 282 public static final String getType(CmsObject cms, String rootPath) { 283 284 String type = null; 285 CmsSolrIndex index = CmsSearchManager.getIndexSolr(cms, null); 286 if (index != null) { 287 I_CmsSearchDocument doc = index.getDocument(CmsSearchField.FIELD_PATH, rootPath); 288 if (doc != null) { 289 type = doc.getFieldValueAsString(CmsSearchField.FIELD_TYPE); 290 } 291 } 292 return type; 293 } 294 295 /** 296 * @see org.opencms.search.CmsSearchIndex#addConfigurationParameter(java.lang.String, java.lang.String) 297 */ 298 @Override 299 public void addConfigurationParameter(String key, String value) { 300 301 switch (key) { 302 case POST_PROCESSOR: 303 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 304 try { 305 setPostProcessor((I_CmsSolrPostSearchProcessor)Class.forName(value).newInstance()); 306 } catch (Exception e) { 307 CmsException ex = new CmsException( 308 Messages.get().container(Messages.LOG_SOLR_ERR_POST_PROCESSOR_NOT_EXIST_1, value), 309 e); 310 LOG.error(ex.getMessage(), ex); 311 } 312 } 313 break; 314 case SOLR_HANDLER_ALLOWED_FIELDS: 315 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 316 m_handlerAllowedFields = Stream.of(value.split(",")).map(v -> v.trim()).toArray(String[]::new); 317 } 318 break; 319 case SOLR_HANDLER_MAX_ALLOWED_RESULTS_PER_PAGE: 320 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 321 try { 322 m_handlerMaxAllowedResultsPerPage = Integer.parseInt(value); 323 } catch (NumberFormatException e) { 324 LOG.warn( 325 "Could not parse parameter \"" 326 + SOLR_HANDLER_MAX_ALLOWED_RESULTS_PER_PAGE 327 + "\" for index \"" 328 + getName() 329 + "\". Results per page will not be restricted."); 330 } 331 } 332 break; 333 case SOLR_HANDLER_MAX_ALLOWED_RESULTS_AT_ALL: 334 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 335 try { 336 m_handlerMaxAllowedResultsAtAll = Integer.parseInt(value); 337 } catch (NumberFormatException e) { 338 LOG.warn( 339 "Could not parse parameter \"" 340 + SOLR_HANDLER_MAX_ALLOWED_RESULTS_AT_ALL 341 + "\" for index \"" 342 + getName() 343 + "\". Results per page will not be restricted."); 344 } 345 } 346 break; 347 case SOLR_HANDLER_DISABLE_SELECT: 348 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 349 m_handlerSelectDisabled = value.trim().toLowerCase().equals("true"); 350 } 351 break; 352 case SOLR_HANDLER_DEBUG_SECRET_FILE: 353 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 354 m_handlerDebugSecretFile = value.trim(); 355 } 356 break; 357 case SOLR_HANDLER_DISABLE_SPELL: 358 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 359 m_handlerSpellDisabled = value.trim().toLowerCase().equals("true"); 360 } 361 break; 362 case SOLR_SEARCH_MAX_PROCESSED_RESULTS: 363 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 364 try { 365 m_maxProcessedResults = Integer.parseInt(value); 366 } catch (NumberFormatException e) { 367 LOG.warn( 368 "Could not parse parameter \"" 369 + SOLR_SEARCH_MAX_PROCESSED_RESULTS 370 + "\" for index \"" 371 + getName() 372 + "\". The global configuration will be used instead."); 373 } 374 } 375 break; 376 case SOLR_SERVER_URL: 377 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) { 378 m_serverUrl = value.trim(); 379 } 380 break; 381 default: 382 super.addConfigurationParameter(key, value); 383 break; 384 } 385 } 386 387 /** 388 * @see org.opencms.search.CmsSearchIndex#createEmptyDocument(org.opencms.file.CmsResource) 389 */ 390 @Override 391 public I_CmsSearchDocument createEmptyDocument(CmsResource resource) { 392 393 CmsSolrDocument doc = new CmsSolrDocument(new SolrInputDocument()); 394 doc.setId(resource.getStructureId()); 395 return doc; 396 } 397 398 /** 399 * @see org.opencms.search.CmsSearchIndex#createIndexWriter(boolean, org.opencms.report.I_CmsReport) 400 */ 401 @Override 402 public I_CmsIndexWriter createIndexWriter(boolean create, I_CmsReport report) { 403 404 return new CmsSolrIndexWriter(m_solr, this); 405 } 406 407 /** 408 * @see org.opencms.search.CmsSearchIndex#excludeFromIndex(CmsObject, CmsResource) 409 */ 410 @Override 411 public boolean excludeFromIndex(CmsObject cms, CmsResource resource) { 412 413 if (resource.isFolder() || resource.isTemporaryFile()) { 414 // don't index folders or temporary files for galleries, but pretty much everything else 415 return true; 416 } 417 // If this is the default offline index than it is used for gallery search that needs all resources indexed. 418 if (this.getName().equals(DEFAULT_INDEX_NAME_OFFLINE)) { 419 return false; 420 } 421 422 boolean isOnlineIndex = getProject().equals(CmsProject.ONLINE_PROJECT_NAME); 423 if (isOnlineIndex && (resource.getDateExpired() <= System.currentTimeMillis())) { 424 return true; 425 } 426 427 try { 428 // do property lookup with folder search 429 String propValue = cms.readPropertyObject( 430 resource, 431 CmsPropertyDefinition.PROPERTY_SEARCH_EXCLUDE, 432 true).getValue(); 433 if (propValue != null) { 434 if (!("false".equalsIgnoreCase(propValue.trim()))) { 435 return true; 436 } 437 } 438 } catch (CmsException e) { 439 if (LOG.isDebugEnabled()) { 440 LOG.debug( 441 org.opencms.search.Messages.get().getBundle().key( 442 org.opencms.search.Messages.LOG_UNABLE_TO_READ_PROPERTY_1, 443 resource.getRootPath())); 444 } 445 } 446 if (!USE_ALL_LOCALE.equalsIgnoreCase(getLocale().getLanguage())) { 447 // check if any resource default locale has a match with the index locale, if not skip resource 448 List<Locale> locales = OpenCms.getLocaleManager().getDefaultLocales(cms, resource); 449 Locale match = OpenCms.getLocaleManager().getFirstMatchingLocale( 450 Collections.singletonList(getLocale()), 451 locales); 452 return (match == null); 453 } 454 return false; 455 456 } 457 458 /** 459 * Performs a search with according to the gallery search parameters.<p> 460 * 461 * @param cms the cms context 462 * @param params the search parameters 463 * 464 * @return the search result 465 */ 466 public CmsGallerySearchResultList gallerySearch(CmsObject cms, CmsGallerySearchParameters params) { 467 468 CmsGallerySearchResultList resultList = new CmsGallerySearchResultList(); 469 if (params.isForceEmptyResult()) { 470 return resultList; 471 } 472 473 try { 474 CmsSolrResultList list = search( 475 cms, 476 params.getQuery(cms), 477 false, 478 null, 479 true, 480 CmsResourceFilter.ONLY_VISIBLE_NO_DELETED, 481 MAX_RESULTS_GALLERY); // ignore the maximally searched number of contents. 482 483 if (null == list) { 484 return null; 485 } 486 487 resultList.setHitCount(Long.valueOf(list.getNumFound()).intValue()); 488 for (CmsSearchResource resource : list) { 489 I_CmsSearchDocument document = resource.getDocument(); 490 Locale locale = CmsLocaleManager.getLocale(params.getLocale()); 491 492 CmsGallerySearchResult result = new CmsGallerySearchResult( 493 document, 494 cms, 495 (int)document.getScore(), 496 locale); 497 498 resultList.add(result); 499 } 500 } catch (CmsSearchException e) { 501 LOG.error(e.getMessage(), e); 502 } 503 return resultList; 504 } 505 506 /** 507 * @see org.opencms.search.CmsSearchIndex#getConfiguration() 508 */ 509 @Override 510 public CmsParameterConfiguration getConfiguration() { 511 512 CmsParameterConfiguration result = super.getConfiguration(); 513 if (getPostProcessor() != null) { 514 result.put(POST_PROCESSOR, getPostProcessor().getClass().getName()); 515 } 516 return result; 517 } 518 519 /** 520 * Returns the name of the core of the index. 521 * NOTE: Index and core name differ since OpenCms 10.5 due to new naming rules for cores in SOLR. 522 * 523 * @return the name of the core of the index. 524 */ 525 public String getCoreName() { 526 527 return m_coreName; 528 } 529 530 /** 531 * @see org.opencms.search.CmsSearchIndex#getDocument(java.lang.String, java.lang.String) 532 */ 533 @Override 534 public synchronized I_CmsSearchDocument getDocument(String fieldname, String term) { 535 536 return getDocument(fieldname, term, null); 537 } 538 539 /** 540 * Version of {@link org.opencms.search.CmsSearchIndex#getDocument(java.lang.String, java.lang.String)} where 541 * the returned fields can be restricted. 542 * 543 * @param fieldname the field to query in 544 * @param term the query 545 * @param fls the returned fields. 546 * @return the document. 547 */ 548 public synchronized I_CmsSearchDocument getDocument(String fieldname, String term, String[] fls) { 549 550 try { 551 SolrQuery query = new SolrQuery(); 552 if (CmsSearchField.FIELD_PATH.equals(fieldname)) { 553 query.setQuery(fieldname + ":\"" + term + "\""); 554 } else { 555 query.setQuery(fieldname + ":" + term); 556 } 557 // We could have more than one document due to serial dates. We only want one arbitrary document per id/path 558 query.setRows(Integer.valueOf(1)); 559 if (null != fls) { 560 query.setFields(fls); 561 } 562 QueryResponse res = m_solr.query(getCoreName(), query); 563 if (res != null) { 564 SolrDocumentList sdl = m_solr.query(getCoreName(), query).getResults(); 565 if ((sdl.getNumFound() > 0L) && (sdl.get(0) != null)) { 566 return new CmsSolrDocument(sdl.get(0)); 567 } 568 } 569 } catch (Exception e) { 570 // ignore and assume that the document could not be found 571 LOG.error(e.getMessage(), e); 572 } 573 return null; 574 } 575 576 /** 577 * Returns the language locale for the given resource in this index.<p> 578 * 579 * @param cms the current OpenCms user context 580 * @param resource the resource to check 581 * @param availableLocales a list of locales supported by the resource 582 * 583 * @return the language locale for the given resource in this index 584 */ 585 @Override 586 public Locale getLocaleForResource(CmsObject cms, CmsResource resource, List<Locale> availableLocales) { 587 588 Locale result = null; 589 List<Locale> defaultLocales = OpenCms.getLocaleManager().getDefaultLocales(cms, resource); 590 if ((availableLocales != null) && (availableLocales.size() > 0)) { 591 result = OpenCms.getLocaleManager().getBestMatchingLocale( 592 defaultLocales.get(0), 593 defaultLocales, 594 availableLocales); 595 } 596 if (result == null) { 597 result = ((availableLocales != null) && availableLocales.isEmpty()) 598 ? availableLocales.get(0) 599 : defaultLocales.get(0); 600 } 601 return result; 602 } 603 604 /** 605 * Returns the maximal number of results (start + rows) that are processed for each search query unless another 606 * maximum is explicitly specified in {@link #search(CmsObject, CmsSolrQuery, boolean, ServletResponse, boolean, CmsResourceFilter, int)}. 607 * 608 * @return the maximal number of results (start + rows) that are processed for a search query. 609 */ 610 public int getMaxProcessedResults() { 611 612 return m_maxProcessedResults; 613 } 614 615 /** 616 * Returns the search post processor.<p> 617 * 618 * @return the post processor to use 619 */ 620 public I_CmsSolrPostSearchProcessor getPostProcessor() { 621 622 return m_postProcessor; 623 } 624 625 /** 626 * Returns the Solr server URL to connect to for this specific index, or <code>null</code> if no specific URL is configured. 627 * @return the Solr server URL to connect to for this specific index, or <code>null</code> if no specific URL is configured. 628 */ 629 public String getServerUrl() { 630 631 return m_serverUrl; 632 } 633 634 /** 635 * @see org.opencms.search.CmsSearchIndex#initialize() 636 */ 637 @Override 638 public void initialize() throws CmsSearchException { 639 640 super.initialize(); 641 if (m_maxProcessedResults == -2) { 642 m_maxProcessedResults = OpenCms.getSearchManager().getSolrServerConfiguration().getMaxProcessedResults(); 643 } 644 try { 645 OpenCms.getSearchManager().registerSolrIndex(this); 646 } catch (CmsConfigurationException ex) { 647 LOG.error(ex.getMessage(), ex); 648 setEnabled(false); 649 } 650 } 651 652 /** Returns a flag, indicating if the Solr server is not yet set. 653 * @return a flag, indicating if the Solr server is not yet set. 654 */ 655 public boolean isNoSolrServerSet() { 656 657 return null == m_solr; 658 } 659 660 /** 661 * Not yet implemented for Solr.<p> 662 * 663 * <code> 664 * #################<br> 665 * ### DON'T USE ###<br> 666 * #################<br> 667 * </code> 668 * 669 * @deprecated Use {@link #search(CmsObject, SolrQuery)} or {@link #search(CmsObject, String)} instead 670 */ 671 @Override 672 @Deprecated 673 public synchronized CmsSearchResultList search(CmsObject cms, CmsSearchParameters params) { 674 675 throw new UnsupportedOperationException(); 676 } 677 678 /** 679 * Default search method.<p> 680 * 681 * @param cms the current CMS object 682 * @param query the query 683 * 684 * @return the results 685 * 686 * @throws CmsSearchException if something goes wrong 687 * 688 * @see #search(CmsObject, String) 689 */ 690 public CmsSolrResultList search(CmsObject cms, CmsSolrQuery query) throws CmsSearchException { 691 692 return search(cms, query, false); 693 } 694 695 /** 696 * Performs a search.<p> 697 * 698 * Returns a list of 'OpenCms resource documents' 699 * ({@link CmsSearchResource}) encapsulated within the class {@link CmsSolrResultList}. 700 * This list can be accessed exactly like an {@link List} which entries are 701 * {@link CmsSearchResource} that extend {@link CmsResource} and holds the Solr 702 * implementation of {@link I_CmsSearchDocument} as member. <b>This enables you to deal 703 * with the resulting list as you do with well known {@link List} and work on it's entries 704 * like you do on {@link CmsResource}.</b> 705 * 706 * <h4>What will be done with the Solr search result?</h4> 707 * <ul> 708 * <li>Although it can happen, that there are less results returned than rows were requested 709 * (imagine an index containing less documents than requested rows) we try to guarantee 710 * the requested amount of search results and to provide a working pagination with 711 * security check.</li> 712 * 713 * <li>To be sure we get enough documents left even the permission check reduces the amount 714 * of found documents, the rows are multiplied by <code>'5'</code> and the current page 715 * additionally the offset is added. The count of documents we don't have enough 716 * permissions for grows with increasing page number, that's why we also multiply 717 * the rows by the current page count.</li> 718 * 719 * <li>Also make sure we perform the permission check for all found documents, so start with 720 * the first found doc.</li> 721 * </ul> 722 * 723 * <b>NOTE:</b> If latter pages than the current one are containing protected documents the 724 * total hit count will be incorrect, because the permission check ends if we have 725 * enough results found for the page to display. With other words latter pages than 726 * the current can contain documents that will first be checked if those pages are 727 * requested to be displayed, what causes a incorrect hit count.<p> 728 * 729 * @param cms the current OpenCms context 730 * @param ignoreMaxRows <code>true</code> to return all all requested rows, <code>false</code> to use max rows 731 * @param query the OpenCms Solr query 732 * 733 * @return the list of found documents 734 * 735 * @throws CmsSearchException if something goes wrong 736 * 737 * @see org.opencms.search.solr.CmsSolrResultList 738 * @see org.opencms.search.CmsSearchResource 739 * @see org.opencms.search.I_CmsSearchDocument 740 * @see org.opencms.search.solr.CmsSolrQuery 741 */ 742 public CmsSolrResultList search(CmsObject cms, final CmsSolrQuery query, boolean ignoreMaxRows) 743 throws CmsSearchException { 744 745 return search(cms, query, ignoreMaxRows, null, false, null); 746 } 747 748 /** 749 * Like {@link #search(CmsObject, CmsSolrQuery, boolean)}, but additionally a resource filter can be specified. 750 * By default, the filter depends on the index. 751 * 752 * @param cms the current OpenCms context 753 * @param ignoreMaxRows <code>true</code> to return all all requested rows, <code>false</code> to use max rows 754 * @param query the OpenCms Solr query 755 * @param filter the resource filter to use for post-processing. 756 * 757 * @return the list of documents found. 758 * 759 * @throws CmsSearchException if something goes wrong 760 */ 761 public CmsSolrResultList search( 762 CmsObject cms, 763 final CmsSolrQuery query, 764 boolean ignoreMaxRows, 765 final CmsResourceFilter filter) 766 throws CmsSearchException { 767 768 return search(cms, query, ignoreMaxRows, null, false, filter); 769 } 770 771 /** 772 * Performs the actual search.<p> 773 * 774 * @param cms the current OpenCms context 775 * @param query the OpenCms Solr query 776 * @param ignoreMaxRows <code>true</code> to return all all requested rows, <code>false</code> to use max rows 777 * @param response the servlet response to write the query result to, may also be <code>null</code> 778 * @param ignoreSearchExclude if set to false, only contents with search_exclude unset or "false" will be found - typical for the the non-gallery case 779 * @param filter the resource filter to use 780 * 781 * @return the found documents 782 * 783 * @throws CmsSearchException if something goes wrong 784 * 785 * @see #search(CmsObject, CmsSolrQuery, boolean) 786 */ 787 public CmsSolrResultList search( 788 CmsObject cms, 789 final CmsSolrQuery query, 790 boolean ignoreMaxRows, 791 ServletResponse response, 792 boolean ignoreSearchExclude, 793 CmsResourceFilter filter) 794 throws CmsSearchException { 795 796 return search(cms, query, ignoreMaxRows, response, ignoreSearchExclude, filter, getMaxProcessedResults()); 797 } 798 799 /** 800 * Performs the actual search.<p> 801 * 802 * To provide for correct permissions two queries are performed and the response is fused from that queries: 803 * <ol> 804 * <li>a query for permission checking, where fl, start and rows is adjusted. From this query result we take for the response: 805 * <ul> 806 * <li>facets</li> 807 * <li>spellcheck</li> 808 * <li>suggester</li> 809 * <li>morelikethis</li> 810 * <li>clusters</li> 811 * </ul> 812 * </li> 813 * <li>a query that collects only the resources determined by the first query and performs highlighting. From this query we take for the response: 814 * <li>result</li> 815 * <li>highlighting</li> 816 * </li> 817 *</ol> 818 * 819 * Currently not or only partly supported Solr features are: 820 * <ul> 821 * <li>groups</li> 822 * <li>collapse - representatives of the collapsed group might be filtered by the permission check</li> 823 * <li>expand is disabled</li> 824 * </ul> 825 * 826 * @param cms the current OpenCms context 827 * @param query the OpenCms Solr query 828 * @param ignoreMaxRows <code>true</code> to return all requested rows, <code>false</code> to use max rows 829 * @param response the servlet response to write the query result to, may also be <code>null</code> 830 * @param ignoreSearchExclude if set to false, only contents with search_exclude unset or "false" will be found - typical for the the non-gallery case 831 * @param filter the resource filter to use 832 * @param maxNumResults the maximal number of results to search for 833 * 834 * @return the found documents 835 * 836 * @throws CmsSearchException if something goes wrong 837 * 838 * @see #search(CmsObject, CmsSolrQuery, boolean) 839 */ 840 @SuppressWarnings("unchecked") 841 public CmsSolrResultList search( 842 CmsObject cms, 843 final CmsSolrQuery query, 844 boolean ignoreMaxRows, 845 ServletResponse response, 846 boolean ignoreSearchExclude, 847 CmsResourceFilter filter, 848 int maxNumResults) 849 throws CmsSearchException { 850 851 CmsSolrResultList result = null; 852 long startTime = System.currentTimeMillis(); 853 854 // TODO: 855 // - fall back to "last found results" if none are present at the "last page"? 856 // - deal with cursorMarks? 857 // - deal with groups? 858 // - deal with result clustering? 859 // - remove max score calculation? 860 861 if (LOG.isDebugEnabled()) { 862 LOG.debug(Messages.get().getBundle().key(Messages.LOG_SOLR_DEBUG_ORIGINAL_QUERY_2, query, getName())); 863 } 864 865 // change thread priority in order to reduce search impact on overall system performance 866 int previousPriority = Thread.currentThread().getPriority(); 867 if (getPriority() > 0) { 868 Thread.currentThread().setPriority(getPriority()); 869 } 870 871 // check if the user is allowed to access this index 872 checkOfflineAccess(cms); 873 874 if (!ignoreSearchExclude) { 875 if (LOG.isInfoEnabled()) { 876 LOG.info( 877 Messages.get().getBundle().key( 878 Messages.LOG_SOLR_INFO_ADDING_SEARCH_EXCLUDE_FILTER_FOR_QUERY_2, 879 query, 880 getName())); 881 } 882 String fqSearchExclude = CmsSearchField.FIELD_SEARCH_EXCLUDE + ":\"false\""; 883 query.removeFilterQuery(fqSearchExclude); 884 query.addFilterQuery(fqSearchExclude); 885 } 886 887 if (CmsProject.ONLINE_PROJECT_NAME.equals(getProject())) { 888 query.addFilterQuery( 889 "-" 890 + CmsPropertyDefinition.PROPERTY_SEARCH_EXCLUDE_ONLINE 891 + CmsSearchField.FIELD_DYNAMIC_PROPERTIES 892 + ":\"true\""); 893 } 894 895 // get start parameter from the request 896 int start = null == query.getStart() ? 0 : query.getStart().intValue(); 897 898 // correct negative start values to 0. 899 if (start < 0) { 900 query.setStart(Integer.valueOf(0)); 901 start = 0; 902 } 903 904 // Adjust the maximal number of results to process in case it is unlimited. 905 if (maxNumResults < 0) { 906 maxNumResults = Integer.MAX_VALUE; 907 if (LOG.isInfoEnabled()) { 908 LOG.info( 909 Messages.get().getBundle().key( 910 Messages.LOG_SOLR_INFO_LIMITING_MAX_PROCESSED_RESULTS_3, 911 query, 912 getName(), 913 Integer.valueOf(maxNumResults))); 914 } 915 } 916 917 // Correct the rows parameter 918 // Set the default rows, if rows are not set in the original query. 919 int rows = null == query.getRows() ? CmsSolrQuery.DEFAULT_ROWS.intValue() : query.getRows().intValue(); 920 921 // Restrict the rows, such that the maximal number of queryable results is not exceeded. 922 if ((((rows + start) > maxNumResults) || ((rows + start) < 0))) { 923 rows = maxNumResults - start; 924 } 925 // Restrict the rows to the maximally allowed number, if they should be restricted. 926 if (!ignoreMaxRows && (rows > ROWS_MAX)) { 927 if (LOG.isInfoEnabled()) { 928 LOG.info( 929 Messages.get().getBundle().key( 930 Messages.LOG_SOLR_INFO_LIMITING_MAX_ROWS_4, 931 new Object[] {query, getName(), Integer.valueOf(rows), Integer.valueOf(ROWS_MAX)})); 932 } 933 rows = ROWS_MAX; 934 } 935 // If start is higher than maxNumResults, the rows could be negative here - correct this. 936 if (rows < 0) { 937 if (LOG.isInfoEnabled()) { 938 LOG.info( 939 Messages.get().getBundle().key( 940 Messages.LOG_SOLR_INFO_CORRECTING_ROWS_4, 941 new Object[] {query, getName(), Integer.valueOf(rows), Integer.valueOf(0)})); 942 } 943 rows = 0; 944 } 945 // Set the corrected rows for the query. 946 query.setRows(Integer.valueOf(rows)); 947 948 // remove potentially set expand parameter 949 if (null != query.getParams(QUERY_PARAM_EXPAND)) { 950 LOG.info(Messages.get().getBundle().key(Messages.LOG_SOLR_INFO_REMOVING_EXPAND_2, query, getName())); 951 query.remove("expand"); 952 } 953 954 float maxScore = 0; 955 956 LocalSolrQueryRequest solrQueryRequest = null; 957 SolrCore core = null; 958 String[] sortParamValues = query.getParams(QUERY_SORT_NAME); 959 boolean sortByScoreDesc = (null == sortParamValues) 960 || (sortParamValues.length == 0) 961 || Objects.equal(sortParamValues[0], "score desc"); 962 963 try { 964 965 // initialize the search context 966 CmsObject searchCms = OpenCms.initCmsObject(cms); 967 968 //////////////////////////////////////////////////////////////////////////////////////////////////////////////// 969 //////////////////////// QUERY FOR PERMISSION CHECK, FACETS, SPELLCHECK, SUGGESTIONS /////////////////////////// 970 //////////////////////////////////////////////////////////////////////////////////////////////////////////////// 971 972 // Clone the query and keep the original one 973 CmsSolrQuery checkQuery = query.clone(); 974 // Initialize rows, offset, end and the current page. 975 int end = start + rows; 976 int itemsToCheck = 0 == end ? 0 : Math.max(10, end + (end / 5)); // request 20 percent more, but at least 10 results if permissions are filtered 977 // use a set to prevent double entries if multiple check queries are performed. 978 Set<String> resultSolrIds = new HashSet<>(rows); // rows are set before definitely. 979 980 // counter for the documents found and accessible 981 int cnt = 0; 982 long hitCount = 0; 983 long visibleHitCount = 0; 984 int processedResults = 0; 985 long solrPermissionTime = 0; 986 // disable highlighting - it's done in the next query. 987 checkQuery.setHighlight(false); 988 // adjust rows and start for the permission check. 989 checkQuery.setRows(Integer.valueOf(Math.min(maxNumResults - processedResults, itemsToCheck))); 990 checkQuery.setStart(Integer.valueOf(processedResults)); 991 // return only the fields required for the permission check and for scoring 992 checkQuery.setFields(CmsSearchField.FIELD_TYPE, CmsSearchField.FIELD_SOLR_ID, CmsSearchField.FIELD_PATH); 993 List<String> originalFields = Arrays.asList(query.getFields().split(",")); 994 if (originalFields.contains(CmsSearchField.FIELD_SCORE)) { 995 checkQuery.addField(CmsSearchField.FIELD_SCORE); 996 } 997 if (LOG.isDebugEnabled()) { 998 LOG.debug(Messages.get().getBundle().key(Messages.LOG_SOLR_DEBUG_CHECK_QUERY_2, checkQuery, getName())); 999 } 1000 // perform the permission check Solr query and remember the response and time Solr took. 1001 long solrCheckTime = System.currentTimeMillis(); 1002 QueryResponse checkQueryResponse = m_solr.query(getCoreName(), checkQuery); 1003 solrCheckTime = System.currentTimeMillis() - solrCheckTime; 1004 solrPermissionTime += solrCheckTime; 1005 1006 // initialize the counts 1007 hitCount = checkQueryResponse.getResults().getNumFound(); 1008 int maxToProcess = Long.valueOf(Math.min(hitCount, maxNumResults)).intValue(); 1009 visibleHitCount = hitCount; 1010 1011 // process found documents 1012 for (SolrDocument doc : checkQueryResponse.getResults()) { 1013 try { 1014 CmsSolrDocument searchDoc = new CmsSolrDocument(doc); 1015 if (needsPermissionCheck(searchDoc) && !hasPermissions(searchCms, searchDoc, filter)) { 1016 visibleHitCount--; 1017 } else { 1018 if (cnt >= start) { 1019 resultSolrIds.add(searchDoc.getFieldValueAsString(CmsSearchField.FIELD_SOLR_ID)); 1020 } 1021 if (sortByScoreDesc && (searchDoc.getScore() > maxScore)) { 1022 maxScore = searchDoc.getScore(); 1023 } 1024 if (++cnt >= end) { 1025 break; 1026 } 1027 } 1028 } catch (Exception e) { 1029 // should not happen, but if it does we want to go on with the next result nevertheless 1030 visibleHitCount--; 1031 LOG.warn(Messages.get().getBundle().key(Messages.LOG_SOLR_ERR_RESULT_ITERATION_FAILED_0), e); 1032 } 1033 } 1034 processedResults += checkQueryResponse.getResults().size(); 1035 1036 if ((resultSolrIds.size() < rows) && (processedResults < maxToProcess)) { 1037 CmsSolrQuery secondCheckQuery = checkQuery.clone(); 1038 // disable all features not necessary, since results are present from the first check query. 1039 secondCheckQuery.setFacet(false); 1040 secondCheckQuery.setMoreLikeThis(false); 1041 secondCheckQuery.set(QUERY_SPELLCHECK_NAME, false); 1042 do { 1043 // query directly more under certain conditions to reduce number of queries 1044 itemsToCheck = itemsToCheck < 3000 ? itemsToCheck * 4 : itemsToCheck; 1045 // adjust rows and start for the permission check. 1046 secondCheckQuery.setRows( 1047 Integer.valueOf( 1048 Long.valueOf(Math.min(maxToProcess - processedResults, itemsToCheck)).intValue())); 1049 secondCheckQuery.setStart(Integer.valueOf(processedResults)); 1050 1051 if (LOG.isDebugEnabled()) { 1052 LOG.debug( 1053 Messages.get().getBundle().key( 1054 Messages.LOG_SOLR_DEBUG_SECONDCHECK_QUERY_2, 1055 secondCheckQuery, 1056 getName())); 1057 } 1058 1059 long solrSecondCheckTime = System.currentTimeMillis(); 1060 QueryResponse secondCheckQueryResponse = m_solr.query(getCoreName(), secondCheckQuery); 1061 processedResults += secondCheckQueryResponse.getResults().size(); 1062 solrSecondCheckTime = System.currentTimeMillis() - solrSecondCheckTime; 1063 solrPermissionTime += solrCheckTime; 1064 1065 // process found documents 1066 for (SolrDocument doc : secondCheckQueryResponse.getResults()) { 1067 try { 1068 CmsSolrDocument searchDoc = new CmsSolrDocument(doc); 1069 String docSolrId = searchDoc.getFieldValueAsString(CmsSearchField.FIELD_SOLR_ID); 1070 if ((needsPermissionCheck(searchDoc) && !hasPermissions(searchCms, searchDoc, filter)) 1071 || resultSolrIds.contains(docSolrId)) { 1072 visibleHitCount--; 1073 } else { 1074 if (cnt >= start) { 1075 resultSolrIds.add(docSolrId); 1076 } 1077 if (sortByScoreDesc && (searchDoc.getScore() > maxScore)) { 1078 maxScore = searchDoc.getScore(); 1079 } 1080 if (++cnt >= end) { 1081 break; 1082 } 1083 } 1084 } catch (Exception e) { 1085 // should not happen, but if it does we want to go on with the next result nevertheless 1086 visibleHitCount--; 1087 LOG.warn( 1088 Messages.get().getBundle().key(Messages.LOG_SOLR_ERR_RESULT_ITERATION_FAILED_0), 1089 e); 1090 } 1091 } 1092 1093 } while ((resultSolrIds.size() < rows) && (processedResults < maxToProcess)); 1094 } 1095 1096 //////////////////////////////////////////////////////////////////////////////////////////////////////////////// 1097 //////////////////////// QUERY FOR RESULTS AND HIGHLIGHTING //////////////////////////////////////////////////// 1098 //////////////////////////////////////////////////////////////////////////////////////////////////////////////// 1099 1100 // the lists storing the found documents that will be returned 1101 List<CmsSearchResource> resourceDocumentList = new ArrayList<CmsSearchResource>(resultSolrIds.size()); 1102 SolrDocumentList solrDocumentList = new SolrDocumentList(); 1103 1104 long solrResultTime = 0; 1105 1106 // If we're using a post-processor, (re-)initialize it before using it 1107 if (m_postProcessor != null) { 1108 m_postProcessor.init(); 1109 } 1110 1111 // build the query for getting the results 1112 SolrQuery queryForResults = query.clone(); 1113 // we add an additional filter, such that we can only find the documents we want to retrieve, as we figured out in the check query. 1114 if (!resultSolrIds.isEmpty()) { 1115 String queryFilterString = resultSolrIds.stream().collect(Collectors.joining(",")); 1116 queryForResults.addFilterQuery( 1117 "{!terms f=" + CmsSearchField.FIELD_SOLR_ID + " separator=\",\"}" + queryFilterString); 1118 } 1119 queryForResults.setRows(Integer.valueOf(resultSolrIds.size())); 1120 queryForResults.setStart(Integer.valueOf(0)); 1121 1122 if (LOG.isDebugEnabled()) { 1123 LOG.debug( 1124 Messages.get().getBundle().key(Messages.LOG_SOLR_DEBUG_RESULT_QUERY_2, queryForResults, getName())); 1125 } 1126 // perform the result query. 1127 solrResultTime = System.currentTimeMillis(); 1128 QueryResponse resultQueryResponse = m_solr.query(getCoreName(), queryForResults); 1129 solrResultTime = System.currentTimeMillis() - solrResultTime; 1130 1131 // List containing solr ids of filtered contents for which highlighting has to be removed. 1132 // Since we checked permissions just a few milliseconds ago, this should typically stay empty. 1133 List<String> filteredResultIds = new ArrayList<>(5); 1134 1135 for (SolrDocument doc : resultQueryResponse.getResults()) { 1136 try { 1137 CmsSolrDocument searchDoc = new CmsSolrDocument(doc); 1138 if (needsPermissionCheck(searchDoc)) { 1139 CmsResource resource = filter == null 1140 ? getResource(searchCms, searchDoc) 1141 : getResource(searchCms, searchDoc, filter); 1142 if (null != resource) { 1143 if (m_postProcessor != null) { 1144 doc = m_postProcessor.process( 1145 searchCms, 1146 resource, 1147 (SolrInputDocument)searchDoc.getDocument()); 1148 } 1149 resourceDocumentList.add(new CmsSearchResource(resource, searchDoc)); 1150 solrDocumentList.add(doc); 1151 } else { 1152 filteredResultIds.add(searchDoc.getFieldValueAsString(CmsSearchField.FIELD_SOLR_ID)); 1153 } 1154 } else { // should not happen unless the index has changed since the first query. 1155 resourceDocumentList.add(new CmsSearchResource(PSEUDO_RES, searchDoc)); 1156 solrDocumentList.add(doc); 1157 visibleHitCount--; 1158 } 1159 } catch (Exception e) { 1160 // should not happen, but if it does we want to go on with the next result nevertheless 1161 visibleHitCount--; 1162 LOG.warn(Messages.get().getBundle().key(Messages.LOG_SOLR_ERR_RESULT_ITERATION_FAILED_0), e); 1163 } 1164 } 1165 1166 long processTime = System.currentTimeMillis() - startTime - solrPermissionTime - solrResultTime; 1167 1168 //////////////////////////////////////////////////////////////////////////////////////////////////////////// 1169 //////////////////////// CREATE THE FINAL RESPONSE ///////////////////////////////////////////////////////// 1170 //////////////////////////////////////////////////////////////////////////////////////////////////////////// 1171 1172 // we are manipulating the checkQueryResponse to set up the final response, we want to deliver. 1173 1174 // adjust start, max score and hit count displayed in the result list. 1175 solrDocumentList.setStart(start); 1176 Float finalMaxScore = sortByScoreDesc ? Float.valueOf(maxScore) : checkQueryResponse.getResults().getMaxScore(); 1177 solrDocumentList.setMaxScore(finalMaxScore); 1178 solrDocumentList.setNumFound(visibleHitCount); 1179 1180 // Exchange the search parameters in the response header by the ones from the (adjusted) original query. 1181 NamedList<Object> params = ((NamedList<Object>)(checkQueryResponse.getHeader().get(HEADER_PARAMS_NAME))); 1182 params.clear(); 1183 for (String paramName : query.getParameterNames()) { 1184 params.add(paramName, query.get(paramName)); 1185 } 1186 1187 // Fill in the documents to return. 1188 checkQueryResponse.getResponse().setVal( 1189 checkQueryResponse.getResponse().indexOf(QUERY_RESPONSE_NAME, 0), 1190 solrDocumentList); 1191 1192 // Fill in the time, the overall query took, including processing and permission check. 1193 checkQueryResponse.getResponseHeader().setVal( 1194 checkQueryResponse.getResponseHeader().indexOf(QUERY_TIME_NAME, 0), 1195 Integer.valueOf(Long.valueOf(System.currentTimeMillis() - startTime).intValue())); 1196 1197 // Fill in the highlighting information from the result query. 1198 if (query.getHighlight()) { 1199 NamedList<Object> highlighting = (NamedList<Object>)resultQueryResponse.getResponse().get( 1200 QUERY_HIGHLIGHTING_NAME); 1201 // filter out highlighting for documents where access is not permitted. 1202 for (String filteredId : filteredResultIds) { 1203 highlighting.remove(filteredId); 1204 } 1205 NamedList<Object> completeResponse = new SimpleOrderedMap<Object>(1); 1206 completeResponse.addAll(checkQueryResponse.getResponse()); 1207 completeResponse.add(QUERY_HIGHLIGHTING_NAME, highlighting); 1208 checkQueryResponse.setResponse(completeResponse); 1209 } 1210 1211 // build the result 1212 result = new CmsSolrResultList( 1213 query, 1214 checkQueryResponse, 1215 solrDocumentList, 1216 resourceDocumentList, 1217 start, 1218 Integer.valueOf(rows), 1219 Math.min(end, (start + solrDocumentList.size())), 1220 rows > 0 ? (start / rows) + 1 : 0, //page - but matches only in case of equally sized pages and is zero for rows=0 (because this was this way before!?!) 1221 visibleHitCount, 1222 finalMaxScore, 1223 startTime, 1224 System.currentTimeMillis()); 1225 if (LOG.isDebugEnabled()) { 1226 Object[] logParams = new Object[] { 1227 Long.valueOf(System.currentTimeMillis() - startTime), 1228 Long.valueOf(result.getNumFound()), 1229 Long.valueOf(solrPermissionTime + solrResultTime), 1230 Long.valueOf(processTime), 1231 Long.valueOf(result.getHighlightEndTime() != 0 ? result.getHighlightEndTime() - startTime : 0)}; 1232 LOG.debug( 1233 query.toString() 1234 + "\n" 1235 + Messages.get().getBundle().key(Messages.LOG_SOLR_SEARCH_EXECUTED_5, logParams)); 1236 } 1237 // write the response for the handler 1238 if (response != null) { 1239 // create and return the result 1240 core = m_solr instanceof EmbeddedSolrServer 1241 ? ((EmbeddedSolrServer)m_solr).getCoreContainer().getCore(getCoreName()) 1242 : null; 1243 1244 solrQueryRequest = new LocalSolrQueryRequest(core, query); 1245 SolrQueryResponse solrQueryResponse = new SolrQueryResponse(); 1246 solrQueryResponse.setAllValues(checkQueryResponse.getResponse()); 1247 writeResp(response, solrQueryRequest, solrQueryResponse); 1248 } 1249 } catch ( 1250 1251 Exception e) { 1252 throw new CmsSearchException( 1253 Messages.get().container( 1254 Messages.LOG_SOLR_ERR_SEARCH_EXECUTION_FAILD_1, 1255 CmsEncoder.decode(query.toString()), 1256 e), 1257 e); 1258 } finally { 1259 if (solrQueryRequest != null) { 1260 solrQueryRequest.close(); 1261 } 1262 if (null != core) { 1263 core.close(); 1264 } 1265 // re-set thread to previous priority 1266 Thread.currentThread().setPriority(previousPriority); 1267 } 1268 return result; 1269 } 1270 1271 /** 1272 * Default search method.<p> 1273 * 1274 * @param cms the current CMS object 1275 * @param query the query 1276 * 1277 * @return the results 1278 * 1279 * @throws CmsSearchException if something goes wrong 1280 * 1281 * @see #search(CmsObject, String) 1282 */ 1283 public CmsSolrResultList search(CmsObject cms, SolrQuery query) throws CmsSearchException { 1284 1285 return search(cms, CmsEncoder.decode(query.toString())); 1286 } 1287 1288 /** 1289 * Performs a search.<p> 1290 * 1291 * @param cms the cms object 1292 * @param solrQuery the Solr query 1293 * 1294 * @return a list of documents 1295 * 1296 * @throws CmsSearchException if something goes wrong 1297 * 1298 * @see #search(CmsObject, CmsSolrQuery, boolean) 1299 */ 1300 public CmsSolrResultList search(CmsObject cms, String solrQuery) throws CmsSearchException { 1301 1302 return search(cms, new CmsSolrQuery(null, CmsRequestUtil.createParameterMap(solrQuery)), false); 1303 } 1304 1305 /** 1306 * Writes the response into the writer.<p> 1307 * 1308 * NOTE: Currently not available for HTTP server.<p> 1309 * 1310 * @param response the servlet response 1311 * @param cms the CMS object to use for search 1312 * @param query the Solr query 1313 * @param ignoreMaxRows if to return unlimited results 1314 * 1315 * @throws Exception if there is no embedded server 1316 */ 1317 public void select(ServletResponse response, CmsObject cms, CmsSolrQuery query, boolean ignoreMaxRows) 1318 throws Exception { 1319 1320 throwExceptionIfSafetyRestrictionsAreViolated(cms, query, false); 1321 boolean isOnline = cms.getRequestContext().getCurrentProject().isOnlineProject(); 1322 CmsResourceFilter filter = isOnline ? null : CmsResourceFilter.IGNORE_EXPIRATION; 1323 1324 search(cms, query, ignoreMaxRows, response, false, filter); 1325 } 1326 1327 /** 1328 * Sets the logical key/name of this search index.<p> 1329 * 1330 * @param name the logical key/name of this search index 1331 * 1332 * @throws CmsIllegalArgumentException if the given name is null, empty or already taken by another search index 1333 */ 1334 @Override 1335 public void setName(String name) throws CmsIllegalArgumentException { 1336 1337 super.setName(name); 1338 updateCoreName(); 1339 } 1340 1341 /** 1342 * Sets the search post processor.<p> 1343 * 1344 * @param postProcessor the search post processor to set 1345 */ 1346 public void setPostProcessor(I_CmsSolrPostSearchProcessor postProcessor) { 1347 1348 m_postProcessor = postProcessor; 1349 } 1350 1351 /** 1352 * Sets the Solr server used by this index.<p> 1353 * 1354 * @param client the server to set 1355 */ 1356 public void setSolrServer(SolrClient client) { 1357 1358 m_solr = client; 1359 } 1360 1361 /** 1362 * Executes a spell checking Solr query and returns the Solr query response.<p> 1363 * 1364 * @param res the servlet response 1365 * @param cms the CMS object 1366 * @param q the query 1367 * 1368 * @throws CmsSearchException if something goes wrong 1369 */ 1370 public void spellCheck(ServletResponse res, CmsObject cms, CmsSolrQuery q) throws CmsSearchException { 1371 1372 throwExceptionIfSafetyRestrictionsAreViolated(cms, q, true); 1373 SolrCore core = null; 1374 LocalSolrQueryRequest solrQueryRequest = null; 1375 try { 1376 q.setRequestHandler("/spell"); 1377 q.setRows(Integer.valueOf(0)); 1378 1379 QueryResponse queryResponse = m_solr.query(getCoreName(), q); 1380 1381 List<CmsSearchResource> resourceDocumentList = new ArrayList<CmsSearchResource>(); 1382 SolrDocumentList solrDocumentList = new SolrDocumentList(); 1383 if (m_postProcessor != null) { 1384 for (int i = 0; (i < queryResponse.getResults().size()); i++) { 1385 try { 1386 SolrDocument doc = queryResponse.getResults().get(i); 1387 CmsSolrDocument searchDoc = new CmsSolrDocument(doc); 1388 if (needsPermissionCheck(searchDoc)) { 1389 // only if the document is an OpenCms internal resource perform the permission check 1390 CmsResource resource = getResource(cms, searchDoc); 1391 if (resource != null) { 1392 // permission check performed successfully: the user has read permissions! 1393 if (m_postProcessor != null) { 1394 doc = m_postProcessor.process( 1395 cms, 1396 resource, 1397 (SolrInputDocument)searchDoc.getDocument()); 1398 } 1399 resourceDocumentList.add(new CmsSearchResource(resource, searchDoc)); 1400 solrDocumentList.add(doc); 1401 } 1402 } 1403 } catch (Exception e) { 1404 // should not happen, but if it does we want to go on with the next result nevertheless 1405 LOG.warn(Messages.get().getBundle().key(Messages.LOG_SOLR_ERR_RESULT_ITERATION_FAILED_0), e); 1406 } 1407 } 1408 queryResponse.getResponse().setVal( 1409 queryResponse.getResponse().indexOf(QUERY_RESPONSE_NAME, 0), 1410 solrDocumentList); 1411 } 1412 1413 // create and return the result 1414 core = m_solr instanceof EmbeddedSolrServer 1415 ? ((EmbeddedSolrServer)m_solr).getCoreContainer().getCore(getCoreName()) 1416 : null; 1417 1418 SolrQueryResponse solrQueryResponse = new SolrQueryResponse(); 1419 solrQueryResponse.setAllValues(queryResponse.getResponse()); 1420 1421 // create and initialize the solr request 1422 solrQueryRequest = new LocalSolrQueryRequest(core, solrQueryResponse.getResponseHeader()); 1423 // set the OpenCms Solr query as parameters to the request 1424 solrQueryRequest.setParams(q); 1425 1426 writeResp(res, solrQueryRequest, solrQueryResponse); 1427 1428 } catch (Exception e) { 1429 throw new CmsSearchException( 1430 Messages.get().container(Messages.LOG_SOLR_ERR_SEARCH_EXECUTION_FAILD_1, q), 1431 e); 1432 } finally { 1433 if (solrQueryRequest != null) { 1434 solrQueryRequest.close(); 1435 } 1436 if (core != null) { 1437 core.close(); 1438 } 1439 } 1440 } 1441 1442 /** 1443 * @see org.opencms.search.CmsSearchIndex#createIndexBackup() 1444 */ 1445 @Override 1446 protected String createIndexBackup() { 1447 1448 if (!isBackupReindexing()) { 1449 // if no backup is generated we don't need to do anything 1450 return null; 1451 } 1452 if (m_solr instanceof EmbeddedSolrServer) { 1453 EmbeddedSolrServer ser = (EmbeddedSolrServer)m_solr; 1454 CoreContainer con = ser.getCoreContainer(); 1455 SolrCore core = con.getCore(getCoreName()); 1456 if (core != null) { 1457 try { 1458 SolrRequestHandler h = core.getRequestHandler("/replication"); 1459 if (h instanceof ReplicationHandler) { 1460 h.handleRequest( 1461 new LocalSolrQueryRequest(core, CmsRequestUtil.createParameterMap("?command=backup")), 1462 new SolrQueryResponse()); 1463 } 1464 } finally { 1465 core.close(); 1466 } 1467 } 1468 } 1469 return null; 1470 } 1471 1472 /** 1473 * Check, if the current user has permissions on the document's resource. 1474 * @param cms the context 1475 * @param doc the solr document (from the search result) 1476 * @param filter the resource filter to use for checking permissions 1477 * @return <code>true</code> iff the resource mirrored by the search result can be read by the current user. 1478 */ 1479 protected boolean hasPermissions(CmsObject cms, CmsSolrDocument doc, CmsResourceFilter filter) { 1480 1481 return null != (filter == null ? getResource(cms, doc) : getResource(cms, doc, filter)); 1482 } 1483 1484 /** 1485 * @see org.opencms.search.CmsSearchIndex#indexSearcherClose() 1486 */ 1487 @SuppressWarnings("sync-override") 1488 @Override 1489 protected void indexSearcherClose() { 1490 1491 // nothing to do here 1492 } 1493 1494 /** 1495 * @see org.opencms.search.CmsSearchIndex#indexSearcherOpen(java.lang.String) 1496 */ 1497 @SuppressWarnings("sync-override") 1498 @Override 1499 protected void indexSearcherOpen(final String path) { 1500 1501 // nothing to do here 1502 } 1503 1504 /** 1505 * @see org.opencms.search.CmsSearchIndex#indexSearcherUpdate() 1506 */ 1507 @SuppressWarnings("sync-override") 1508 @Override 1509 protected void indexSearcherUpdate() { 1510 1511 // nothing to do here 1512 } 1513 1514 /** 1515 * Checks if the current user is allowed to access non-online indexes.<p> 1516 * 1517 * To access non-online indexes the current user must be a workplace user at least.<p> 1518 * 1519 * @param cms the CMS object initialized with the current request context / user 1520 * 1521 * @throws CmsSearchException thrown if the access is not permitted 1522 */ 1523 private void checkOfflineAccess(CmsObject cms) throws CmsSearchException { 1524 1525 // If an offline index is being selected, check permissions 1526 if (!CmsProject.ONLINE_PROJECT_NAME.equals(getProject())) { 1527 // only if the user has the role Workplace user, he is allowed to access the Offline index 1528 try { 1529 OpenCms.getRoleManager().checkRole(cms, CmsRole.ELEMENT_AUTHOR); 1530 } catch (CmsRoleViolationException e) { 1531 throw new CmsSearchException( 1532 Messages.get().container( 1533 Messages.LOG_SOLR_ERR_SEARCH_PERMISSION_VIOLATION_2, 1534 getName(), 1535 cms.getRequestContext().getCurrentUser()), 1536 e); 1537 } 1538 } 1539 } 1540 1541 /** 1542 * Generates a valid core name from the provided name (the index name). 1543 * @param name the index name. 1544 * @return the core name 1545 */ 1546 private String generateCoreName(final String name) { 1547 1548 if (name != null) { 1549 return name.replace(" ", "-"); 1550 } 1551 return null; 1552 } 1553 1554 /** 1555 * Checks if the query should be executed using the debug mode where the security restrictions do not apply. 1556 * @param cms the current context. 1557 * @param query the query to execute. 1558 * @return a flag, indicating, if the query should be performed in debug mode. 1559 */ 1560 private boolean isDebug(CmsObject cms, CmsSolrQuery query) { 1561 1562 String[] debugSecretValues = query.remove(REQUEST_PARAM_DEBUG_SECRET); 1563 String debugSecret = (debugSecretValues == null) || (debugSecretValues.length < 1) 1564 ? null 1565 : debugSecretValues[0]; 1566 if ((null != debugSecret) && !debugSecret.trim().isEmpty() && (null != m_handlerDebugSecretFile)) { 1567 try { 1568 CmsFile secretFile = cms.readFile(m_handlerDebugSecretFile); 1569 String secret = new String(secretFile.getContents(), CmsFileUtil.getEncoding(cms, secretFile)); 1570 return secret.trim().equals(debugSecret.trim()); 1571 } catch (Exception e) { 1572 LOG.info( 1573 "Failed to read secret file for index \"" 1574 + getName() 1575 + "\" at path \"" 1576 + m_handlerDebugSecretFile 1577 + "\"."); 1578 } 1579 } 1580 return false; 1581 } 1582 1583 /** 1584 * Throws an exception if the request can for security reasons not be performed. 1585 * Security restrictions can be set via parameters of the index. 1586 * 1587 * @param cms the current context. 1588 * @param query the query. 1589 * @param isSpell flag, indicating if the spellcheck handler is requested. 1590 * @throws CmsSearchException thrown if the query cannot be executed due to security reasons. 1591 */ 1592 private void throwExceptionIfSafetyRestrictionsAreViolated(CmsObject cms, CmsSolrQuery query, boolean isSpell) 1593 throws CmsSearchException { 1594 1595 if (!isDebug(cms, query)) { 1596 if (isSpell) { 1597 if (m_handlerSpellDisabled) { 1598 throw new CmsSearchException(Messages.get().container(Messages.GUI_HANDLER_REQUEST_NOT_ALLOWED_0)); 1599 } 1600 } else { 1601 if (m_handlerSelectDisabled) { 1602 throw new CmsSearchException(Messages.get().container(Messages.GUI_HANDLER_REQUEST_NOT_ALLOWED_0)); 1603 } 1604 int start = null != query.getStart() ? query.getStart().intValue() : 0; 1605 int rows = null != query.getRows() ? query.getRows().intValue() : CmsSolrQuery.DEFAULT_ROWS.intValue(); 1606 if ((m_handlerMaxAllowedResultsAtAll >= 0) && ((rows + start) > m_handlerMaxAllowedResultsAtAll)) { 1607 throw new CmsSearchException( 1608 Messages.get().container( 1609 Messages.GUI_HANDLER_TOO_MANY_RESULTS_REQUESTED_AT_ALL_2, 1610 Integer.valueOf(m_handlerMaxAllowedResultsAtAll), 1611 Integer.valueOf(rows + start))); 1612 } 1613 if ((m_handlerMaxAllowedResultsPerPage >= 0) && (rows > m_handlerMaxAllowedResultsPerPage)) { 1614 throw new CmsSearchException( 1615 Messages.get().container( 1616 Messages.GUI_HANDLER_TOO_MANY_RESULTS_REQUESTED_PER_PAGE_2, 1617 Integer.valueOf(m_handlerMaxAllowedResultsPerPage), 1618 Integer.valueOf(rows))); 1619 } 1620 if ((null != m_handlerAllowedFields) && (Stream.of(m_handlerAllowedFields).anyMatch(x -> true))) { 1621 if (query.getFields().equals(CmsSolrQuery.ALL_RETURN_FIELDS)) { 1622 query.setFields(m_handlerAllowedFields); 1623 } else { 1624 for (String requestedField : query.getFields().split(",")) { 1625 if (Stream.of(m_handlerAllowedFields).noneMatch( 1626 allowedField -> allowedField.equals(requestedField))) { 1627 throw new CmsSearchException( 1628 Messages.get().container( 1629 Messages.GUI_HANDLER_REQUESTED_FIELD_NOT_ALLOWED_2, 1630 requestedField, 1631 Stream.of(m_handlerAllowedFields).reduce("", (a, b) -> a + "," + b))); 1632 } 1633 } 1634 } 1635 } 1636 } 1637 } 1638 } 1639 1640 /** 1641 * Updates the core name to be in sync with the index name. 1642 */ 1643 private void updateCoreName() { 1644 1645 m_coreName = generateCoreName(getName()); 1646 1647 } 1648 1649 /** 1650 * Writes the Solr response.<p> 1651 * 1652 * @param response the servlet response 1653 * @param queryRequest the Solr request 1654 * @param queryResponse the Solr response to write 1655 * 1656 * @throws IOException if sth. goes wrong 1657 * @throws UnsupportedEncodingException if sth. goes wrong 1658 */ 1659 private void writeResp(ServletResponse response, SolrQueryRequest queryRequest, SolrQueryResponse queryResponse) 1660 throws IOException, UnsupportedEncodingException { 1661 1662 if (m_solr instanceof EmbeddedSolrServer) { 1663 SolrCore core = ((EmbeddedSolrServer)m_solr).getCoreContainer().getCore(getCoreName()); 1664 Writer out = null; 1665 try { 1666 QueryResponseWriter responseWriter = core.getQueryResponseWriter(queryRequest); 1667 1668 final String ct = responseWriter.getContentType(queryRequest, queryResponse); 1669 if (null != ct) { 1670 response.setContentType(ct); 1671 } 1672 1673 if (responseWriter instanceof BinaryQueryResponseWriter) { 1674 BinaryQueryResponseWriter binWriter = (BinaryQueryResponseWriter)responseWriter; 1675 binWriter.write(response.getOutputStream(), queryRequest, queryResponse); 1676 } else { 1677 String charset = ContentStreamBase.getCharsetFromContentType(ct); 1678 out = ((charset == null) || charset.equalsIgnoreCase(UTF8.toString())) 1679 ? new OutputStreamWriter(response.getOutputStream(), UTF8) 1680 : new OutputStreamWriter(response.getOutputStream(), charset); 1681 out = new FastWriter(out); 1682 responseWriter.write(out, queryRequest, queryResponse); 1683 out.flush(); 1684 } 1685 } finally { 1686 core.close(); 1687 if (out != null) { 1688 out.close(); 1689 } 1690 } 1691 } else { 1692 throw new UnsupportedOperationException(); 1693 } 1694 } 1695}