001/* 002 * File : $Source$ 003 * Date : $Date$ 004 * Version: $Revision$ 005 * 006 * This library is part of OpenCms - 007 * the Open Source Content Management System 008 * 009 * Copyright (C) 2002 - 2009 Alkacon Software (http://www.alkacon.com) 010 * 011 * This library is free software; you can redistribute it and/or 012 * modify it under the terms of the GNU Lesser General Public 013 * License as published by the Free Software Foundation; either 014 * version 2.1 of the License, or (at your option) any later version. 015 * 016 * This library is distributed in the hope that it will be useful, 017 * but WITHOUT ANY WARRANTY; without even the implied warranty of 018 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 019 * Lesser General Public License for more details. 020 * 021 * For further information about Alkacon Software, please see the 022 * company website: http://www.alkacon.com 023 * 024 * For further information about OpenCms, please see the 025 * project website: http://www.opencms.org 026 * 027 * You should have received a copy of the GNU Lesser General Public 028 * License along with this library; if not, write to the Free Software 029 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 030 */ 031 032package org.opencms.search.solr; 033 034import org.opencms.configuration.I_CmsXmlConfiguration; 035import org.opencms.file.CmsFile; 036import org.opencms.file.CmsObject; 037import org.opencms.file.CmsProperty; 038import org.opencms.file.CmsPropertyDefinition; 039import org.opencms.file.CmsResource; 040import org.opencms.file.types.CmsResourceTypeJsp; 041import org.opencms.file.types.CmsResourceTypeXmlContainerPage; 042import org.opencms.file.types.CmsResourceTypeXmlContent; 043import org.opencms.file.types.CmsResourceTypeXmlPage; 044import org.opencms.loader.CmsResourceManager; 045import org.opencms.main.CmsException; 046import org.opencms.main.CmsLog; 047import org.opencms.main.OpenCms; 048import org.opencms.search.CmsSearchIndexSource; 049import org.opencms.search.CmsSearchUtil; 050import org.opencms.search.I_CmsSearchDocument; 051import org.opencms.search.documents.CmsDocumentDependency; 052import org.opencms.search.extractors.I_CmsExtractionResult; 053import org.opencms.search.fields.CmsLuceneField; 054import org.opencms.search.fields.CmsSearchField; 055import org.opencms.search.fields.CmsSearchFieldConfiguration; 056import org.opencms.search.fields.CmsSearchFieldMapping; 057import org.opencms.search.fields.CmsSearchFieldMappingType; 058import org.opencms.search.fields.I_CmsSearchFieldMapping; 059import org.opencms.util.CmsStringUtil; 060import org.opencms.xml.CmsXmlContentDefinition; 061import org.opencms.xml.containerpage.CmsContainerElementBean; 062import org.opencms.xml.containerpage.CmsContainerPageBean; 063import org.opencms.xml.containerpage.CmsXmlContainerPage; 064import org.opencms.xml.containerpage.CmsXmlContainerPageFactory; 065import org.opencms.xml.content.I_CmsXmlContentHandler; 066 067import java.util.ArrayList; 068import java.util.Arrays; 069import java.util.Collection; 070import java.util.Collections; 071import java.util.Date; 072import java.util.HashMap; 073import java.util.List; 074import java.util.Locale; 075import java.util.Map; 076import java.util.Set; 077 078import org.apache.commons.logging.Log; 079import org.apache.solr.common.SolrInputDocument; 080 081/** 082 * The search field implementation for Solr.<p> 083 * 084 * @since 8.5.0 085 */ 086public class CmsSolrFieldConfiguration extends CmsSearchFieldConfiguration { 087 088 /** The log object for this class. */ 089 private static final Log LOG = CmsLog.getLog(CmsSolrFieldConfiguration.class); 090 091 /** The content locale for the indexed document is stored in order to save performance. */ 092 private Collection<Locale> m_contentLocales; 093 094 /** A list of Solr fields. */ 095 private Map<String, CmsSolrField> m_solrFields = new HashMap<String, CmsSolrField>(); 096 097 /** 098 * Default constructor.<p> 099 */ 100 public CmsSolrFieldConfiguration() { 101 102 super(); 103 } 104 105 /** 106 * Adds the additional fields to the configuration, if they are not null.<p> 107 * 108 * @param additionalFields the additional fields to add 109 */ 110 public void addAdditionalFields(List<CmsSolrField> additionalFields) { 111 112 if (additionalFields != null) { 113 for (CmsSolrField solrField : additionalFields) { 114 m_solrFields.put(solrField.getName(), solrField); 115 } 116 } 117 } 118 119 /** 120 * Returns all configured Solr fields.<p> 121 * 122 * @return all configured Solr fields 123 */ 124 public Map<String, CmsSolrField> getSolrFields() { 125 126 return Collections.unmodifiableMap(m_solrFields); 127 } 128 129 /** 130 * @see org.opencms.search.fields.CmsSearchFieldConfiguration#init() 131 */ 132 @Override 133 public void init() { 134 135 super.init(); 136 addAdditionalFields(); 137 } 138 139 /** 140 * @see org.opencms.search.fields.CmsSearchFieldConfiguration#appendAdditionalValuesToDcoument(org.opencms.search.I_CmsSearchDocument, org.opencms.file.CmsObject, org.opencms.file.CmsResource, org.opencms.search.extractors.I_CmsExtractionResult, java.util.List, java.util.List) 141 */ 142 @Override 143 protected I_CmsSearchDocument appendAdditionalValuesToDcoument( 144 I_CmsSearchDocument document, 145 CmsObject cms, 146 CmsResource resource, 147 I_CmsExtractionResult extractionResult, 148 List<CmsProperty> properties, 149 List<CmsProperty> propertiesSearched) { 150 151 String mimeType = OpenCms.getResourceManager().getMimeType(resource.getName(), null); 152 if (mimeType != null) { 153 document.addSearchField(m_solrFields.get(CmsSearchField.FIELD_MIMETYPE), mimeType); 154 } 155 156 document.addSearchField(m_solrFields.get(CmsSearchField.FIELD_FILENAME), resource.getName()); 157 158 document.addSearchField(m_solrFields.get(CmsSearchField.FIELD_VERSION), "" + resource.getVersion()); 159 160 try { 161 if (CmsResourceTypeXmlContent.isXmlContent(resource)) { 162 I_CmsXmlContentHandler handler = CmsXmlContentDefinition.getContentHandlerForResource(cms, resource); 163 if ((handler != null) && handler.isContainerPageOnly()) { 164 if (document.getDocument() instanceof SolrInputDocument) { 165 SolrInputDocument doc = (SolrInputDocument)document.getDocument(); 166 doc.removeField(CmsSearchField.FIELD_SEARCH_EXCLUDE); 167 } else { 168 //TODO: Warning - but should not happen. 169 } 170 document.addSearchField(m_solrFields.get(CmsSearchField.FIELD_SEARCH_EXCLUDE), "true"); 171 } 172 } 173 } catch (CmsException e) { 174 LOG.error(e.getMessage(), e); 175 } 176 177 List<String> searchExcludeOptions = document.getMultivaluedFieldAsStringList( 178 CmsSearchField.FIELD_SEARCH_EXCLUDE); 179 if ((searchExcludeOptions == null) || searchExcludeOptions.isEmpty()) { 180 document.addSearchField(m_solrFields.get(CmsSearchField.FIELD_SEARCH_EXCLUDE), "false"); 181 } 182 if (resource.getRootPath().startsWith("/system") 183 || (CmsResourceTypeJsp.getJSPTypeId() == resource.getTypeId())) { 184 document.addSearchField(m_solrFields.get(CmsSearchField.FIELD_SEARCH_CHANNEL), "gallery"); 185 } else { 186 document.addSearchField(m_solrFields.get(CmsSearchField.FIELD_SEARCH_CHANNEL), "content"); 187 } 188 189 document = appendFieldsForListSortOptions(document); 190 191 if (resource.getRootPath().startsWith(OpenCms.getSiteManager().getSharedFolder()) 192 || (null != OpenCms.getSiteManager().getSiteRoot(resource.getRootPath()))) { 193 appendSpellFields(document); 194 } 195 196 return document; 197 } 198 199 /** 200 * @see org.opencms.search.fields.CmsSearchFieldConfiguration#appendDates(org.opencms.search.I_CmsSearchDocument, org.opencms.file.CmsObject, org.opencms.file.CmsResource, org.opencms.search.extractors.I_CmsExtractionResult, java.util.List, java.util.List) 201 */ 202 @Override 203 protected I_CmsSearchDocument appendDates( 204 I_CmsSearchDocument document, 205 CmsObject cms, 206 CmsResource resource, 207 I_CmsExtractionResult extractionResult, 208 List<CmsProperty> properties, 209 List<CmsProperty> propertiesSearched) { 210 211 document.addDateField(CmsSearchField.FIELD_DATE_CREATED, resource.getDateCreated(), false); 212 document.addDateField(CmsSearchField.FIELD_DATE_LASTMODIFIED, resource.getDateLastModified(), false); 213 document.addDateField(CmsSearchField.FIELD_DATE_CONTENT, resource.getDateContent(), false); 214 document.addDateField(CmsSearchField.FIELD_DATE_RELEASED, resource.getDateReleased(), false); 215 document.addDateField(CmsSearchField.FIELD_DATE_EXPIRED, resource.getDateExpired(), false); 216 217 return document; 218 } 219 220 /** 221 * @see org.opencms.search.fields.CmsSearchFieldConfiguration#appendFieldMapping(org.opencms.search.I_CmsSearchDocument, org.opencms.search.fields.CmsSearchField, org.opencms.file.CmsObject, org.opencms.file.CmsResource, org.opencms.search.extractors.I_CmsExtractionResult, java.util.List, java.util.List) 222 */ 223 @Override 224 protected I_CmsSearchDocument appendFieldMapping( 225 I_CmsSearchDocument document, 226 CmsSearchField sfield, 227 CmsObject cms, 228 CmsResource resource, 229 I_CmsExtractionResult extractionResult, 230 List<CmsProperty> properties, 231 List<CmsProperty> propertiesSearched) { 232 233 CmsSolrField field = (CmsSolrField)sfield; 234 try { 235 StringBuffer text = new StringBuffer(); 236 for (I_CmsSearchFieldMapping mapping : field.getMappings()) { 237 // loop over the mappings of the given field 238 if (extractionResult != null) { 239 String mapResult = null; 240 if ((field.getLocale() != null) && mapping.getType().equals(CmsSearchFieldMappingType.CONTENT)) { 241 // this is a localized content field, try to retrieve the localized content extraction 242 mapResult = extractionResult.getContent(field.getLocale()); 243 if (mapResult == null) { 244 // no localized content extracted 245 if (!(CmsResourceTypeXmlContent.isXmlContent(resource) 246 || CmsResourceTypeXmlPage.isXmlPage(resource))) { 247 // the resource is no XML content nor an XML page 248 if ((m_contentLocales != null) && m_contentLocales.contains(field.getLocale())) { 249 // the resource to get the extracted content for has the locale of this field, 250 // so store the extraction content into this field 251 mapResult = extractionResult.getContent(); 252 } 253 } 254 } 255 } else { 256 // this is not a localized content field, just perform the regular mapping 257 mapResult = mapping.getStringValue( 258 cms, 259 resource, 260 extractionResult, 261 properties, 262 propertiesSearched); 263 } 264 if (text.length() > 0) { 265 text.append('\n'); 266 } 267 if (mapResult != null) { 268 text.append(mapResult); 269 } else if (mapping.getDefaultValue() != null) { 270 // no mapping result found, but a default is configured 271 text.append(mapping.getDefaultValue()); 272 } 273 } else if (mapping.getStringValue( 274 cms, 275 resource, 276 extractionResult, 277 properties, 278 propertiesSearched) != null) { 279 String value = mapping.getStringValue( 280 cms, 281 resource, 282 extractionResult, 283 properties, 284 propertiesSearched); 285 if (value != null) { 286 document.addSearchField(field, value); 287 } 288 } 289 } 290 if ((text.length() <= 0) && (field.getDefaultValue() != null)) { 291 text.append(field.getDefaultValue()); 292 } 293 if (text.length() > 0) { 294 document.addSearchField(field, text.toString()); 295 } 296 } catch (Exception e) { 297 // nothing to do just log 298 LOG.error(e.getLocalizedMessage(), e); 299 } 300 return document; 301 } 302 303 /** 304 * @see org.opencms.search.fields.CmsSearchFieldConfiguration#appendFieldMappings(org.opencms.search.I_CmsSearchDocument, org.opencms.file.CmsObject, org.opencms.file.CmsResource, org.opencms.search.extractors.I_CmsExtractionResult, java.util.List, java.util.List) 305 */ 306 @Override 307 protected I_CmsSearchDocument appendFieldMappings( 308 I_CmsSearchDocument document, 309 CmsObject cms, 310 CmsResource resource, 311 I_CmsExtractionResult extractionResult, 312 List<CmsProperty> properties, 313 List<CmsProperty> propertiesSearched) { 314 315 List<String> systemFields = new ArrayList<String>(); 316 // append field mappings directly stored in the extraction result 317 if (null != extractionResult) { 318 Map<String, String> fieldMappings = extractionResult.getFieldMappings(); 319 for (String fieldName : fieldMappings.keySet()) { 320 String value = fieldMappings.get(fieldName); 321 CmsSolrField f = new CmsSolrField(fieldName, null, null, null); 322 document.addSearchField(f, value); 323 systemFields.add(fieldName); 324 } 325 } 326 327 Set<CmsSearchField> mappedFields = getXSDMappings(cms, resource); 328 if (mappedFields != null) { 329 for (CmsSearchField field : mappedFields) { 330 if (!systemFields.contains(field.getName())) { 331 document = appendFieldMapping( 332 document, 333 field, 334 cms, 335 resource, 336 extractionResult, 337 properties, 338 propertiesSearched); 339 } else { 340 LOG.error( 341 Messages.get().getBundle().key( 342 Messages.LOG_SOLR_ERR_MAPPING_TO_INTERNALLY_USED_FIELD_2, 343 resource.getRootPath(), 344 field.getName())); 345 } 346 } 347 } 348 349 // add field mappings from elements of a container page 350 if (CmsResourceTypeXmlContainerPage.isContainerPage(resource)) { 351 document = appendFieldMappingsFromElementsOnThePage(document, cms, resource, systemFields); 352 353 } 354 355 for (CmsSolrField field : m_solrFields.values()) { 356 document = appendFieldMapping( 357 document, 358 field, 359 cms, 360 resource, 361 extractionResult, 362 properties, 363 propertiesSearched); 364 } 365 366 return document; 367 } 368 369 /** 370 * Adds search fields from elements on a container page to a container page's document. 371 * @param document The document for the container page 372 * @param cms The current CmsObject 373 * @param resource The resource of the container page 374 * @param systemFields The list of field names for fields where mappings to should be discarded, since these fields are used system internally. 375 * @return the manipulated document 376 */ 377 protected I_CmsSearchDocument appendFieldMappingsFromElementsOnThePage( 378 I_CmsSearchDocument document, 379 CmsObject cms, 380 CmsResource resource, 381 List<String> systemFields) { 382 383 try { 384 CmsFile file = cms.readFile(resource); 385 CmsXmlContainerPage containerPage = CmsXmlContainerPageFactory.unmarshal(cms, file); 386 CmsContainerPageBean containerBean = containerPage.getContainerPage(cms); 387 if (containerBean != null) { 388 for (CmsContainerElementBean element : containerBean.getElements()) { 389 element.initResource(cms); 390 CmsResource elemResource = element.getResource(); 391 Set<CmsSearchField> mappedFields = getXSDMappingsForPage(cms, elemResource); 392 if (mappedFields != null) { 393 394 for (CmsSearchField field : mappedFields) { 395 if (!systemFields.contains(field.getName())) { 396 try { 397 I_CmsExtractionResult extractionResult = CmsSolrDocumentXmlContent.extractXmlContent( 398 cms, 399 elemResource, 400 getIndex()); 401 document = appendFieldMapping( 402 document, 403 field, 404 cms, 405 elemResource, 406 extractionResult, 407 cms.readPropertyObjects(resource, false), 408 cms.readPropertyObjects(resource, true)); 409 } catch (Exception e) { 410 LOG.error( 411 Messages.get().getBundle().key( 412 Messages.LOG_SOLR_ERR_MAPPING_UNREADABLE_CONTENT_3, 413 elemResource.getRootPath(), 414 field.getName(), 415 resource.getRootPath()), 416 e); 417 } 418 } else { 419 LOG.error( 420 Messages.get().getBundle().key( 421 Messages.LOG_SOLR_ERR_MAPPING_TO_INTERNALLY_USED_FIELD_3, 422 elemResource.getRootPath(), 423 field.getName(), 424 resource.getRootPath())); 425 } 426 } 427 } 428 } 429 } 430 } catch (CmsException e) { 431 // Should be thrown if element on the page does not exist anymore - this is possible, but not necessarily an error. 432 // Hence, just notice it in the debug log. 433 if (LOG.isDebugEnabled()) { 434 LOG.debug(e.getLocalizedMessage(), e); 435 } 436 } 437 return document; 438 } 439 440 /** 441 * @see org.opencms.search.fields.CmsSearchFieldConfiguration#appendLocales(org.opencms.search.I_CmsSearchDocument, org.opencms.file.CmsObject, org.opencms.file.CmsResource, org.opencms.search.extractors.I_CmsExtractionResult, java.util.List, java.util.List) 442 */ 443 @Override 444 protected I_CmsSearchDocument appendLocales( 445 I_CmsSearchDocument document, 446 CmsObject cms, 447 CmsResource resource, 448 I_CmsExtractionResult extraction, 449 List<CmsProperty> properties, 450 List<CmsProperty> propertiesSearched) { 451 452 // append the resource locales 453 Collection<Locale> resourceLocales = new ArrayList<Locale>(); 454 if ((extraction != null) && (!extraction.getLocales().isEmpty())) { 455 456 CmsResourceManager resMan = OpenCms.getResourceManager(); 457 resourceLocales = extraction.getLocales(); 458 boolean isGroup = false; 459 for (String groupType : Arrays.asList( 460 CmsResourceTypeXmlContainerPage.GROUP_CONTAINER_TYPE_NAME, 461 CmsResourceTypeXmlContainerPage.INHERIT_CONTAINER_TYPE_NAME)) { 462 if (resMan.matchResourceType(groupType, resource.getTypeId())) { 463 isGroup = true; 464 break; 465 } 466 } 467 if (isGroup) { 468 // groups are locale independent, so they have to have *all* locales so they are found for each one 469 m_contentLocales = OpenCms.getLocaleManager().getAvailableLocales(); 470 } else { 471 m_contentLocales = resourceLocales; 472 } 473 } else { 474 // For all other resources add all default locales 475 resourceLocales = OpenCms.getLocaleManager().getDefaultLocales(cms, resource); 476 477 /* 478 * A problem is likely to arise when dealing with multilingual fields: 479 * Only values extracted from XML resources are written into the Solr locale-aware fields (e.g. 480 * "title_<locale>_s"), therefore sorting by them will not work as non-XML (unilingual) resources extract 481 * the information by the resource property facility and will not write to an Solr locale-aware field. 482 * 483 * The following code is used to fix this behavior, at least for "Title". 484 */ 485 486 // Check all passed properties for "Title"... 487 for (final CmsProperty prop : propertiesSearched) { 488 if (prop.getName().equals(CmsPropertyDefinition.PROPERTY_TITLE)) { 489 final String value = prop.getValue(); 490 491 // Write a Solr locale-aware field for every locale the system supports... 492 final List<Locale> availableLocales = OpenCms.getLocaleManager().getAvailableLocales(); 493 for (final Locale locale : availableLocales) { 494 final String lang = locale.getLanguage(); 495 // Don't proceed if a field has already written for this locale. 496 if (!resourceLocales.contains(lang)) { 497 final String effFieldName = CmsSearchFieldConfiguration.getLocaleExtendedName( 498 CmsSearchField.FIELD_TITLE_UNSTORED, 499 locale) + "_s"; 500 501 final CmsSolrField f = new CmsSolrField(effFieldName, null, null, null); 502 document.addSearchField(f, value); 503 } 504 } 505 } 506 } 507 m_contentLocales = getContentLocales(cms, resource, extraction); 508 } 509 510 document.addResourceLocales(resourceLocales); 511 document.addContentLocales(m_contentLocales); 512 513 // append document dependencies if configured 514 if (hasLocaleDependencies()) { 515 CmsDocumentDependency dep = CmsDocumentDependency.load(cms, resource); 516 ((CmsSolrDocument)document).addDocumentDependency(cms, dep); 517 } 518 return document; 519 } 520 521 /** 522 * @see org.opencms.search.fields.CmsSearchFieldConfiguration#appendProperties(org.opencms.search.I_CmsSearchDocument, org.opencms.file.CmsObject, org.opencms.file.CmsResource, org.opencms.search.extractors.I_CmsExtractionResult, java.util.List, java.util.List) 523 */ 524 @Override 525 protected I_CmsSearchDocument appendProperties( 526 I_CmsSearchDocument document, 527 CmsObject cms, 528 CmsResource resource, 529 I_CmsExtractionResult extraction, 530 List<CmsProperty> properties, 531 List<CmsProperty> propertiesSearched) { 532 533 for (CmsProperty prop : propertiesSearched) { 534 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(prop.getValue())) { 535 String value = CmsSearchUtil.stripHtmlFromPropertyIfNecessary(prop.getName(), prop.getValue()); 536 document.addSearchField( 537 new CmsSolrField(prop.getName() + CmsSearchField.FIELD_DYNAMIC_PROPERTIES, null, null, null), 538 value); 539 540 // Also write the property using the dynamic field '_s' in order to prevent tokenization 541 // of the property. The resulting field is named '<property>_prop_s'. 542 document.addSearchField( 543 new CmsSolrField(prop.getName() + CmsSearchField.FIELD_DYNAMIC_PROPERTIES + "_s", null, null, null), 544 value); 545 } 546 } 547 548 for (CmsProperty prop : properties) { 549 if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(prop.getValue())) { 550 String value = CmsSearchUtil.stripHtmlFromPropertyIfNecessary(prop.getName(), prop.getValue()); 551 document.addSearchField( 552 new CmsSolrField(prop.getName() + CmsSearchField.FIELD_DYNAMIC_PROPERTIES_DIRECT, null, null, null), 553 value); 554 555 // Also write the property using the dynamic field '_s' in order to prevent tokenization 556 // of the property. The resulting field is named '<property>_prop_nosearch_s'. 557 document.addSearchField( 558 new CmsSolrField( 559 prop.getName() + CmsSearchField.FIELD_DYNAMIC_PROPERTIES_DIRECT + "_s", 560 null, 561 null, 562 null), 563 value); 564 } 565 } 566 return document; 567 } 568 569 /** 570 * Retrieves the locales for an content, that is whether an XML content nor an XML page.<p> 571 * 572 * Uses following strategy: 573 * <ul> 574 * <li>first by file name</li> 575 * <li>then by detection and</li> 576 * <li>otherwise take the first configured default locale for this resource</li> 577 * </ul> 578 * 579 * @param cms the current CmsObject 580 * @param resource the resource to get the content locales for 581 * @param extraction the extraction result 582 * 583 * @return the determined locales for the given resource 584 */ 585 protected List<Locale> getContentLocales(CmsObject cms, CmsResource resource, I_CmsExtractionResult extraction) { 586 587 // try to detect locale by filename 588 Locale detectedLocale = CmsStringUtil.getLocaleForName(resource.getRootPath()); 589 if (!OpenCms.getLocaleManager().getAvailableLocales(cms, resource).contains(detectedLocale)) { 590 detectedLocale = null; 591 } 592 // try to detect locale by language detector 593 if (getIndex().isLanguageDetection() 594 && (detectedLocale == null) 595 && (extraction != null) 596 && (extraction.getContent() != null)) { 597 detectedLocale = CmsStringUtil.getLocaleForText(extraction.getContent()); 598 } 599 // take the detected locale or use the first configured default locale for this resource 600 List<Locale> result = new ArrayList<Locale>(); 601 if (detectedLocale != null) { 602 // take the found locale 603 result.add(detectedLocale); 604 } else { 605 606 // take all locales set via locale-available or the configured default locales as fall-back for this resource 607 result.addAll(OpenCms.getLocaleManager().getAvailableLocales(cms, resource)); 608 LOG.debug(Messages.get().getBundle().key(Messages.LOG_LANGUAGE_DETECTION_FAILED_1, resource)); 609 } 610 return result; 611 } 612 613 /** 614 * Returns the search field mappings declared within the XSD.<p> 615 * 616 * @param cms the CmsObject 617 * @param resource the resource 618 * 619 * @return the fields to map 620 */ 621 protected Set<CmsSearchField> getXSDMappings(CmsObject cms, CmsResource resource) { 622 623 try { 624 if (CmsResourceTypeXmlContent.isXmlContent(resource)) { 625 I_CmsXmlContentHandler handler = CmsXmlContentDefinition.getContentHandlerForResource(cms, resource); 626 if ((handler != null) && !handler.getSearchFields().isEmpty()) { 627 return handler.getSearchFields(); 628 } 629 } 630 } catch (CmsException e) { 631 LOG.error(e.getMessage(), e); 632 } 633 return null; 634 } 635 636 /** 637 * Returns the search field mappings declared within the XSD that should be applied to the container page.<p> 638 * 639 * @param cms the CmsObject 640 * @param resource the resource 641 * 642 * @return the fields to map 643 */ 644 protected Set<CmsSearchField> getXSDMappingsForPage(CmsObject cms, CmsResource resource) { 645 646 try { 647 if (CmsResourceTypeXmlContent.isXmlContent(resource)) { 648 I_CmsXmlContentHandler handler = CmsXmlContentDefinition.getContentHandlerForResource(cms, resource); 649 if ((handler != null) && !handler.getSearchFieldsForPage().isEmpty()) { 650 return handler.getSearchFieldsForPage(); 651 } 652 } 653 } catch (CmsException e) { 654 LOG.error(e.getMessage(), e); 655 } 656 return null; 657 } 658 659 /** 660 * Adds additional fields to this field configuration.<p> 661 */ 662 private void addAdditionalFields() { 663 664 /* 665 * Add fields from opencms-search.xml (Lucene fields) 666 */ 667 for (CmsSearchField field : getFields()) { 668 if (field instanceof CmsLuceneField) { 669 CmsSolrField newSolrField = new CmsSolrField((CmsLuceneField)field); 670 m_solrFields.put(newSolrField.getName(), newSolrField); 671 } 672 } 673 674 /* 675 * Add the content fields (multiple for contents with more than one locale) 676 */ 677 // add the content_<locale> fields to this configuration 678 CmsSolrField solrField = new CmsSolrField(CmsSearchField.FIELD_CONTENT, null, null, null); 679 solrField.addMapping( 680 new CmsSearchFieldMapping(CmsSearchFieldMappingType.CONTENT, CmsSearchField.FIELD_CONTENT)); 681 m_solrFields.put(solrField.getName(), solrField); 682 for (Locale locale : OpenCms.getLocaleManager().getAvailableLocales()) { 683 solrField = new CmsSolrField( 684 CmsSearchFieldConfiguration.getLocaleExtendedName(CmsSearchField.FIELD_CONTENT, locale), 685 Collections.singletonList(locale.toString() + CmsSearchField.FIELD_EXCERPT), 686 locale, 687 null); 688 solrField.addMapping( 689 new CmsSearchFieldMapping(CmsSearchFieldMappingType.CONTENT, CmsSearchField.FIELD_CONTENT)); 690 m_solrFields.put(solrField.getName(), solrField); 691 } 692 693 /* 694 * Fields filled within appendFields 695 */ 696 CmsSolrField sfield = new CmsSolrField(CmsSearchField.FIELD_MIMETYPE, null, null, null); 697 m_solrFields.put(sfield.getName(), sfield); 698 699 sfield = new CmsSolrField(CmsSearchField.FIELD_FILENAME, null, null, null); 700 m_solrFields.put(sfield.getName(), sfield); 701 702 sfield = new CmsSolrField(CmsSearchField.FIELD_VERSION, null, null, null); 703 m_solrFields.put(sfield.getName(), sfield); 704 705 sfield = new CmsSolrField(CmsSearchField.FIELD_SEARCH_CHANNEL, null, null, null); 706 m_solrFields.put(sfield.getName(), sfield); 707 708 /* 709 * Fields with mapping 710 */ 711 sfield = new CmsSolrField(CmsSearchField.FIELD_STATE, null, null, null); 712 CmsSearchFieldMapping map = new CmsSearchFieldMapping( 713 CmsSearchFieldMappingType.ATTRIBUTE, 714 CmsSearchField.FIELD_STATE); 715 sfield.addMapping(map); 716 m_solrFields.put(sfield.getName(), sfield); 717 718 sfield = new CmsSolrField(CmsSearchField.FIELD_USER_LAST_MODIFIED, null, null, null); 719 map = new CmsSearchFieldMapping(CmsSearchFieldMappingType.ATTRIBUTE, CmsSearchField.FIELD_USER_LAST_MODIFIED); 720 sfield.addMapping(map); 721 m_solrFields.put(sfield.getName(), sfield); 722 723 sfield = new CmsSolrField(CmsSearchField.FIELD_USER_CREATED, null, null, null); 724 map = new CmsSearchFieldMapping(CmsSearchFieldMappingType.ATTRIBUTE, CmsSearchField.FIELD_USER_CREATED); 725 sfield.addMapping(map); 726 m_solrFields.put(sfield.getName(), sfield); 727 728 sfield = new CmsSolrField(CmsSearchField.FIELD_META, null, null, null); 729 map = new CmsSearchFieldMapping(CmsSearchFieldMappingType.PROPERTY, CmsPropertyDefinition.PROPERTY_TITLE); 730 sfield.addMapping(map); 731 map = new CmsSearchFieldMapping(CmsSearchFieldMappingType.PROPERTY, CmsPropertyDefinition.PROPERTY_DESCRIPTION); 732 sfield.addMapping(map); 733 map = new CmsSearchFieldMapping(CmsSearchFieldMappingType.ATTRIBUTE, I_CmsXmlConfiguration.A_NAME); 734 sfield.addMapping(map); 735 m_solrFields.put(sfield.getName(), sfield); 736 737 sfield = new CmsSolrField(CmsSearchField.FIELD_SEARCH_EXCLUDE, null, null, null); 738 map = new CmsSearchFieldMapping( 739 CmsSearchFieldMappingType.PROPERTY_SEARCH, 740 CmsPropertyDefinition.PROPERTY_SEARCH_EXCLUDE); 741 sfield.addMapping(map); 742 m_solrFields.put(sfield.getName(), sfield); 743 744 } 745 746 /** 747 * Adds multiple fields to the document that are used for the sort options in the list app. 748 * 749 * <p>The fields are: 750 * <ul> 751 * <li>instancedate_dt</li> 752 * <li>instancedatecurrenttill_dt</li> 753 * <li>instancedaterange_dr</li> 754 * <li>disptitle_s</li> 755 * <li>disporder_i</li> 756 * </ul> 757 * and localized versions for each content locale.</p> 758 * 759 * @param document the document to index with all other fields already added. 760 * @return the document extended by the fields used by the list. 761 */ 762 private I_CmsSearchDocument appendFieldsForListSortOptions(I_CmsSearchDocument document) { 763 764 // add non-localized fields 765 // add instance date 766 String fieldName = CmsSearchField.FIELD_INSTANCEDATE + CmsSearchField.FIELD_POSTFIX_DATE; 767 Date instanceDate = document.getFieldValueAsDate(fieldName); 768 if ((null == instanceDate) || (instanceDate.getTime() == 0)) { 769 String instanceDateCopyField = document.getFieldValueAsString( 770 CmsPropertyDefinition.PROPERTY_INSTANCEDATE_COPYFIELD + CmsSearchField.FIELD_DYNAMIC_PROPERTIES); 771 if (null != instanceDateCopyField) { 772 instanceDate = document.getFieldValueAsDate(instanceDateCopyField); 773 } 774 if ((null == instanceDate) || (instanceDate.getTime() == 0)) { 775 instanceDate = document.getFieldValueAsDate(CmsSearchField.FIELD_DATE_RELEASED); 776 } 777 if ((null == instanceDate) || (instanceDate.getTime() == 0)) { 778 instanceDate = document.getFieldValueAsDate(CmsSearchField.FIELD_DATE_LASTMODIFIED); 779 } 780 document.addDateField(fieldName, instanceDate.getTime(), false); 781 } 782 // Set instancedaterange_dr 783 fieldName = CmsSearchField.FIELD_INSTANCEDATE_RANGE + CmsSearchField.FIELD_POSTFIX_DATE_RANGE; 784 String instanceDateString = document.getFieldValueAsString( 785 CmsSearchField.FIELD_INSTANCEDATE + CmsSearchField.FIELD_POSTFIX_DATE); 786 String instanceDateRangeString = "[" + instanceDateString + " TO " + instanceDateString + "]"; 787 ((SolrInputDocument)document.getDocument()).setField(fieldName, instanceDateRangeString); 788 // Set instancedatecurrenttill_dt to instancedate_dt if not set yet 789 fieldName = CmsSearchField.FIELD_INSTANCEDATE_CURRENT_TILL + CmsSearchField.FIELD_POSTFIX_DATE; 790 Date instanceDateCurrentTill = document.getFieldValueAsDate(fieldName); 791 if ((null == instanceDateCurrentTill) || (instanceDateCurrentTill.getTime() == 0)) { 792 document.addDateField(fieldName, instanceDate.getTime(), false); 793 } 794 // add disp-title field 795 fieldName = CmsSearchField.FIELD_DISPTITLE + CmsSearchField.FIELD_POSTFIX_SORT; 796 String dispTitle = document.getFieldValueAsString(fieldName); 797 if (null == dispTitle) { 798 dispTitle = document.getFieldValueAsString( 799 CmsPropertyDefinition.PROPERTY_TITLE + CmsSearchField.FIELD_DYNAMIC_PROPERTIES_DIRECT); 800 if (null == dispTitle) { 801 dispTitle = document.getFieldValueAsString(CmsSearchField.FIELD_FILENAME); 802 } 803 document.addSearchField(new CmsSolrField(fieldName, null, null, null), dispTitle); 804 } 805 806 // add disp-order field 807 fieldName = CmsSearchField.FIELD_DISPORDER + CmsSearchField.FIELD_POSTFIX_INT; 808 String dispOrder = document.getFieldValueAsString(fieldName); 809 if (null == dispOrder) { 810 dispOrder = document.getFieldValueAsString( 811 CmsPropertyDefinition.PROPERTY_DISPLAY_ORDER + CmsSearchField.FIELD_DYNAMIC_PROPERTIES); 812 if (null != dispOrder) { 813 try { 814 int o = Integer.parseInt(dispOrder); 815 dispOrder = String.valueOf(o); 816 } catch (NullPointerException | NumberFormatException e) { 817 LOG.warn( 818 "Property " 819 + CmsPropertyDefinition.PROPERTY_DISPLAY_ORDER 820 + " contains not a valid integer number."); 821 dispOrder = "0"; 822 } 823 } else { 824 dispOrder = "0"; 825 } 826 document.addSearchField(new CmsSolrField(fieldName, null, null, null), dispOrder); 827 } 828 829 // add localized fields 830 for (String locale : document.getMultivaluedFieldAsStringList(CmsSearchField.FIELD_CONTENT_LOCALES)) { 831 // instance date 832 fieldName = CmsSearchField.FIELD_INSTANCEDATE + "_" + locale + CmsSearchField.FIELD_POSTFIX_DATE; 833 Date localeInstanceDate = document.getFieldValueAsDate(fieldName); 834 if ((null == localeInstanceDate) || (localeInstanceDate.getTime() == 0)) { 835 localeInstanceDate = instanceDate; 836 document.addDateField(fieldName, localeInstanceDate.getTime(), false); 837 } 838 // instance date range 839 fieldName = CmsSearchField.FIELD_INSTANCEDATE_RANGE 840 + "_" 841 + locale 842 + CmsSearchField.FIELD_POSTFIX_DATE_RANGE; 843 String localeInstanceDateString = document.getFieldValueAsString( 844 CmsSearchField.FIELD_INSTANCEDATE + "_" + locale + CmsSearchField.FIELD_POSTFIX_DATE); 845 String localeInstanceDateRangeString = "[" 846 + localeInstanceDateString 847 + " TO " 848 + localeInstanceDateString 849 + "]"; 850 ((SolrInputDocument)document.getDocument()).setField(fieldName, localeInstanceDateRangeString); 851 // Set instancedatecurrenttill_dt to instancedate_dt if not set yet 852 fieldName = CmsSearchField.FIELD_INSTANCEDATE_CURRENT_TILL 853 + "_" 854 + locale 855 + CmsSearchField.FIELD_POSTFIX_DATE; 856 Date localeInstanceDateCurrentTill = document.getFieldValueAsDate(fieldName); 857 if ((null == localeInstanceDateCurrentTill) || (localeInstanceDateCurrentTill.getTime() == 0)) { 858 document.addDateField(fieldName, localeInstanceDate.getTime(), false); 859 } 860 // disp-title field for title display and sorting 861 fieldName = CmsSearchField.FIELD_DISPTITLE + "_" + locale + CmsSearchField.FIELD_POSTFIX_SORT; 862 if (null == document.getFieldValueAsString(fieldName)) { 863 String localizedTitle = document.getFieldValueAsString( 864 CmsPropertyDefinition.PROPERTY_TITLE 865 + "_" 866 + locale 867 + CmsSearchField.FIELD_DYNAMIC_PROPERTIES_DIRECT); 868 document.addSearchField( 869 new CmsSolrField(fieldName, null, null, null), 870 null == localizedTitle ? dispTitle : localizedTitle); 871 } 872 // disp-order field 873 fieldName = CmsSearchField.FIELD_DISPORDER + "_" + locale + CmsSearchField.FIELD_POSTFIX_INT; 874 if (null == document.getFieldValueAsString(fieldName)) { 875 String localizedOrder = document.getFieldValueAsString( 876 CmsPropertyDefinition.PROPERTY_DISPLAY_ORDER 877 + "_" 878 + locale 879 + CmsSearchField.FIELD_DYNAMIC_PROPERTIES); 880 if (null != localizedOrder) { 881 try { 882 int o = Integer.parseInt(localizedOrder); 883 localizedOrder = String.valueOf(o); 884 } catch (NullPointerException | NumberFormatException e) { 885 LOG.warn( 886 "Property " 887 + CmsPropertyDefinition.PROPERTY_DISPLAY_ORDER 888 + "_" 889 + locale 890 + " contains not a valid integer number."); 891 } 892 } 893 document.addSearchField( 894 new CmsSolrField(fieldName, null, null, null), 895 null == localizedOrder ? dispOrder : localizedOrder); 896 } 897 } 898 899 return document; 900 } 901 902 /** 903 * Copy the content and the title property of the document to a spell field / a language specific spell field. 904 * @param document the document that gets extended by the spell fields. 905 */ 906 private void appendSpellFields(I_CmsSearchDocument document) { 907 908 /* 909 * Add the content fields (multiple for contents with more than one locale) 910 */ 911 // add the content_<locale> fields to this configuration 912 String title = document.getFieldValueAsString( 913 CmsPropertyDefinition.PROPERTY_TITLE + CmsSearchField.FIELD_DYNAMIC_PROPERTIES_DIRECT); 914 document.addSearchField( 915 new CmsSolrField(CmsSearchField.FIELD_SPELL, null, null, null), 916 document.getFieldValueAsString(CmsSearchField.FIELD_CONTENT) + "\n" + title); 917 for (Locale locale : OpenCms.getLocaleManager().getAvailableLocales()) { 918 document.addSearchField( 919 new CmsSolrField(locale + "_" + CmsSearchField.FIELD_SPELL, null, locale, null), 920 document.getFieldValueAsString( 921 CmsSearchFieldConfiguration.getLocaleExtendedName(CmsSearchField.FIELD_CONTENT, locale)) 922 + "\n" 923 + title); 924 } 925 } 926 927 /** 928 * Returns <code>true</code> if at least one of the index sources uses a VFS indexer that is able 929 * to index locale dependent resources.<p> 930 * 931 * TODO This should be improved somehow 932 * 933 * @return <code>true</code> if this field configuration should resolve locale dependencies 934 */ 935 private boolean hasLocaleDependencies() { 936 937 for (CmsSearchIndexSource source : getIndex().getSources()) { 938 if (source.getIndexer().isLocaleDependenciesEnable()) { 939 return true; 940 } 941 } 942 return false; 943 } 944}