001/*
002 * File   : $Source$
003 * Date   : $Date$
004 * Version: $Revision$
005 *
006 * This library is part of OpenCms -
007 * the Open Source Content Management System
008 *
009 * Copyright (C) 2002 - 2009 Alkacon Software (http://www.alkacon.com)
010 *
011 * This library is free software; you can redistribute it and/or
012 * modify it under the terms of the GNU Lesser General Public
013 * License as published by the Free Software Foundation; either
014 * version 2.1 of the License, or (at your option) any later version.
015 *
016 * This library is distributed in the hope that it will be useful,
017 * but WITHOUT ANY WARRANTY; without even the implied warranty of
018 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
019 * Lesser General Public License for more details.
020 *
021 * For further information about Alkacon Software, please see the
022 * company website: http://www.alkacon.com
023 *
024 * For further information about OpenCms, please see the
025 * project website: http://www.opencms.org
026 *
027 * You should have received a copy of the GNU Lesser General Public
028 * License along with this library; if not, write to the Free Software
029 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
030 */
031
032package org.opencms.search.solr;
033
034import org.opencms.configuration.CmsConfigurationException;
035import org.opencms.configuration.CmsParameterConfiguration;
036import org.opencms.file.CmsFile;
037import org.opencms.file.CmsObject;
038import org.opencms.file.CmsProject;
039import org.opencms.file.CmsPropertyDefinition;
040import org.opencms.file.CmsResource;
041import org.opencms.file.CmsResourceFilter;
042import org.opencms.i18n.CmsEncoder;
043import org.opencms.i18n.CmsLocaleManager;
044import org.opencms.main.CmsException;
045import org.opencms.main.CmsIllegalArgumentException;
046import org.opencms.main.CmsLog;
047import org.opencms.main.OpenCms;
048import org.opencms.report.I_CmsReport;
049import org.opencms.search.CmsSearchException;
050import org.opencms.search.CmsSearchIndex;
051import org.opencms.search.CmsSearchManager;
052import org.opencms.search.CmsSearchParameters;
053import org.opencms.search.CmsSearchResource;
054import org.opencms.search.CmsSearchResultList;
055import org.opencms.search.I_CmsIndexWriter;
056import org.opencms.search.I_CmsSearchDocument;
057import org.opencms.search.fields.CmsSearchField;
058import org.opencms.search.galleries.CmsGallerySearchParameters;
059import org.opencms.search.galleries.CmsGallerySearchResult;
060import org.opencms.search.galleries.CmsGallerySearchResultList;
061import org.opencms.security.CmsRole;
062import org.opencms.security.CmsRoleViolationException;
063import org.opencms.util.CmsFileUtil;
064import org.opencms.util.CmsRequestUtil;
065import org.opencms.util.CmsStringUtil;
066
067import java.io.IOException;
068import java.io.OutputStreamWriter;
069import java.io.UnsupportedEncodingException;
070import java.io.Writer;
071import java.nio.charset.Charset;
072import java.util.ArrayList;
073import java.util.Arrays;
074import java.util.Collections;
075import java.util.HashSet;
076import java.util.List;
077import java.util.Locale;
078import java.util.Optional;
079import java.util.Set;
080import java.util.stream.Stream;
081
082import javax.servlet.ServletResponse;
083
084import org.apache.commons.logging.Log;
085import org.apache.solr.client.solrj.SolrClient;
086import org.apache.solr.client.solrj.SolrQuery;
087import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
088import org.apache.solr.client.solrj.response.QueryResponse;
089import org.apache.solr.common.SolrDocument;
090import org.apache.solr.common.SolrDocumentList;
091import org.apache.solr.common.SolrInputDocument;
092import org.apache.solr.common.util.ContentStreamBase;
093import org.apache.solr.common.util.FastWriter;
094import org.apache.solr.common.util.NamedList;
095import org.apache.solr.common.util.SimpleOrderedMap;
096import org.apache.solr.core.CoreContainer;
097import org.apache.solr.core.SolrCore;
098import org.apache.solr.handler.ReplicationHandler;
099import org.apache.solr.request.LocalSolrQueryRequest;
100import org.apache.solr.request.SolrQueryRequest;
101import org.apache.solr.request.SolrRequestHandler;
102import org.apache.solr.response.BinaryQueryResponseWriter;
103import org.apache.solr.response.QueryResponseWriter;
104import org.apache.solr.response.SolrQueryResponse;
105
106import com.google.common.base.Objects;
107
108/**
109 * Implements the search within an Solr index.<p>
110 *
111 * @since 8.5.0
112 */
113public class CmsSolrIndex extends CmsSearchIndex {
114
115    /** The serial version id. */
116    private static final long serialVersionUID = -1570077792574476721L;
117
118    /** The name of the default Solr Offline index. */
119    public static final String DEFAULT_INDEX_NAME_OFFLINE = "Solr Offline";
120
121    /** The name of the default Solr Online index. */
122    public static final String DEFAULT_INDEX_NAME_ONLINE = "Solr Online";
123
124    /** Constant for additional parameter to set the post processor class name. */
125    public static final String POST_PROCESSOR = "search.solr.postProcessor";
126
127    /**
128     * Constant for additional parameter to set the maximally processed results (start + rows) for searches with this index.
129     * It overwrites the global configuration from {@link CmsSolrConfiguration#getMaxProcessedResults()} for this index.
130    **/
131    public static final String SOLR_SEARCH_MAX_PROCESSED_RESULTS = "search.solr.maxProcessedResults";
132
133    /** Constant for additional parameter to set the fields the select handler should return at maximum. */
134    public static final String SOLR_HANDLER_ALLOWED_FIELDS = "handle.solr.allowedFields";
135
136    /** Constant for additional parameter to set the number results the select handler should return at maxium per request. */
137    public static final String SOLR_HANDLER_MAX_ALLOWED_RESULTS_PER_PAGE = "handle.solr.maxAllowedResultsPerPage";
138
139    /** Constant for additional parameter to set the maximal number of a result, the select handler should return. */
140    public static final String SOLR_HANDLER_MAX_ALLOWED_RESULTS_AT_ALL = "handle.solr.maxAllowedResultsAtAll";
141
142    /** Constant for additional parameter to disable the select handler (except for debug mode). */
143    private static final String SOLR_HANDLER_DISABLE_SELECT = "handle.solr.disableSelectHandler";
144
145    /** Constant for additional parameter to set the VFS path to the file holding the debug secret. */
146    private static final String SOLR_HANDLER_DEBUG_SECRET_FILE = "handle.solr.debugSecretFile";
147
148    /** Constant for additional parameter to disable the spell handler (except for debug mode). */
149    private static final String SOLR_HANDLER_DISABLE_SPELL = "handle.solr.disableSpellHandler";
150
151    /** Constant for additional parameter to configure an external solr server specifically for the index. */
152    private static final String SOLR_SERVER_URL = "server.url";
153
154    /** The solr exclude property. */
155    public static final String PROPERTY_SEARCH_EXCLUDE_VALUE_SOLR = "solr";
156
157    /** Indicates the maximum number of documents from the complete result set to return. */
158    public static final int ROWS_MAX = 50;
159
160    /** The constant for an unlimited maximum number of results to return in a Solr search. */
161    public static final int MAX_RESULTS_UNLIMITED = -1;
162
163    /** The constant for an unlimited maximum number of results to return in a Solr search. */
164    public static final int MAX_RESULTS_GALLERY = 10000;
165
166    /** A constant for debug formatting output. */
167    protected static final int DEBUG_PADDING_RIGHT = 50;
168
169    /** The name for the parameters key of the response header. */
170    private static final String HEADER_PARAMS_NAME = "params";
171
172    /** The log object for this class. */
173    private static final Log LOG = CmsLog.getLog(CmsSolrIndex.class);
174
175    /** Pseudo resource used for not permission checked indexes. */
176    private static final CmsResource PSEUDO_RES = new CmsResource(
177        null,
178        null,
179        null,
180        0,
181        false,
182        0,
183        null,
184        null,
185        0L,
186        null,
187        0L,
188        null,
189        0L,
190        0L,
191        0,
192        0,
193        0L,
194        0);
195
196    /** The name of the key that is used for the result documents inside the Solr query response. */
197    private static final String QUERY_RESPONSE_NAME = "response";
198
199    /** The name of the key that is used for the query time. */
200    private static final String QUERY_TIME_NAME = "QTime";
201
202    /** The name of the key that is used for the query time. */
203    private static final String QUERY_HIGHLIGHTING_NAME = "highlighting";
204
205    /** A constant for UTF-8 charset. */
206    private static final Charset UTF8 = Charset.forName("UTF-8");
207
208    /** The name of the request parameter holding the debug secret. */
209    private static final String REQUEST_PARAM_DEBUG_SECRET = "_debug";
210
211    /** The name of the query parameter enabling spell checking. */
212    private static final String QUERY_SPELLCHECK_NAME = "spellcheck";
213
214    /** The name of the query parameter sorting. */
215    private static final String QUERY_SORT_NAME = "sort";
216
217    /** The name of the query parameter expand. */
218    private static final String QUERY_PARAM_EXPAND = "expand";
219
220    /** The embedded Solr client for this index. */
221    transient SolrClient m_solr;
222
223    /** The post document manipulator. */
224    private transient I_CmsSolrPostSearchProcessor m_postProcessor;
225
226    /** The core name for the index. */
227    private transient String m_coreName;
228
229    /** The list of allowed fields to return. */
230    private String[] m_handlerAllowedFields;
231
232    /** The number of maximally allowed results per page when using the handler. */
233    private int m_handlerMaxAllowedResultsPerPage = -1;
234
235    /** The number of maximally allowed results at all when using the handler. */
236    private int m_handlerMaxAllowedResultsAtAll = -1;
237
238    /** Flag, indicating if the handler only works in debug mode. */
239    private boolean m_handlerSelectDisabled;
240
241    /** Path to the secret file. Must be under /system/.../ or /shared/.../ and readable by all users that should be able to debug. */
242    private String m_handlerDebugSecretFile;
243
244    /** Flag, indicating if the spellcheck handler is disabled for the index. */
245    private boolean m_handlerSpellDisabled;
246
247    /** The maximal number of results to process for search queries. */
248    int m_maxProcessedResults = -2; // special value for not initialized.
249
250    /** Server URL to use specific for the index. If set, it overwrites all other server settings. */
251    private String m_serverUrl;
252
253    /**
254     * Default constructor.<p>
255     */
256    public CmsSolrIndex() {
257
258        super();
259    }
260
261    /**
262     * Public constructor to create a Solr index.<p>
263     *
264     * @param name the name for this index.<p>
265     *
266     * @throws CmsIllegalArgumentException if something goes wrong
267     */
268    public CmsSolrIndex(String name)
269    throws CmsIllegalArgumentException {
270
271        super(name);
272    }
273
274    /**
275     * Returns the resource type for the given root path.<p>
276     *
277     * @param cms the current CMS context
278     * @param rootPath the root path of the resource to get the type for
279     *
280     * @return the resource type for the given root path
281     */
282    public static final String getType(CmsObject cms, String rootPath) {
283
284        String type = null;
285        CmsSolrIndex index = CmsSearchManager.getIndexSolr(cms, null);
286        if (index != null) {
287            I_CmsSearchDocument doc = index.getDocument(CmsSearchField.FIELD_PATH, rootPath);
288            if (doc != null) {
289                type = doc.getFieldValueAsString(CmsSearchField.FIELD_TYPE);
290            }
291        }
292        return type;
293    }
294
295    /**
296     * @see org.opencms.search.CmsSearchIndex#addConfigurationParameter(java.lang.String, java.lang.String)
297     */
298    @Override
299    public void addConfigurationParameter(String key, String value) {
300
301        switch (key) {
302            case POST_PROCESSOR:
303                if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) {
304                    try {
305                        setPostProcessor((I_CmsSolrPostSearchProcessor)Class.forName(value).newInstance());
306                    } catch (Exception e) {
307                        CmsException ex = new CmsException(
308                            Messages.get().container(Messages.LOG_SOLR_ERR_POST_PROCESSOR_NOT_EXIST_1, value),
309                            e);
310                        LOG.error(ex.getMessage(), ex);
311                    }
312                }
313                break;
314            case SOLR_HANDLER_ALLOWED_FIELDS:
315                if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) {
316                    m_handlerAllowedFields = Stream.of(value.split(",")).map(v -> v.trim()).toArray(String[]::new);
317                }
318                break;
319            case SOLR_HANDLER_MAX_ALLOWED_RESULTS_PER_PAGE:
320                if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) {
321                    try {
322                        m_handlerMaxAllowedResultsPerPage = Integer.parseInt(value);
323                    } catch (NumberFormatException e) {
324                        LOG.warn(
325                            "Could not parse parameter \""
326                                + SOLR_HANDLER_MAX_ALLOWED_RESULTS_PER_PAGE
327                                + "\" for index \""
328                                + getName()
329                                + "\". Results per page will not be restricted.");
330                    }
331                }
332                break;
333            case SOLR_HANDLER_MAX_ALLOWED_RESULTS_AT_ALL:
334                if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) {
335                    try {
336                        m_handlerMaxAllowedResultsAtAll = Integer.parseInt(value);
337                    } catch (NumberFormatException e) {
338                        LOG.warn(
339                            "Could not parse parameter \""
340                                + SOLR_HANDLER_MAX_ALLOWED_RESULTS_AT_ALL
341                                + "\" for index \""
342                                + getName()
343                                + "\". Results per page will not be restricted.");
344                    }
345                }
346                break;
347            case SOLR_HANDLER_DISABLE_SELECT:
348                if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) {
349                    m_handlerSelectDisabled = value.trim().toLowerCase().equals("true");
350                }
351                break;
352            case SOLR_HANDLER_DEBUG_SECRET_FILE:
353                if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) {
354                    m_handlerDebugSecretFile = value.trim();
355                }
356                break;
357            case SOLR_HANDLER_DISABLE_SPELL:
358                if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) {
359                    m_handlerSpellDisabled = value.trim().toLowerCase().equals("true");
360                }
361                break;
362            case SOLR_SEARCH_MAX_PROCESSED_RESULTS:
363                if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) {
364                    try {
365                        m_maxProcessedResults = Integer.parseInt(value);
366                    } catch (NumberFormatException e) {
367                        LOG.warn(
368                            "Could not parse parameter \""
369                                + SOLR_SEARCH_MAX_PROCESSED_RESULTS
370                                + "\" for index \""
371                                + getName()
372                                + "\". The global configuration will be used instead.");
373                    }
374                }
375                break;
376            case SOLR_SERVER_URL:
377                if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(value)) {
378                    m_serverUrl = value.trim();
379                }
380                break;
381            default:
382                super.addConfigurationParameter(key, value);
383                break;
384        }
385    }
386
387    /**
388     * @see org.opencms.search.CmsSearchIndex#createEmptyDocument(org.opencms.file.CmsResource)
389     */
390    @Override
391    public I_CmsSearchDocument createEmptyDocument(CmsResource resource) {
392
393        CmsSolrDocument doc = new CmsSolrDocument(new SolrInputDocument());
394        doc.setId(resource.getStructureId());
395        return doc;
396    }
397
398    /**
399     * @see org.opencms.search.CmsSearchIndex#createIndexWriter(boolean, org.opencms.report.I_CmsReport)
400     */
401    @Override
402    public I_CmsIndexWriter createIndexWriter(boolean create, I_CmsReport report) {
403
404        return new CmsSolrIndexWriter(m_solr, this);
405    }
406
407    /**
408     * @see org.opencms.search.CmsSearchIndex#excludeFromIndex(CmsObject, CmsResource)
409     */
410    @Override
411    public boolean excludeFromIndex(CmsObject cms, CmsResource resource) {
412
413        if (resource.isFolder() || resource.isTemporaryFile()) {
414            // don't index  folders or temporary files for galleries, but pretty much everything else
415            return true;
416        }
417        // If this is the default offline index than it is used for gallery search that needs all resources indexed.
418        if (this.getName().equals(DEFAULT_INDEX_NAME_OFFLINE)) {
419            return false;
420        }
421
422        boolean isOnlineIndex = getProject().equals(CmsProject.ONLINE_PROJECT_NAME);
423        if (isOnlineIndex && (resource.getDateExpired() <= System.currentTimeMillis())) {
424            return true;
425        }
426
427        try {
428            // do property lookup with folder search
429            String propValue = cms.readPropertyObject(
430                resource,
431                CmsPropertyDefinition.PROPERTY_SEARCH_EXCLUDE,
432                true).getValue();
433            if (propValue != null) {
434                if (!("false".equalsIgnoreCase(propValue.trim()))) {
435                    return true;
436                }
437            }
438        } catch (CmsException e) {
439            if (LOG.isDebugEnabled()) {
440                LOG.debug(
441                    org.opencms.search.Messages.get().getBundle().key(
442                        org.opencms.search.Messages.LOG_UNABLE_TO_READ_PROPERTY_1,
443                        resource.getRootPath()));
444            }
445        }
446        if (!USE_ALL_LOCALE.equalsIgnoreCase(getLocale().getLanguage())) {
447            // check if any resource default locale has a match with the index locale, if not skip resource
448            List<Locale> locales = OpenCms.getLocaleManager().getDefaultLocales(cms, resource);
449            Locale match = OpenCms.getLocaleManager().getFirstMatchingLocale(
450                Collections.singletonList(getLocale()),
451                locales);
452            return (match == null);
453        }
454        return false;
455
456    }
457
458    /**
459     * Performs a search with according to the gallery search parameters.<p>
460     *
461     * @param cms the cms context
462     * @param params the search parameters
463     *
464     * @return the search result
465     */
466    public CmsGallerySearchResultList gallerySearch(CmsObject cms, CmsGallerySearchParameters params) {
467
468        CmsGallerySearchResultList resultList = new CmsGallerySearchResultList();
469        if (params.isForceEmptyResult()) {
470            return resultList;
471        }
472
473        try {
474            CmsSolrResultList list = search(
475                cms,
476                params.getQuery(cms),
477                false,
478                null,
479                true,
480                CmsResourceFilter.ONLY_VISIBLE_NO_DELETED,
481                MAX_RESULTS_GALLERY); // ignore the maximally searched number of contents.
482
483            if (null == list) {
484                return null;
485            }
486
487            resultList.setHitCount(Long.valueOf(list.getNumFound()).intValue());
488            for (CmsSearchResource resource : list) {
489                I_CmsSearchDocument document = resource.getDocument();
490                Locale locale = CmsLocaleManager.getLocale(params.getLocale());
491
492                CmsGallerySearchResult result = new CmsGallerySearchResult(
493                    document,
494                    cms,
495                    (int)document.getScore(),
496                    locale);
497
498                resultList.add(result);
499            }
500        } catch (CmsSearchException e) {
501            LOG.error(e.getMessage(), e);
502        }
503        return resultList;
504    }
505
506    /**
507     * @see org.opencms.search.CmsSearchIndex#getConfiguration()
508     */
509    @Override
510    public CmsParameterConfiguration getConfiguration() {
511
512        CmsParameterConfiguration result = super.getConfiguration();
513        if (getPostProcessor() != null) {
514            result.put(POST_PROCESSOR, getPostProcessor().getClass().getName());
515        }
516        return result;
517    }
518
519    /**
520     * Returns the name of the core of the index.
521     * NOTE: Index and core name differ since OpenCms 10.5 due to new naming rules for cores in SOLR.
522     *
523     * @return the name of the core of the index.
524     */
525    public String getCoreName() {
526
527        return m_coreName;
528    }
529
530    /**
531     * @see org.opencms.search.CmsSearchIndex#getDocument(java.lang.String, java.lang.String)
532     */
533    @Override
534    public synchronized I_CmsSearchDocument getDocument(String fieldname, String term) {
535
536        return getDocument(fieldname, term, null);
537    }
538
539    /**
540     * Version of {@link org.opencms.search.CmsSearchIndex#getDocument(java.lang.String, java.lang.String)} where
541     * the returned fields can be restricted.
542     *
543     * @param fieldname the field to query in
544     * @param term the query
545     * @param fls the returned fields.
546     * @return the document.
547     */
548    public synchronized I_CmsSearchDocument getDocument(String fieldname, String term, String[] fls) {
549
550        try {
551            SolrQuery query = new SolrQuery();
552            if (CmsSearchField.FIELD_PATH.equals(fieldname)) {
553                query.setQuery(fieldname + ":\"" + term + "\"");
554            } else {
555                query.setQuery(fieldname + ":" + term);
556            }
557            // We could have more than one document due to serial dates. We only want one arbitrary document per id/path
558            query.setRows(Integer.valueOf(1));
559            if (null != fls) {
560                query.setFields(fls);
561            }
562            QueryResponse res = m_solr.query(getCoreName(), query);
563            if (res != null) {
564                SolrDocumentList sdl = m_solr.query(getCoreName(), query).getResults();
565                if ((sdl.getNumFound() > 0L) && (sdl.get(0) != null)) {
566                    return new CmsSolrDocument(sdl.get(0));
567                }
568            }
569        } catch (Exception e) {
570            // ignore and assume that the document could not be found
571            LOG.error(e.getMessage(), e);
572        }
573        return null;
574    }
575
576    /**
577     * Returns the language locale for the given resource in this index.<p>
578     *
579     * @param cms the current OpenCms user context
580     * @param resource the resource to check
581     * @param availableLocales a list of locales supported by the resource
582     *
583     * @return the language locale for the given resource in this index
584     */
585    @Override
586    public Locale getLocaleForResource(CmsObject cms, CmsResource resource, List<Locale> availableLocales) {
587
588        Locale result = null;
589        List<Locale> defaultLocales = OpenCms.getLocaleManager().getDefaultLocales(cms, resource);
590        if ((availableLocales != null) && (availableLocales.size() > 0)) {
591            result = OpenCms.getLocaleManager().getBestMatchingLocale(
592                defaultLocales.get(0),
593                defaultLocales,
594                availableLocales);
595        }
596        if (result == null) {
597            result = ((availableLocales != null) && availableLocales.isEmpty())
598            ? availableLocales.get(0)
599            : defaultLocales.get(0);
600        }
601        return result;
602    }
603
604    /**
605     * Returns the maximal number of results (start + rows) that are processed for each search query unless another
606     * maximum is explicitly specified in {@link #search(CmsObject, CmsSolrQuery, boolean, ServletResponse, boolean, CmsResourceFilter, int)}.
607     *
608     * @return the maximal number of results (start + rows) that are processed for a search query.
609     */
610    public int getMaxProcessedResults() {
611
612        return m_maxProcessedResults;
613    }
614
615    /**
616     * Returns the search post processor.<p>
617     *
618     * @return the post processor to use
619     */
620    public I_CmsSolrPostSearchProcessor getPostProcessor() {
621
622        return m_postProcessor;
623    }
624
625    /**
626     * Returns the Solr server URL to connect to for this specific index, or <code>null</code> if no specific URL is configured.
627     * @return the Solr server URL to connect to for this specific index, or <code>null</code> if no specific URL is configured.
628     */
629    public String getServerUrl() {
630
631        return m_serverUrl;
632    }
633
634    /**
635     * @see org.opencms.search.CmsSearchIndex#initialize()
636     */
637    @Override
638    public void initialize() throws CmsSearchException {
639
640        super.initialize();
641        if (m_maxProcessedResults == -2) {
642            m_maxProcessedResults = OpenCms.getSearchManager().getSolrServerConfiguration().getMaxProcessedResults();
643        }
644        try {
645            OpenCms.getSearchManager().registerSolrIndex(this);
646        } catch (CmsConfigurationException ex) {
647            LOG.error(ex.getMessage(), ex);
648            setEnabled(false);
649        }
650    }
651
652    /** Returns a flag, indicating if the Solr server is not yet set.
653     * @return a flag, indicating if the Solr server is not yet set.
654     */
655    public boolean isNoSolrServerSet() {
656
657        return null == m_solr;
658    }
659
660    /**
661     * Not yet implemented for Solr.<p>
662     *
663     * <code>
664     * #################<br>
665     * ### DON'T USE ###<br>
666     * #################<br>
667     * </code>
668     *
669     * @deprecated Use {@link #search(CmsObject, SolrQuery)} or {@link #search(CmsObject, String)} instead
670     */
671    @Override
672    @Deprecated
673    public synchronized CmsSearchResultList search(CmsObject cms, CmsSearchParameters params) {
674
675        throw new UnsupportedOperationException();
676    }
677
678    /**
679     * Default search method.<p>
680     *
681     * @param cms the current CMS object
682     * @param query the query
683     *
684     * @return the results
685     *
686     * @throws CmsSearchException if something goes wrong
687     *
688     * @see #search(CmsObject, String)
689     */
690    public CmsSolrResultList search(CmsObject cms, CmsSolrQuery query) throws CmsSearchException {
691
692        return search(cms, query, false);
693    }
694
695    /**
696     * Performs a search.<p>
697     *
698     * Returns a list of 'OpenCms resource documents'
699     * ({@link CmsSearchResource}) encapsulated within the class  {@link CmsSolrResultList}.
700     * This list can be accessed exactly like an {@link List} which entries are
701     * {@link CmsSearchResource} that extend {@link CmsResource} and holds the Solr
702     * implementation of {@link I_CmsSearchDocument} as member. <b>This enables you to deal
703     * with the resulting list as you do with well known {@link List} and work on it's entries
704     * like you do on {@link CmsResource}.</b>
705     *
706     * <h4>What will be done with the Solr search result?</h4>
707     * <ul>
708     * <li>Although it can happen, that there are less results returned than rows were requested
709     * (imagine an index containing less documents than requested rows) we try to guarantee
710     * the requested amount of search results and to provide a working pagination with
711     * security check.</li>
712     *
713     * <li>To be sure we get enough documents left even the permission check reduces the amount
714     * of found documents, the rows are multiplied by <code>'5'</code> and the current page
715     * additionally the offset is added. The count of documents we don't have enough
716     * permissions for grows with increasing page number, that's why we also multiply
717     * the rows by the current page count.</li>
718     *
719     * <li>Also make sure we perform the permission check for all found documents, so start with
720     * the first found doc.</li>
721     * </ul>
722     *
723     * <b>NOTE:</b> If latter pages than the current one are containing protected documents the
724     * total hit count will be incorrect, because the permission check ends if we have
725     * enough results found for the page to display. With other words latter pages than
726     * the current can contain documents that will first be checked if those pages are
727     * requested to be displayed, what causes a incorrect hit count.<p>
728     *
729     * @param cms the current OpenCms context
730     * @param ignoreMaxRows <code>true</code> to return all all requested rows, <code>false</code> to use max rows
731     * @param query the OpenCms Solr query
732     *
733     * @return the list of found documents
734     *
735     * @throws CmsSearchException if something goes wrong
736     *
737     * @see org.opencms.search.solr.CmsSolrResultList
738     * @see org.opencms.search.CmsSearchResource
739     * @see org.opencms.search.I_CmsSearchDocument
740     * @see org.opencms.search.solr.CmsSolrQuery
741     */
742    public CmsSolrResultList search(CmsObject cms, final CmsSolrQuery query, boolean ignoreMaxRows)
743    throws CmsSearchException {
744
745        return search(cms, query, ignoreMaxRows, null, false, null);
746    }
747
748    /**
749     * Like {@link #search(CmsObject, CmsSolrQuery, boolean)}, but additionally a resource filter can be specified.
750     * By default, the filter depends on the index.
751     *
752     * @param cms the current OpenCms context
753     * @param ignoreMaxRows <code>true</code> to return all all requested rows, <code>false</code> to use max rows
754     * @param query the OpenCms Solr query
755     * @param filter the resource filter to use for post-processing.
756     *
757     * @return the list of documents found.
758     *
759     * @throws CmsSearchException if something goes wrong
760     */
761    public CmsSolrResultList search(
762        CmsObject cms,
763        final CmsSolrQuery query,
764        boolean ignoreMaxRows,
765        final CmsResourceFilter filter)
766    throws CmsSearchException {
767
768        return search(cms, query, ignoreMaxRows, null, false, filter);
769    }
770
771    /**
772     * Performs the actual search.<p>
773     *
774     * @param cms the current OpenCms context
775     * @param query the OpenCms Solr query
776     * @param ignoreMaxRows <code>true</code> to return all all requested rows, <code>false</code> to use max rows
777     * @param response the servlet response to write the query result to, may also be <code>null</code>
778     * @param ignoreSearchExclude if set to false, only contents with search_exclude unset or "false" will be found - typical for the the non-gallery case
779     * @param filter the resource filter to use
780     *
781     * @return the found documents
782     *
783     * @throws CmsSearchException if something goes wrong
784     *
785     * @see #search(CmsObject, CmsSolrQuery, boolean)
786     */
787    public CmsSolrResultList search(
788        CmsObject cms,
789        final CmsSolrQuery query,
790        boolean ignoreMaxRows,
791        ServletResponse response,
792        boolean ignoreSearchExclude,
793        CmsResourceFilter filter)
794    throws CmsSearchException {
795
796        return search(cms, query, ignoreMaxRows, response, ignoreSearchExclude, filter, getMaxProcessedResults());
797    }
798
799    /**
800     * Performs the actual search.<p>
801     *
802     * To provide for correct permissions two queries are performed and the response is fused from that queries:
803     * <ol>
804     *  <li>a query for permission checking, where fl, start and rows is adjusted. From this query result we take for the response:
805     *      <ul>
806     *          <li>facets</li>
807     *          <li>spellcheck</li>
808     *          <li>suggester</li>
809     *          <li>morelikethis</li>
810     *          <li>clusters</li>
811     *      </ul>
812     *  </li>
813     *  <li>a query that collects only the resources determined by the first query and performs highlighting. From this query we take for the response:
814     *      <li>result</li>
815     *      <li>highlighting</li>
816     *  </li>
817     *</ol>
818     *
819     * Currently not or only partly supported Solr features are:
820     * <ul>
821     *  <li>groups</li>
822     *  <li>collapse - representatives of the collapsed group might be filtered by the permission check</li>
823     *  <li>expand is disabled</li>
824     * </ul>
825     *
826     * @param cms the current OpenCms context
827     * @param query the OpenCms Solr query
828     * @param ignoreMaxRows <code>true</code> to return all requested rows, <code>false</code> to use max rows
829     * @param response the servlet response to write the query result to, may also be <code>null</code>
830     * @param ignoreSearchExclude if set to false, only contents with search_exclude unset or "false" will be found - typical for the the non-gallery case
831     * @param filter the resource filter to use
832     * @param maxNumResults the maximal number of results to search for
833     *
834     * @return the found documents
835     *
836     * @throws CmsSearchException if something goes wrong
837     *
838     * @see #search(CmsObject, CmsSolrQuery, boolean)
839     */
840    @SuppressWarnings("unchecked")
841    public CmsSolrResultList search(
842        CmsObject cms,
843        final CmsSolrQuery query,
844        boolean ignoreMaxRows,
845        ServletResponse response,
846        boolean ignoreSearchExclude,
847        CmsResourceFilter filter,
848        int maxNumResults)
849    throws CmsSearchException {
850
851        CmsSolrResultList result = null;
852        long startTime = System.currentTimeMillis();
853
854        // TODO:
855        // - fall back to "last found results" if none are present at the "last page"?
856        // - deal with cursorMarks?
857        // - deal with groups?
858        // - deal with result clustering?
859        // - remove max score calculation?
860
861        if (LOG.isDebugEnabled()) {
862            LOG.debug(Messages.get().getBundle().key(Messages.LOG_SOLR_DEBUG_ORIGINAL_QUERY_2, query, getName()));
863        }
864
865        // change thread priority in order to reduce search impact on overall system performance
866        int previousPriority = Thread.currentThread().getPriority();
867        if (getPriority() > 0) {
868            Thread.currentThread().setPriority(getPriority());
869        }
870
871        // check if the user is allowed to access this index
872        checkOfflineAccess(cms);
873
874        if (!ignoreSearchExclude) {
875            if (LOG.isInfoEnabled()) {
876                LOG.info(
877                    Messages.get().getBundle().key(
878                        Messages.LOG_SOLR_INFO_ADDING_SEARCH_EXCLUDE_FILTER_FOR_QUERY_2,
879                        query,
880                        getName()));
881            }
882            query.addFilterQuery(CmsSearchField.FIELD_SEARCH_EXCLUDE + ":\"false\"");
883        }
884
885        // get start parameter from the request
886        int start = null == query.getStart() ? 0 : query.getStart().intValue();
887
888        // correct negative start values to 0.
889        if (start < 0) {
890            query.setStart(Integer.valueOf(0));
891            start = 0;
892        }
893
894        // Adjust the maximal number of results to process in case it is unlimited.
895        if (maxNumResults < 0) {
896            maxNumResults = Integer.MAX_VALUE;
897            if (LOG.isInfoEnabled()) {
898                LOG.info(
899                    Messages.get().getBundle().key(
900                        Messages.LOG_SOLR_INFO_LIMITING_MAX_PROCESSED_RESULTS_3,
901                        query,
902                        getName(),
903                        Integer.valueOf(maxNumResults)));
904            }
905        }
906
907        // Correct the rows parameter
908        // Set the default rows, if rows are not set in the original query.
909        int rows = null == query.getRows() ? CmsSolrQuery.DEFAULT_ROWS.intValue() : query.getRows().intValue();
910
911        // Restrict the rows, such that the maximal number of queryable results is not exceeded.
912        if ((((rows + start) > maxNumResults) || ((rows + start) < 0))) {
913            rows = maxNumResults - start;
914        }
915        // Restrict the rows to the maximally allowed number, if they should be restricted.
916        if (!ignoreMaxRows && (rows > ROWS_MAX)) {
917            if (LOG.isInfoEnabled()) {
918                LOG.info(
919                    Messages.get().getBundle().key(
920                        Messages.LOG_SOLR_INFO_LIMITING_MAX_ROWS_4,
921                        new Object[] {query, getName(), Integer.valueOf(rows), Integer.valueOf(ROWS_MAX)}));
922            }
923            rows = ROWS_MAX;
924        }
925        // If start is higher than maxNumResults, the rows could be negative here - correct this.
926        if (rows < 0) {
927            if (LOG.isInfoEnabled()) {
928                LOG.info(
929                    Messages.get().getBundle().key(
930                        Messages.LOG_SOLR_INFO_CORRECTING_ROWS_4,
931                        new Object[] {query, getName(), Integer.valueOf(rows), Integer.valueOf(0)}));
932            }
933            rows = 0;
934        }
935        // Set the corrected rows for the query.
936        query.setRows(Integer.valueOf(rows));
937
938        // remove potentially set expand parameter
939        if (null != query.getParams(QUERY_PARAM_EXPAND)) {
940            LOG.info(Messages.get().getBundle().key(Messages.LOG_SOLR_INFO_REMOVING_EXPAND_2, query, getName()));
941            query.remove("expand");
942        }
943
944        float maxScore = 0;
945
946        LocalSolrQueryRequest solrQueryRequest = null;
947        SolrCore core = null;
948        String[] sortParamValues = query.getParams(QUERY_SORT_NAME);
949        boolean sortByScoreDesc = (null == sortParamValues)
950            || (sortParamValues.length == 0)
951            || Objects.equal(sortParamValues[0], "score desc");
952
953        try {
954
955            // initialize the search context
956            CmsObject searchCms = OpenCms.initCmsObject(cms);
957
958            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
959            //////////////////////// QUERY FOR PERMISSION CHECK, FACETS, SPELLCHECK, SUGGESTIONS ///////////////////////////
960            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
961
962            // Clone the query and keep the original one
963            CmsSolrQuery checkQuery = query.clone();
964            // Initialize rows, offset, end and the current page.
965            int end = start + rows;
966            int itemsToCheck = 0 == end ? 0 : Math.max(10, end + (end / 5)); // request 20 percent more, but at least 10 results if permissions are filtered
967            // use a set to prevent double entries if multiple check queries are performed.
968            Set<String> resultSolrIds = new HashSet<>(rows); // rows are set before definitely.
969
970            // counter for the documents found and accessible
971            int cnt = 0;
972            long hitCount = 0;
973            long visibleHitCount = 0;
974            int processedResults = 0;
975            long solrPermissionTime = 0;
976            // disable highlighting - it's done in the next query.
977            checkQuery.setHighlight(false);
978            // adjust rows and start for the permission check.
979            checkQuery.setRows(Integer.valueOf(Math.min(maxNumResults - processedResults, itemsToCheck)));
980            checkQuery.setStart(Integer.valueOf(processedResults));
981            // return only the fields required for the permission check and for scoring
982            checkQuery.setFields(CmsSearchField.FIELD_TYPE, CmsSearchField.FIELD_SOLR_ID, CmsSearchField.FIELD_PATH);
983            List<String> originalFields = Arrays.asList(query.getFields().split(","));
984            if (originalFields.contains(CmsSearchField.FIELD_SCORE)) {
985                checkQuery.addField(CmsSearchField.FIELD_SCORE);
986            }
987            if (LOG.isDebugEnabled()) {
988                LOG.debug(Messages.get().getBundle().key(Messages.LOG_SOLR_DEBUG_CHECK_QUERY_2, checkQuery, getName()));
989            }
990            // perform the permission check Solr query and remember the response and time Solr took.
991            long solrCheckTime = System.currentTimeMillis();
992            QueryResponse checkQueryResponse = m_solr.query(getCoreName(), checkQuery);
993            solrCheckTime = System.currentTimeMillis() - solrCheckTime;
994            solrPermissionTime += solrCheckTime;
995
996            // initialize the counts
997            hitCount = checkQueryResponse.getResults().getNumFound();
998            int maxToProcess = Long.valueOf(Math.min(hitCount, maxNumResults)).intValue();
999            visibleHitCount = hitCount;
1000
1001            // process found documents
1002            for (SolrDocument doc : checkQueryResponse.getResults()) {
1003                try {
1004                    CmsSolrDocument searchDoc = new CmsSolrDocument(doc);
1005                    if (needsPermissionCheck(searchDoc) && !hasPermissions(searchCms, searchDoc, filter)) {
1006                        visibleHitCount--;
1007                    } else {
1008                        if (cnt >= start) {
1009                            resultSolrIds.add(searchDoc.getFieldValueAsString(CmsSearchField.FIELD_SOLR_ID));
1010                        }
1011                        if (sortByScoreDesc && (searchDoc.getScore() > maxScore)) {
1012                            maxScore = searchDoc.getScore();
1013                        }
1014                        if (++cnt >= end) {
1015                            break;
1016                        }
1017                    }
1018                } catch (Exception e) {
1019                    // should not happen, but if it does we want to go on with the next result nevertheless
1020                    visibleHitCount--;
1021                    LOG.warn(Messages.get().getBundle().key(Messages.LOG_SOLR_ERR_RESULT_ITERATION_FAILED_0), e);
1022                }
1023            }
1024            processedResults += checkQueryResponse.getResults().size();
1025
1026            if ((resultSolrIds.size() < rows) && (processedResults < maxToProcess)) {
1027                CmsSolrQuery secondCheckQuery = checkQuery.clone();
1028                // disable all features not necessary, since results are present from the first check query.
1029                secondCheckQuery.setFacet(false);
1030                secondCheckQuery.setMoreLikeThis(false);
1031                secondCheckQuery.set(QUERY_SPELLCHECK_NAME, false);
1032                do {
1033                    // query directly more under certain conditions to reduce number of queries
1034                    itemsToCheck = itemsToCheck < 3000 ? itemsToCheck * 4 : itemsToCheck;
1035                    // adjust rows and start for the permission check.
1036                    secondCheckQuery.setRows(
1037                        Integer.valueOf(
1038                            Long.valueOf(Math.min(maxToProcess - processedResults, itemsToCheck)).intValue()));
1039                    secondCheckQuery.setStart(Integer.valueOf(processedResults));
1040
1041                    if (LOG.isDebugEnabled()) {
1042                        LOG.debug(
1043                            Messages.get().getBundle().key(
1044                                Messages.LOG_SOLR_DEBUG_SECONDCHECK_QUERY_2,
1045                                secondCheckQuery,
1046                                getName()));
1047                    }
1048
1049                    long solrSecondCheckTime = System.currentTimeMillis();
1050                    QueryResponse secondCheckQueryResponse = m_solr.query(getCoreName(), secondCheckQuery);
1051                    processedResults += secondCheckQueryResponse.getResults().size();
1052                    solrSecondCheckTime = System.currentTimeMillis() - solrSecondCheckTime;
1053                    solrPermissionTime += solrCheckTime;
1054
1055                    // process found documents
1056                    for (SolrDocument doc : secondCheckQueryResponse.getResults()) {
1057                        try {
1058                            CmsSolrDocument searchDoc = new CmsSolrDocument(doc);
1059                            String docSolrId = searchDoc.getFieldValueAsString(CmsSearchField.FIELD_SOLR_ID);
1060                            if ((needsPermissionCheck(searchDoc) && !hasPermissions(searchCms, searchDoc, filter))
1061                                || resultSolrIds.contains(docSolrId)) {
1062                                visibleHitCount--;
1063                            } else {
1064                                if (cnt >= start) {
1065                                    resultSolrIds.add(docSolrId);
1066                                }
1067                                if (sortByScoreDesc && (searchDoc.getScore() > maxScore)) {
1068                                    maxScore = searchDoc.getScore();
1069                                }
1070                                if (++cnt >= end) {
1071                                    break;
1072                                }
1073                            }
1074                        } catch (Exception e) {
1075                            // should not happen, but if it does we want to go on with the next result nevertheless
1076                            visibleHitCount--;
1077                            LOG.warn(
1078                                Messages.get().getBundle().key(Messages.LOG_SOLR_ERR_RESULT_ITERATION_FAILED_0),
1079                                e);
1080                        }
1081                    }
1082
1083                } while ((resultSolrIds.size() < rows) && (processedResults < maxToProcess));
1084            }
1085
1086            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
1087            //////////////////////// QUERY FOR RESULTS AND HIGHLIGHTING ////////////////////////////////////////////////////
1088            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
1089
1090            // the lists storing the found documents that will be returned
1091            List<CmsSearchResource> resourceDocumentList = new ArrayList<CmsSearchResource>(resultSolrIds.size());
1092            SolrDocumentList solrDocumentList = new SolrDocumentList();
1093
1094            long solrResultTime = 0;
1095
1096            // If we're using a post-processor, (re-)initialize it before using it
1097            if (m_postProcessor != null) {
1098                m_postProcessor.init();
1099            }
1100
1101            // build the query for getting the results
1102            SolrQuery queryForResults = query.clone();
1103            // we add an additional filter, such that we can only find the documents we want to retrieve, as we figured out in the check query.
1104            if (!resultSolrIds.isEmpty()) {
1105                Optional<String> queryFilterString = resultSolrIds.stream().map(a -> '"' + a + '"').reduce(
1106                    (a, b) -> a + " OR " + b);
1107                queryForResults.addFilterQuery(CmsSearchField.FIELD_SOLR_ID + ":(" + queryFilterString.get() + ")");
1108            }
1109            queryForResults.setRows(Integer.valueOf(resultSolrIds.size()));
1110            queryForResults.setStart(Integer.valueOf(0));
1111
1112            if (LOG.isDebugEnabled()) {
1113                LOG.debug(
1114                    Messages.get().getBundle().key(Messages.LOG_SOLR_DEBUG_RESULT_QUERY_2, queryForResults, getName()));
1115            }
1116            // perform the result query.
1117            solrResultTime = System.currentTimeMillis();
1118            QueryResponse resultQueryResponse = m_solr.query(getCoreName(), queryForResults);
1119            solrResultTime = System.currentTimeMillis() - solrResultTime;
1120
1121            // List containing solr ids of filtered contents for which highlighting has to be removed.
1122            // Since we checked permissions just a few milliseconds ago, this should typically stay empty.
1123            List<String> filteredResultIds = new ArrayList<>(5);
1124
1125            for (SolrDocument doc : resultQueryResponse.getResults()) {
1126                try {
1127                    CmsSolrDocument searchDoc = new CmsSolrDocument(doc);
1128                    if (needsPermissionCheck(searchDoc)) {
1129                        CmsResource resource = filter == null
1130                        ? getResource(searchCms, searchDoc)
1131                        : getResource(searchCms, searchDoc, filter);
1132                        if (null != resource) {
1133                            if (m_postProcessor != null) {
1134                                doc = m_postProcessor.process(
1135                                    searchCms,
1136                                    resource,
1137                                    (SolrInputDocument)searchDoc.getDocument());
1138                            }
1139                            resourceDocumentList.add(new CmsSearchResource(resource, searchDoc));
1140                            solrDocumentList.add(doc);
1141                        } else {
1142                            filteredResultIds.add(searchDoc.getFieldValueAsString(CmsSearchField.FIELD_SOLR_ID));
1143                        }
1144                    } else { // should not happen unless the index has changed since the first query.
1145                        resourceDocumentList.add(new CmsSearchResource(PSEUDO_RES, searchDoc));
1146                        solrDocumentList.add(doc);
1147                        visibleHitCount--;
1148                    }
1149                } catch (Exception e) {
1150                    // should not happen, but if it does we want to go on with the next result nevertheless
1151                    visibleHitCount--;
1152                    LOG.warn(Messages.get().getBundle().key(Messages.LOG_SOLR_ERR_RESULT_ITERATION_FAILED_0), e);
1153                }
1154            }
1155
1156            long processTime = System.currentTimeMillis() - startTime - solrPermissionTime - solrResultTime;
1157
1158            ////////////////////////////////////////////////////////////////////////////////////////////////////////////
1159            //////////////////////// CREATE THE FINAL RESPONSE /////////////////////////////////////////////////////////
1160            ////////////////////////////////////////////////////////////////////////////////////////////////////////////
1161
1162            // we are manipulating the checkQueryResponse to set up the final response, we want to deliver.
1163
1164            // adjust start, max score and hit count displayed in the result list.
1165            solrDocumentList.setStart(start);
1166            Float finalMaxScore = sortByScoreDesc ? new Float(maxScore) : checkQueryResponse.getResults().getMaxScore();
1167            solrDocumentList.setMaxScore(finalMaxScore);
1168            solrDocumentList.setNumFound(visibleHitCount);
1169
1170            // Exchange the search parameters in the response header by the ones from the (adjusted) original query.
1171            NamedList<Object> params = ((NamedList<Object>)(checkQueryResponse.getHeader().get(HEADER_PARAMS_NAME)));
1172            params.clear();
1173            for (String paramName : query.getParameterNames()) {
1174                params.add(paramName, query.get(paramName));
1175            }
1176
1177            // Fill in the documents to return.
1178            checkQueryResponse.getResponse().setVal(
1179                checkQueryResponse.getResponse().indexOf(QUERY_RESPONSE_NAME, 0),
1180                solrDocumentList);
1181
1182            // Fill in the time, the overall query took, including processing and permission check.
1183            checkQueryResponse.getResponseHeader().setVal(
1184                checkQueryResponse.getResponseHeader().indexOf(QUERY_TIME_NAME, 0),
1185                new Integer(new Long(System.currentTimeMillis() - startTime).intValue()));
1186
1187            // Fill in the highlighting information from the result query.
1188            if (query.getHighlight()) {
1189                NamedList<Object> highlighting = (NamedList<Object>)resultQueryResponse.getResponse().get(
1190                    QUERY_HIGHLIGHTING_NAME);
1191                // filter out highlighting for documents where access is not permitted.
1192                for (String filteredId : filteredResultIds) {
1193                    highlighting.remove(filteredId);
1194                }
1195                NamedList<Object> completeResponse = new SimpleOrderedMap<Object>(1);
1196                completeResponse.addAll(checkQueryResponse.getResponse());
1197                completeResponse.add(QUERY_HIGHLIGHTING_NAME, highlighting);
1198                checkQueryResponse.setResponse(completeResponse);
1199            }
1200
1201            // build the result
1202            result = new CmsSolrResultList(
1203                query,
1204                checkQueryResponse,
1205                solrDocumentList,
1206                resourceDocumentList,
1207                start,
1208                new Integer(rows),
1209                Math.min(end, (start + solrDocumentList.size())),
1210                rows > 0 ? (start / rows) + 1 : 0, //page - but matches only in case of equally sized pages and is zero for rows=0 (because this was this way before!?!)
1211                visibleHitCount,
1212                finalMaxScore,
1213                startTime,
1214                System.currentTimeMillis());
1215            if (LOG.isDebugEnabled()) {
1216                Object[] logParams = new Object[] {
1217                    new Long(System.currentTimeMillis() - startTime),
1218                    new Long(result.getNumFound()),
1219                    new Long(solrPermissionTime + solrResultTime),
1220                    new Long(processTime),
1221                    new Long(result.getHighlightEndTime() != 0 ? result.getHighlightEndTime() - startTime : 0)};
1222                LOG.debug(
1223                    query.toString()
1224                        + "\n"
1225                        + Messages.get().getBundle().key(Messages.LOG_SOLR_SEARCH_EXECUTED_5, logParams));
1226            }
1227            // write the response for the handler
1228            if (response != null) {
1229                // create and return the result
1230                core = m_solr instanceof EmbeddedSolrServer
1231                ? ((EmbeddedSolrServer)m_solr).getCoreContainer().getCore(getCoreName())
1232                : null;
1233
1234                solrQueryRequest = new LocalSolrQueryRequest(core, query);
1235                SolrQueryResponse solrQueryResponse = new SolrQueryResponse();
1236                solrQueryResponse.setAllValues(checkQueryResponse.getResponse());
1237                writeResp(response, solrQueryRequest, solrQueryResponse);
1238            }
1239        } catch (
1240
1241        Exception e) {
1242            throw new CmsSearchException(
1243                Messages.get().container(
1244                    Messages.LOG_SOLR_ERR_SEARCH_EXECUTION_FAILD_1,
1245                    CmsEncoder.decode(query.toString()),
1246                    e),
1247                e);
1248        } finally {
1249            if (solrQueryRequest != null) {
1250                solrQueryRequest.close();
1251            }
1252            if (null != core) {
1253                core.close();
1254            }
1255            // re-set thread to previous priority
1256            Thread.currentThread().setPriority(previousPriority);
1257        }
1258        return result;
1259    }
1260
1261    /**
1262     * Default search method.<p>
1263     *
1264     * @param cms the current CMS object
1265     * @param query the query
1266     *
1267     * @return the results
1268     *
1269     * @throws CmsSearchException if something goes wrong
1270     *
1271     * @see #search(CmsObject, String)
1272     */
1273    public CmsSolrResultList search(CmsObject cms, SolrQuery query) throws CmsSearchException {
1274
1275        return search(cms, CmsEncoder.decode(query.toString()));
1276    }
1277
1278    /**
1279     * Performs a search.<p>
1280     *
1281     * @param cms the cms object
1282     * @param solrQuery the Solr query
1283     *
1284     * @return a list of documents
1285     *
1286     * @throws CmsSearchException if something goes wrong
1287     *
1288     * @see #search(CmsObject, CmsSolrQuery, boolean)
1289     */
1290    public CmsSolrResultList search(CmsObject cms, String solrQuery) throws CmsSearchException {
1291
1292        return search(cms, new CmsSolrQuery(null, CmsRequestUtil.createParameterMap(solrQuery)), false);
1293    }
1294
1295    /**
1296     * Writes the response into the writer.<p>
1297     *
1298     * NOTE: Currently not available for HTTP server.<p>
1299     *
1300     * @param response the servlet response
1301     * @param cms the CMS object to use for search
1302     * @param query the Solr query
1303     * @param ignoreMaxRows if to return unlimited results
1304     *
1305     * @throws Exception if there is no embedded server
1306     */
1307    public void select(ServletResponse response, CmsObject cms, CmsSolrQuery query, boolean ignoreMaxRows)
1308    throws Exception {
1309
1310        throwExceptionIfSafetyRestrictionsAreViolated(cms, query, false);
1311        boolean isOnline = cms.getRequestContext().getCurrentProject().isOnlineProject();
1312        CmsResourceFilter filter = isOnline ? null : CmsResourceFilter.IGNORE_EXPIRATION;
1313
1314        search(cms, query, ignoreMaxRows, response, false, filter);
1315    }
1316
1317    /**
1318     * Sets the logical key/name of this search index.<p>
1319     *
1320     * @param name the logical key/name of this search index
1321     *
1322     * @throws CmsIllegalArgumentException if the given name is null, empty or already taken by another search index
1323     */
1324    @Override
1325    public void setName(String name) throws CmsIllegalArgumentException {
1326
1327        super.setName(name);
1328        updateCoreName();
1329    }
1330
1331    /**
1332     * Sets the search post processor.<p>
1333     *
1334     * @param postProcessor the search post processor to set
1335     */
1336    public void setPostProcessor(I_CmsSolrPostSearchProcessor postProcessor) {
1337
1338        m_postProcessor = postProcessor;
1339    }
1340
1341    /**
1342     * Sets the Solr server used by this index.<p>
1343     *
1344     * @param client the server to set
1345     */
1346    public void setSolrServer(SolrClient client) {
1347
1348        m_solr = client;
1349    }
1350
1351    /**
1352     * Executes a spell checking Solr query and returns the Solr query response.<p>
1353     *
1354     * @param res the servlet response
1355     * @param cms the CMS object
1356     * @param q the query
1357     *
1358     * @throws CmsSearchException if something goes wrong
1359     */
1360    public void spellCheck(ServletResponse res, CmsObject cms, CmsSolrQuery q) throws CmsSearchException {
1361
1362        throwExceptionIfSafetyRestrictionsAreViolated(cms, q, true);
1363        SolrCore core = null;
1364        LocalSolrQueryRequest solrQueryRequest = null;
1365        try {
1366            q.setRequestHandler("/spell");
1367            q.setRows(Integer.valueOf(0));
1368
1369            QueryResponse queryResponse = m_solr.query(getCoreName(), q);
1370
1371            List<CmsSearchResource> resourceDocumentList = new ArrayList<CmsSearchResource>();
1372            SolrDocumentList solrDocumentList = new SolrDocumentList();
1373            if (m_postProcessor != null) {
1374                for (int i = 0; (i < queryResponse.getResults().size()); i++) {
1375                    try {
1376                        SolrDocument doc = queryResponse.getResults().get(i);
1377                        CmsSolrDocument searchDoc = new CmsSolrDocument(doc);
1378                        if (needsPermissionCheck(searchDoc)) {
1379                            // only if the document is an OpenCms internal resource perform the permission check
1380                            CmsResource resource = getResource(cms, searchDoc);
1381                            if (resource != null) {
1382                                // permission check performed successfully: the user has read permissions!
1383                                if (m_postProcessor != null) {
1384                                    doc = m_postProcessor.process(
1385                                        cms,
1386                                        resource,
1387                                        (SolrInputDocument)searchDoc.getDocument());
1388                                }
1389                                resourceDocumentList.add(new CmsSearchResource(resource, searchDoc));
1390                                solrDocumentList.add(doc);
1391                            }
1392                        }
1393                    } catch (Exception e) {
1394                        // should not happen, but if it does we want to go on with the next result nevertheless
1395                        LOG.warn(Messages.get().getBundle().key(Messages.LOG_SOLR_ERR_RESULT_ITERATION_FAILED_0), e);
1396                    }
1397                }
1398                queryResponse.getResponse().setVal(
1399                    queryResponse.getResponse().indexOf(QUERY_RESPONSE_NAME, 0),
1400                    solrDocumentList);
1401            }
1402
1403            // create and return the result
1404            core = m_solr instanceof EmbeddedSolrServer
1405            ? ((EmbeddedSolrServer)m_solr).getCoreContainer().getCore(getCoreName())
1406            : null;
1407
1408            SolrQueryResponse solrQueryResponse = new SolrQueryResponse();
1409            solrQueryResponse.setAllValues(queryResponse.getResponse());
1410
1411            // create and initialize the solr request
1412            solrQueryRequest = new LocalSolrQueryRequest(core, solrQueryResponse.getResponseHeader());
1413            // set the OpenCms Solr query as parameters to the request
1414            solrQueryRequest.setParams(q);
1415
1416            writeResp(res, solrQueryRequest, solrQueryResponse);
1417
1418        } catch (Exception e) {
1419            throw new CmsSearchException(
1420                Messages.get().container(Messages.LOG_SOLR_ERR_SEARCH_EXECUTION_FAILD_1, q),
1421                e);
1422        } finally {
1423            if (solrQueryRequest != null) {
1424                solrQueryRequest.close();
1425            }
1426            if (core != null) {
1427                core.close();
1428            }
1429        }
1430    }
1431
1432    /**
1433     * @see org.opencms.search.CmsSearchIndex#createIndexBackup()
1434     */
1435    @Override
1436    protected String createIndexBackup() {
1437
1438        if (!isBackupReindexing()) {
1439            // if no backup is generated we don't need to do anything
1440            return null;
1441        }
1442        if (m_solr instanceof EmbeddedSolrServer) {
1443            EmbeddedSolrServer ser = (EmbeddedSolrServer)m_solr;
1444            CoreContainer con = ser.getCoreContainer();
1445            SolrCore core = con.getCore(getCoreName());
1446            if (core != null) {
1447                try {
1448                    SolrRequestHandler h = core.getRequestHandler("/replication");
1449                    if (h instanceof ReplicationHandler) {
1450                        h.handleRequest(
1451                            new LocalSolrQueryRequest(core, CmsRequestUtil.createParameterMap("?command=backup")),
1452                            new SolrQueryResponse());
1453                    }
1454                } finally {
1455                    core.close();
1456                }
1457            }
1458        }
1459        return null;
1460    }
1461
1462    /**
1463     * Check, if the current user has permissions on the document's resource.
1464     * @param cms the context
1465     * @param doc the solr document (from the search result)
1466     * @param filter the resource filter to use for checking permissions
1467     * @return <code>true</code> iff the resource mirrored by the search result can be read by the current user.
1468     */
1469    protected boolean hasPermissions(CmsObject cms, CmsSolrDocument doc, CmsResourceFilter filter) {
1470
1471        return null != (filter == null ? getResource(cms, doc) : getResource(cms, doc, filter));
1472    }
1473
1474    /**
1475     * @see org.opencms.search.CmsSearchIndex#indexSearcherClose()
1476     */
1477    @SuppressWarnings("sync-override")
1478    @Override
1479    protected void indexSearcherClose() {
1480
1481        // nothing to do here
1482    }
1483
1484    /**
1485     * @see org.opencms.search.CmsSearchIndex#indexSearcherOpen(java.lang.String)
1486     */
1487    @SuppressWarnings("sync-override")
1488    @Override
1489    protected void indexSearcherOpen(final String path) {
1490
1491        // nothing to do here
1492    }
1493
1494    /**
1495     * @see org.opencms.search.CmsSearchIndex#indexSearcherUpdate()
1496     */
1497    @SuppressWarnings("sync-override")
1498    @Override
1499    protected void indexSearcherUpdate() {
1500
1501        // nothing to do here
1502    }
1503
1504    /**
1505     * Checks if the current user is allowed to access non-online indexes.<p>
1506     *
1507     * To access non-online indexes the current user must be a workplace user at least.<p>
1508     *
1509     * @param cms the CMS object initialized with the current request context / user
1510     *
1511     * @throws CmsSearchException thrown if the access is not permitted
1512     */
1513    private void checkOfflineAccess(CmsObject cms) throws CmsSearchException {
1514
1515        // If an offline index is being selected, check permissions
1516        if (!CmsProject.ONLINE_PROJECT_NAME.equals(getProject())) {
1517            // only if the user has the role Workplace user, he is allowed to access the Offline index
1518            try {
1519                OpenCms.getRoleManager().checkRole(cms, CmsRole.ELEMENT_AUTHOR);
1520            } catch (CmsRoleViolationException e) {
1521                throw new CmsSearchException(
1522                    Messages.get().container(
1523                        Messages.LOG_SOLR_ERR_SEARCH_PERMISSION_VIOLATION_2,
1524                        getName(),
1525                        cms.getRequestContext().getCurrentUser()),
1526                    e);
1527            }
1528        }
1529    }
1530
1531    /**
1532     * Generates a valid core name from the provided name (the index name).
1533     * @param name the index name.
1534     * @return the core name
1535     */
1536    private String generateCoreName(final String name) {
1537
1538        if (name != null) {
1539            return name.replace(" ", "-");
1540        }
1541        return null;
1542    }
1543
1544    /**
1545     * Checks if the query should be executed using the debug mode where the security restrictions do not apply.
1546     * @param cms the current context.
1547     * @param query the query to execute.
1548     * @return a flag, indicating, if the query should be performed in debug mode.
1549     */
1550    private boolean isDebug(CmsObject cms, CmsSolrQuery query) {
1551
1552        String[] debugSecretValues = query.remove(REQUEST_PARAM_DEBUG_SECRET);
1553        String debugSecret = (debugSecretValues == null) || (debugSecretValues.length < 1)
1554        ? null
1555        : debugSecretValues[0];
1556        if ((null != debugSecret) && !debugSecret.trim().isEmpty() && (null != m_handlerDebugSecretFile)) {
1557            try {
1558                CmsFile secretFile = cms.readFile(m_handlerDebugSecretFile);
1559                String secret = new String(secretFile.getContents(), CmsFileUtil.getEncoding(cms, secretFile));
1560                return secret.trim().equals(debugSecret.trim());
1561            } catch (Exception e) {
1562                LOG.info(
1563                    "Failed to read secret file for index \""
1564                        + getName()
1565                        + "\" at path \""
1566                        + m_handlerDebugSecretFile
1567                        + "\".");
1568            }
1569        }
1570        return false;
1571    }
1572
1573    /**
1574     * Throws an exception if the request can for security reasons not be performed.
1575     * Security restrictions can be set via parameters of the index.
1576     *
1577     * @param cms the current context.
1578     * @param query the query.
1579     * @param isSpell flag, indicating if the spellcheck handler is requested.
1580     * @throws CmsSearchException thrown if the query cannot be executed due to security reasons.
1581     */
1582    private void throwExceptionIfSafetyRestrictionsAreViolated(CmsObject cms, CmsSolrQuery query, boolean isSpell)
1583    throws CmsSearchException {
1584
1585        if (!isDebug(cms, query)) {
1586            if (isSpell) {
1587                if (m_handlerSpellDisabled) {
1588                    throw new CmsSearchException(Messages.get().container(Messages.GUI_HANDLER_REQUEST_NOT_ALLOWED_0));
1589                }
1590            } else {
1591                if (m_handlerSelectDisabled) {
1592                    throw new CmsSearchException(Messages.get().container(Messages.GUI_HANDLER_REQUEST_NOT_ALLOWED_0));
1593                }
1594                int start = null != query.getStart() ? query.getStart().intValue() : 0;
1595                int rows = null != query.getRows() ? query.getRows().intValue() : CmsSolrQuery.DEFAULT_ROWS.intValue();
1596                if ((m_handlerMaxAllowedResultsAtAll >= 0) && ((rows + start) > m_handlerMaxAllowedResultsAtAll)) {
1597                    throw new CmsSearchException(
1598                        Messages.get().container(
1599                            Messages.GUI_HANDLER_TOO_MANY_RESULTS_REQUESTED_AT_ALL_2,
1600                            Integer.valueOf(m_handlerMaxAllowedResultsAtAll),
1601                            Integer.valueOf(rows + start)));
1602                }
1603                if ((m_handlerMaxAllowedResultsPerPage >= 0) && (rows > m_handlerMaxAllowedResultsPerPage)) {
1604                    throw new CmsSearchException(
1605                        Messages.get().container(
1606                            Messages.GUI_HANDLER_TOO_MANY_RESULTS_REQUESTED_PER_PAGE_2,
1607                            Integer.valueOf(m_handlerMaxAllowedResultsPerPage),
1608                            Integer.valueOf(rows)));
1609                }
1610                if ((null != m_handlerAllowedFields) && (Stream.of(m_handlerAllowedFields).anyMatch(x -> true))) {
1611                    if (query.getFields().equals(CmsSolrQuery.ALL_RETURN_FIELDS)) {
1612                        query.setFields(m_handlerAllowedFields);
1613                    } else {
1614                        for (String requestedField : query.getFields().split(",")) {
1615                            if (Stream.of(m_handlerAllowedFields).noneMatch(
1616                                allowedField -> allowedField.equals(requestedField))) {
1617                                throw new CmsSearchException(
1618                                    Messages.get().container(
1619                                        Messages.GUI_HANDLER_REQUESTED_FIELD_NOT_ALLOWED_2,
1620                                        requestedField,
1621                                        Stream.of(m_handlerAllowedFields).reduce("", (a, b) -> a + "," + b)));
1622                            }
1623                        }
1624                    }
1625                }
1626            }
1627        }
1628    }
1629
1630    /**
1631     * Updates the core name to be in sync with the index name.
1632     */
1633    private void updateCoreName() {
1634
1635        m_coreName = generateCoreName(getName());
1636
1637    }
1638
1639    /**
1640     * Writes the Solr response.<p>
1641     *
1642     * @param response the servlet response
1643     * @param queryRequest the Solr request
1644     * @param queryResponse the Solr response to write
1645     *
1646     * @throws IOException if sth. goes wrong
1647     * @throws UnsupportedEncodingException if sth. goes wrong
1648     */
1649    private void writeResp(ServletResponse response, SolrQueryRequest queryRequest, SolrQueryResponse queryResponse)
1650    throws IOException, UnsupportedEncodingException {
1651
1652        if (m_solr instanceof EmbeddedSolrServer) {
1653            SolrCore core = ((EmbeddedSolrServer)m_solr).getCoreContainer().getCore(getCoreName());
1654            Writer out = null;
1655            try {
1656                QueryResponseWriter responseWriter = core.getQueryResponseWriter(queryRequest);
1657
1658                final String ct = responseWriter.getContentType(queryRequest, queryResponse);
1659                if (null != ct) {
1660                    response.setContentType(ct);
1661                }
1662
1663                if (responseWriter instanceof BinaryQueryResponseWriter) {
1664                    BinaryQueryResponseWriter binWriter = (BinaryQueryResponseWriter)responseWriter;
1665                    binWriter.write(response.getOutputStream(), queryRequest, queryResponse);
1666                } else {
1667                    String charset = ContentStreamBase.getCharsetFromContentType(ct);
1668                    out = ((charset == null) || charset.equalsIgnoreCase(UTF8.toString()))
1669                    ? new OutputStreamWriter(response.getOutputStream(), UTF8)
1670                    : new OutputStreamWriter(response.getOutputStream(), charset);
1671                    out = new FastWriter(out);
1672                    responseWriter.write(out, queryRequest, queryResponse);
1673                    out.flush();
1674                }
1675            } finally {
1676                core.close();
1677                if (out != null) {
1678                    out.close();
1679                }
1680            }
1681        } else {
1682            throw new UnsupportedOperationException();
1683        }
1684    }
1685}