001/*
002 * This library is part of OpenCms -
003 * the Open Source Content Management System
004 *
005 * Copyright (c) Alkacon Software GmbH & Co. KG (http://www.alkacon.com)
006 *
007 * This library is free software; you can redistribute it and/or
008 * modify it under the terms of the GNU Lesser General Public
009 * License as published by the Free Software Foundation; either
010 * version 2.1 of the License, or (at your option) any later version.
011 *
012 * This library is distributed in the hope that it will be useful,
013 * but WITHOUT ANY WARRANTY; without even the implied warranty of
014 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
015 * Lesser General Public License for more details.
016 *
017 * For further information about Alkacon Software GmbH & Co. KG, please see the
018 * company website: http://www.alkacon.com
019 *
020 * For further information about OpenCms, please see the
021 * project website: http://www.opencms.org
022 *
023 * You should have received a copy of the GNU Lesser General Public
024 * License along with this library; if not, write to the Free Software
025 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
026 */
027
028package org.opencms.search;
029
030import org.opencms.ade.containerpage.CmsDetailOnlyContainerUtil;
031import org.opencms.configuration.CmsConfigurationException;
032import org.opencms.db.CmsDriverManager;
033import org.opencms.db.CmsPublishedResource;
034import org.opencms.db.CmsResourceState;
035import org.opencms.file.CmsObject;
036import org.opencms.file.CmsProject;
037import org.opencms.file.CmsResource;
038import org.opencms.file.CmsResourceFilter;
039import org.opencms.file.types.CmsResourceTypeXmlContainerPage;
040import org.opencms.file.types.CmsResourceTypeXmlContent;
041import org.opencms.i18n.CmsMessageContainer;
042import org.opencms.loader.CmsLoaderException;
043import org.opencms.main.CmsEvent;
044import org.opencms.main.CmsException;
045import org.opencms.main.CmsIllegalArgumentException;
046import org.opencms.main.CmsIllegalStateException;
047import org.opencms.main.CmsLog;
048import org.opencms.main.I_CmsEventListener;
049import org.opencms.main.OpenCms;
050import org.opencms.main.OpenCmsSolrHandler;
051import org.opencms.relations.CmsRelation;
052import org.opencms.relations.CmsRelationFilter;
053import org.opencms.relations.CmsRelationType;
054import org.opencms.report.CmsLogReport;
055import org.opencms.report.I_CmsReport;
056import org.opencms.scheduler.I_CmsScheduledJob;
057import org.opencms.search.documents.A_CmsVfsDocument;
058import org.opencms.search.documents.CmsExtractionResultCache;
059import org.opencms.search.documents.I_CmsDocumentFactory;
060import org.opencms.search.documents.I_CmsTermHighlighter;
061import org.opencms.search.fields.CmsLuceneField;
062import org.opencms.search.fields.CmsLuceneFieldConfiguration;
063import org.opencms.search.fields.CmsSearchField;
064import org.opencms.search.fields.CmsSearchFieldConfiguration;
065import org.opencms.search.fields.CmsSearchFieldMapping;
066import org.opencms.search.fields.I_CmsSearchFieldConfiguration;
067import org.opencms.search.solr.CmsSolrConfiguration;
068import org.opencms.search.solr.CmsSolrFieldConfiguration;
069import org.opencms.search.solr.CmsSolrIndex;
070import org.opencms.search.solr.I_CmsSolrIndexWriter;
071import org.opencms.search.solr.spellchecking.CmsSolrSpellchecker;
072import org.opencms.search.solr.spellchecking.CmsSpellcheckDictionaryIndexer;
073import org.opencms.security.CmsRole;
074import org.opencms.security.CmsRoleViolationException;
075import org.opencms.util.A_CmsModeStringEnumeration;
076import org.opencms.util.CmsFileUtil;
077import org.opencms.util.CmsStringUtil;
078import org.opencms.util.CmsUUID;
079import org.opencms.util.CmsWaitHandle;
080
081import java.io.File;
082import java.io.IOException;
083import java.nio.file.FileSystems;
084import java.nio.file.Paths;
085import java.util.ArrayList;
086import java.util.Collection;
087import java.util.Collections;
088import java.util.HashMap;
089import java.util.HashSet;
090import java.util.Iterator;
091import java.util.List;
092import java.util.Locale;
093import java.util.Map;
094import java.util.Set;
095import java.util.TreeMap;
096import java.util.concurrent.locks.ReentrantLock;
097
098import org.apache.commons.logging.Log;
099import org.apache.lucene.analysis.Analyzer;
100import org.apache.lucene.analysis.CharArraySet;
101import org.apache.lucene.analysis.standard.StandardAnalyzer;
102import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
103import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder;
104import org.apache.solr.core.CoreContainer;
105import org.apache.solr.core.CoreDescriptor;
106import org.apache.solr.core.SolrCore;
107
108/**
109 * Implements the general management and configuration of the search and
110 * indexing facilities in OpenCms.<p>
111 *
112 * @since 6.0.0
113 */
114public class CmsSearchManager implements I_CmsScheduledJob, I_CmsEventListener {
115
116    /**
117     *  Enumeration class for force unlock types.<p>
118     */
119    public static final class CmsSearchForceUnlockMode extends A_CmsModeStringEnumeration {
120
121        /** Force unlock type "always". */
122        public static final CmsSearchForceUnlockMode ALWAYS = new CmsSearchForceUnlockMode("always");
123
124        /** Force unlock type "never". */
125        public static final CmsSearchForceUnlockMode NEVER = new CmsSearchForceUnlockMode("never");
126
127        /** Force unlock type "only full". */
128        public static final CmsSearchForceUnlockMode ONLYFULL = new CmsSearchForceUnlockMode("onlyfull");
129
130        /** Serializable version id. */
131        private static final long serialVersionUID = 74746076708908673L;
132
133        /**
134         * Creates a new force unlock type with the given name.<p>
135         *
136         * @param mode the mode id to use
137         */
138        protected CmsSearchForceUnlockMode(String mode) {
139
140            super(mode);
141        }
142
143        /**
144         * Returns the lock type for the given type value.<p>
145         *
146         * @param type the type value to get the lock type for
147         *
148         * @return the lock type for the given type value
149         */
150        public static CmsSearchForceUnlockMode valueOf(String type) {
151
152            if (type.equals(ALWAYS.toString())) {
153                return ALWAYS;
154            } else if (type.equals(NEVER.toString())) {
155                return NEVER;
156            } else {
157                return ONLYFULL;
158            }
159        }
160    }
161
162    /**
163     * Handles offline index generation.<p>
164     */
165    protected class CmsSearchOfflineHandler implements I_CmsEventListener {
166
167        /** Indicates if the event handlers for the offline search have been already registered. */
168        private boolean m_isEventRegistered;
169
170        /** The list of resources to index. */
171        private List<CmsPublishedResource> m_resourcesToIndex;
172
173        /**
174         * Initializes the offline index handler.<p>
175         */
176        protected CmsSearchOfflineHandler() {
177
178            m_resourcesToIndex = new ArrayList<CmsPublishedResource>();
179        }
180
181        /**
182         * Implements the event listener of this class.<p>
183         *
184         * @see org.opencms.main.I_CmsEventListener#cmsEvent(org.opencms.main.CmsEvent)
185         */
186        @SuppressWarnings("unchecked")
187        public void cmsEvent(CmsEvent event) {
188
189            switch (event.getType()) {
190                case I_CmsEventListener.EVENT_PROPERTY_MODIFIED:
191                case I_CmsEventListener.EVENT_RESOURCE_CREATED:
192                case I_CmsEventListener.EVENT_RESOURCE_AND_PROPERTIES_MODIFIED:
193                case I_CmsEventListener.EVENT_RESOURCE_MODIFIED:
194                    Object change = event.getData().get(I_CmsEventListener.KEY_CHANGE);
195                    if ((change != null) && change.equals(new Integer(CmsDriverManager.NOTHING_CHANGED))) {
196                        // skip lock & unlock
197                        return;
198                    }
199                    // skip indexing if flag is set in event
200                    Object skip = event.getData().get(I_CmsEventListener.KEY_SKIPINDEX);
201                    if (skip != null) {
202                        return;
203                    }
204
205                    // a resource has been modified - offline indexes require (re)indexing
206                    List<CmsResource> resources = Collections.singletonList(
207                        (CmsResource)event.getData().get(I_CmsEventListener.KEY_RESOURCE));
208                    reIndexResources(resources);
209                    break;
210                case I_CmsEventListener.EVENT_RESOURCE_DELETED:
211                    List<CmsResource> eventResources = (List<CmsResource>)event.getData().get(
212                        I_CmsEventListener.KEY_RESOURCES);
213                    List<CmsResource> resourcesToDelete = new ArrayList<CmsResource>(eventResources);
214                    for (CmsResource res : resourcesToDelete) {
215                        if (res.getState().isNew()) {
216                            // if the resource is new and a delete action was performed
217                            // --> set the state of the resource to deleted
218                            res.setState(CmsResourceState.STATE_DELETED);
219                        }
220                    }
221                    reIndexResources(resourcesToDelete);
222                    break;
223                case I_CmsEventListener.EVENT_RESOURCES_AND_PROPERTIES_MODIFIED:
224                case I_CmsEventListener.EVENT_RESOURCE_MOVED:
225                case I_CmsEventListener.EVENT_RESOURCE_COPIED:
226                case I_CmsEventListener.EVENT_RESOURCES_MODIFIED:
227                    // a list of resources has been modified - offline indexes require (re)indexing
228                    reIndexResources((List<CmsResource>)event.getData().get(I_CmsEventListener.KEY_RESOURCES));
229                    break;
230                default:
231                    // no operation
232            }
233        }
234
235        /**
236         * Adds a list of {@link CmsPublishedResource} objects to be indexed.<p>
237         *
238         * @param resourcesToIndex the list of {@link CmsPublishedResource} objects to be indexed
239         */
240        protected synchronized void addResourcesToIndex(List<CmsPublishedResource> resourcesToIndex) {
241
242            m_resourcesToIndex.addAll(resourcesToIndex);
243        }
244
245        /**
246         * Returns the list of {@link CmsPublishedResource} objects to index.<p>
247         *
248         * @return the resources to index
249         */
250        protected List<CmsPublishedResource> getResourcesToIndex() {
251
252            List<CmsPublishedResource> result;
253            synchronized (this) {
254                result = m_resourcesToIndex;
255                m_resourcesToIndex = new ArrayList<CmsPublishedResource>();
256            }
257            try {
258                CmsObject cms = m_adminCms;
259                CmsProject offline = getOfflineIndexProject();
260                if (offline != null) {
261                    // switch to the offline project if available
262                    cms = OpenCms.initCmsObject(m_adminCms);
263                    cms.getRequestContext().setCurrentProject(offline);
264                }
265                addAdditionallyAffectedResources(cms, result);
266            } catch (CmsException e) {
267                LOG.error(e.getLocalizedMessage(), e);
268            }
269            return result;
270        }
271
272        /**
273         * Initializes this offline search handler, registering the event handlers if required.<p>
274         */
275        protected void initialize() {
276
277            if (m_offlineIndexes.size() > 0) {
278                // there is at least one offline index configured
279                if ((m_offlineIndexThread == null) || !m_offlineIndexThread.isAlive()) {
280                    // create the offline indexing thread
281                    m_offlineIndexThread = new CmsSearchOfflineIndexThread(this);
282                    // start the offline index thread
283                    m_offlineIndexThread.start();
284                }
285            } else {
286                if ((m_offlineIndexThread != null) && m_offlineIndexThread.isAlive()) {
287                    // no offline indexes but thread still running, stop the thread
288                    m_offlineIndexThread.shutDown();
289                    m_offlineIndexThread = null;
290                }
291            }
292            // do this only in case there are offline indexes configured
293            if (!m_isEventRegistered && (m_offlineIndexes.size() > 0)) {
294                m_isEventRegistered = true;
295                // register this object as event listener
296                OpenCms.addCmsEventListener(
297                    this,
298                    new int[] {
299                        I_CmsEventListener.EVENT_PROPERTY_MODIFIED,
300                        I_CmsEventListener.EVENT_RESOURCE_CREATED,
301                        I_CmsEventListener.EVENT_RESOURCE_AND_PROPERTIES_MODIFIED,
302                        I_CmsEventListener.EVENT_RESOURCE_MODIFIED,
303                        I_CmsEventListener.EVENT_RESOURCES_AND_PROPERTIES_MODIFIED,
304                        I_CmsEventListener.EVENT_RESOURCE_MOVED,
305                        I_CmsEventListener.EVENT_RESOURCE_DELETED,
306                        I_CmsEventListener.EVENT_RESOURCE_COPIED,
307                        I_CmsEventListener.EVENT_RESOURCES_MODIFIED});
308            }
309        }
310
311        /**
312         * Updates all offline indexes for the given list of {@link CmsResource} objects.<p>
313         *
314         * @param resources a list of {@link CmsResource} objects to update in the offline indexes
315         */
316        protected synchronized void reIndexResources(List<CmsResource> resources) {
317
318            List<CmsPublishedResource> resourcesToIndex = new ArrayList<CmsPublishedResource>(resources.size());
319            for (CmsResource res : resources) {
320                CmsPublishedResource pubRes = new CmsPublishedResource(res);
321                resourcesToIndex.add(pubRes);
322            }
323            if (resourcesToIndex.size() > 0) {
324                // add the resources found to the offline index thread
325                addResourcesToIndex(resourcesToIndex);
326            }
327        }
328    }
329
330    /**
331     * The offline indexer thread runs periodically and indexes all resources added by the event handler.<p>
332     */
333    protected class CmsSearchOfflineIndexThread extends Thread {
334
335        /** The event handler that triggers this thread. */
336        CmsSearchOfflineHandler m_handler;
337
338        /** Indicates if this thread is still alive. */
339        boolean m_isAlive;
340
341        /** Indicates that an index update thread is currently running. */
342        private boolean m_isUpdating;
343
344        /** If true a manual update (after file upload) was triggered. */
345        private boolean m_updateTriggered;
346
347        /** The wait handle used for signalling when the worker thread has finished. */
348        private CmsWaitHandle m_waitHandle = new CmsWaitHandle();
349
350        /**
351         * Constructor.<p>
352         *
353         * @param handler the offline index event handler
354         */
355        protected CmsSearchOfflineIndexThread(CmsSearchOfflineHandler handler) {
356
357            super("OpenCms: Offline Search Indexer");
358            m_handler = handler;
359        }
360
361        /**
362         * Gets the wait handle used for signalling when the worker thread has finished.
363         *
364         * @return the wait handle
365         **/
366        public CmsWaitHandle getWaitHandle() {
367
368            return m_waitHandle;
369        }
370
371        /**
372         * @see java.lang.Thread#interrupt()
373         */
374        @Override
375        public void interrupt() {
376
377            super.interrupt();
378            m_updateTriggered = true;
379        }
380
381        /**
382         * @see java.lang.Thread#run()
383         */
384        @Override
385        public void run() {
386
387            // create a log report for the output
388            I_CmsReport report = new CmsLogReport(m_adminCms.getRequestContext().getLocale(), CmsSearchManager.class);
389            long offlineUpdateFrequency = getOfflineUpdateFrequency();
390            m_updateTriggered = false;
391            try {
392                while (m_isAlive) {
393                    if (!m_updateTriggered) {
394                        try {
395                            sleep(offlineUpdateFrequency);
396                        } catch (InterruptedException e) {
397                            // continue the thread after interruption
398                            if (!m_isAlive) {
399                                // the thread has been shut down while sleeping
400                                continue;
401                            }
402                            if (offlineUpdateFrequency != getOfflineUpdateFrequency()) {
403                                // offline update frequency change - clear interrupt status
404                                offlineUpdateFrequency = getOfflineUpdateFrequency();
405                            }
406                            LOG.info(e.getLocalizedMessage(), e);
407                        }
408                    }
409                    if (m_isAlive) {
410                        // set update trigger to false since we do the update now
411                        m_updateTriggered = false;
412                        // get list of resource to update
413                        List<CmsPublishedResource> resourcesToIndex = getResourcesToIndex();
414                        if (resourcesToIndex.size() > 0) {
415                            // only start indexing if there is at least one resource
416                            startOfflineUpdateThread(report, resourcesToIndex);
417                        } else {
418                            getWaitHandle().release();
419                        }
420                        // this is just called to clear the interrupt status of the thread
421                        interrupted();
422                    }
423                }
424            } finally {
425                // make sure that live status is reset in case of Exceptions
426                m_isAlive = false;
427            }
428
429        }
430
431        /**
432         * @see java.lang.Thread#start()
433         */
434        @Override
435        public synchronized void start() {
436
437            m_isAlive = true;
438            super.start();
439        }
440
441        /**
442         * Obtains the list of resource to update in the offline index,
443         * then optimizes the list by removing duplicate entries.<p>
444         *
445         * @return the list of resource to update in the offline index
446         */
447        protected List<CmsPublishedResource> getResourcesToIndex() {
448
449            List<CmsPublishedResource> resourcesToIndex = m_handler.getResourcesToIndex();
450            List<CmsPublishedResource> result = new ArrayList<CmsPublishedResource>(resourcesToIndex.size());
451
452            // Reverse to always keep the last list entries
453            Collections.reverse(resourcesToIndex);
454            for (CmsPublishedResource pubRes : resourcesToIndex) {
455                boolean addResource = true;
456                for (CmsPublishedResource resRes : result) {
457                    if (pubRes.equals(resRes)
458                        && (pubRes.getState() == resRes.getState())
459                        && (pubRes.getMovedState() == resRes.getMovedState())
460                        && pubRes.getRootPath().equals(resRes.getRootPath())) {
461                        // resource already in the update list
462                        addResource = false;
463                        break;
464                    }
465                }
466                if (addResource) {
467                    result.add(pubRes);
468                }
469
470            }
471            Collections.reverse(result);
472            return changeStateOfMoveOriginsToDeleted(result);
473        }
474
475        /**
476         * Shuts down this offline index thread.<p>
477         */
478        protected void shutDown() {
479
480            m_isAlive = false;
481            interrupt();
482            if (m_isUpdating) {
483                long waitTime = getOfflineUpdateFrequency() / 2;
484                int waitSteps = 0;
485                do {
486                    try {
487                        // wait half the time of the offline index frequency for the thread to finish
488                        Thread.sleep(waitTime);
489                    } catch (InterruptedException e) {
490                        // continue
491                        LOG.info(e.getLocalizedMessage(), e);
492                    }
493                    waitSteps++;
494                    // wait 5 times then stop waiting
495                } while ((waitSteps < 5) && m_isUpdating);
496            }
497        }
498
499        /**
500         * Updates the offline search indexes for the given list of resources.<p>
501         *
502         * @param report the report to write the index information to
503         * @param resourcesToIndex the list of {@link CmsPublishedResource} objects to index
504         */
505        protected void startOfflineUpdateThread(I_CmsReport report, List<CmsPublishedResource> resourcesToIndex) {
506
507            CmsSearchOfflineIndexWorkThread thread = new CmsSearchOfflineIndexWorkThread(report, resourcesToIndex);
508            long startTime = System.currentTimeMillis();
509            long waitTime = getOfflineUpdateFrequency() / 2;
510            if (LOG.isDebugEnabled()) {
511                LOG.debug(
512                    Messages.get().getBundle().key(
513                        Messages.LOG_OI_UPDATE_START_1,
514                        Integer.valueOf(resourcesToIndex.size())));
515            }
516
517            m_isUpdating = true;
518            thread.start();
519
520            do {
521                try {
522                    // wait half the time of the offline index frequency for the thread to finish
523                    thread.join(waitTime);
524                } catch (InterruptedException e) {
525                    // continue
526                    LOG.info(e.getLocalizedMessage(), e);
527                }
528                if (thread.isAlive()) {
529                    LOG.warn(
530                        Messages.get().getBundle().key(
531                            Messages.LOG_OI_UPDATE_LONG_2,
532                            Integer.valueOf(resourcesToIndex.size()),
533                            Long.valueOf(System.currentTimeMillis() - startTime)));
534                }
535            } while (thread.isAlive());
536            m_isUpdating = false;
537
538            if (LOG.isDebugEnabled()) {
539                LOG.debug(
540                    Messages.get().getBundle().key(
541                        Messages.LOG_OI_UPDATE_FINISH_2,
542                        Integer.valueOf(resourcesToIndex.size()),
543                        Long.valueOf(System.currentTimeMillis() - startTime)));
544            }
545        }
546
547        /**
548         * Helper method which changes the states of resources which are to be indexed but have the wrong path to 'deleted'.
549         * This is needed to deal with moved resources, since the documents with the old paths must be removed from the index,
550         *
551         * @param resourcesToIndex the resources to index
552         *
553         * @return the resources to index, but resource states are set to 'deleted' for resources with outdated paths
554         */
555        private List<CmsPublishedResource> changeStateOfMoveOriginsToDeleted(
556            List<CmsPublishedResource> resourcesToIndex) {
557
558            Map<CmsUUID, String> lastValidPaths = new HashMap<CmsUUID, String>();
559            for (CmsPublishedResource resource : resourcesToIndex) {
560                if (resource.getState().isDeleted()) {
561                    // we don't want the last path to be from a deleted resource
562                    continue;
563                }
564                lastValidPaths.put(resource.getStructureId(), resource.getRootPath());
565            }
566            List<CmsPublishedResource> result = new ArrayList<CmsPublishedResource>();
567            for (CmsPublishedResource resource : resourcesToIndex) {
568                if (resource.getState().isDeleted()) {
569                    result.add(resource);
570                    continue;
571                }
572                String lastValidPath = lastValidPaths.get(resource.getStructureId());
573                if (resource.getRootPath().equals(lastValidPath) || resource.getStructureId().isNullUUID()) {
574                    result.add(resource);
575                } else {
576                    result.add(
577                        new CmsPublishedResource(
578                            resource.getStructureId(),
579                            resource.getResourceId(),
580                            resource.getPublishTag(),
581                            resource.getRootPath(),
582                            resource.getType(),
583                            resource.isFolder(),
584                            CmsResource.STATE_DELETED, // make sure index entry with outdated path is deleted
585                            resource.getSiblingCount()));
586                }
587            }
588            return result;
589        }
590    }
591
592    /**
593     * An offline index worker Thread runs each time for every offline index update action.<p>
594     *
595     * This was decoupled from the main {@link CmsSearchOfflineIndexThread} in order to avoid
596     * problems if a single operation "hangs" the Tread.<p>
597     */
598    protected class CmsSearchOfflineIndexWorkThread extends Thread {
599
600        /** The report to write the index information to. */
601        I_CmsReport m_report;
602
603        /** The list of {@link CmsPublishedResource} objects to index. */
604        List<CmsPublishedResource> m_resourcesToIndex;
605
606        /**
607         * Updates the offline search indexes for the given list of resources.<p>
608         *
609         * @param report the report to write the index information to
610         * @param resourcesToIndex the list of {@link CmsPublishedResource} objects to index
611         */
612        protected CmsSearchOfflineIndexWorkThread(I_CmsReport report, List<CmsPublishedResource> resourcesToIndex) {
613
614            super("OpenCms: Offline Search Index Worker");
615            m_report = report;
616            m_resourcesToIndex = resourcesToIndex;
617        }
618
619        /**
620         * @see java.lang.Thread#run()
621         */
622        @Override
623        public void run() {
624
625            updateIndexOffline(m_report, m_resourcesToIndex);
626            if (m_offlineIndexThread != null) {
627                m_offlineIndexThread.getWaitHandle().release();
628            }
629        }
630    }
631
632    /** This needs to be a fair lock to preserve order of threads accessing the search manager. */
633    private static final ReentrantLock SEARCH_MANAGER_LOCK = new ReentrantLock(true);
634
635    /** The default value used for generating search result excerpts (1024 chars). */
636    public static final int DEFAULT_EXCERPT_LENGTH = 1024;
637
638    /** The default value used for keeping the extraction results in the cache (672 hours = 4 weeks). */
639    public static final float DEFAULT_EXTRACTION_CACHE_MAX_AGE = 672.0f;
640
641    /** Default for the maximum number of modifications before a commit in the search index is triggered (500). */
642    public static final int DEFAULT_MAX_MODIFICATIONS_BEFORE_COMMIT = 500;
643
644    /** The default update frequency for offline indexes (15000 msec = 15 sec). */
645    public static final int DEFAULT_OFFLINE_UPDATE_FREQNENCY = 15000;
646
647    /** The default maximal wait time for re-indexing after editing a content. */
648    public static final int DEFAULT_MAX_INDEX_WAITTIME = 30000;
649
650    /** The default timeout value used for generating a document for the search index (60000 msec = 1 min). */
651    public static final int DEFAULT_TIMEOUT = 60000;
652
653    /** Scheduler parameter: Update only a specified list of indexes. */
654    public static final String JOB_PARAM_INDEXLIST = "indexList";
655
656    /** Scheduler parameter: Write the output of the update to the logfile. */
657    public static final String JOB_PARAM_WRITELOG = "writeLog";
658
659    /** Prefix for Lucene default analyzers package (<code>org.apache.lucene.analysis.</code>). */
660    public static final String LUCENE_ANALYZER = "org.apache.lucene.analysis.core.";
661
662    /** The log object for this class. */
663    protected static final Log LOG = CmsLog.getLog(CmsSearchManager.class);
664
665    /** The administrator OpenCms user context to access OpenCms VFS resources. */
666    protected CmsObject m_adminCms;
667
668    /** The list of indexes that are configured for offline index mode. */
669    protected List<I_CmsSearchIndex> m_offlineIndexes;
670
671    /** The thread used of offline indexing. */
672    protected CmsSearchOfflineIndexThread m_offlineIndexThread;
673
674    /** Configured analyzers for languages using &lt;analyzer&gt;. */
675    private HashMap<Locale, CmsSearchAnalyzer> m_analyzers;
676
677    /** Stores the offline update frequency while indexing is paused. */
678    private long m_configuredOfflineIndexingFrequency;
679
680    /** The Solr core container. */
681    private CoreContainer m_coreContainer;
682
683    /** A map of document factory configurations. */
684    private List<CmsSearchDocumentType> m_documentTypeConfigs;
685
686    /** A map of document factories keyed by their matching Cms resource types and/or mimetypes. */
687    private Map<String, I_CmsDocumentFactory> m_documentTypes;
688
689    /** The max age for extraction results to remain in the cache. */
690    private float m_extractionCacheMaxAge;
691
692    /** The cache for the extraction results. */
693    private CmsExtractionResultCache m_extractionResultCache;
694
695    /** Contains the available field configurations. */
696    private Map<String, I_CmsSearchFieldConfiguration> m_fieldConfigurations;
697
698    /** The force unlock type. */
699    private CmsSearchForceUnlockMode m_forceUnlockMode;
700
701    /** The class used to highlight the search terms in the excerpt of a search result. */
702    private I_CmsTermHighlighter m_highlighter;
703
704    /** A list of search indexes. */
705    private List<I_CmsSearchIndex> m_indexes;
706
707    /** Seconds to wait for an index lock. */
708    private int m_indexLockMaxWaitSeconds = 10;
709
710    /** Configured index sources. */
711    private Map<String, CmsSearchIndexSource> m_indexSources;
712
713    /** The max. char. length of the excerpt in the search result. */
714    private int m_maxExcerptLength;
715
716    /** The maximum number of modifications before a commit in the search index is triggered. */
717    private int m_maxModificationsBeforeCommit;
718
719    /** The offline index search handler. */
720    private CmsSearchOfflineHandler m_offlineHandler;
721
722    /** The update frequency of the offline indexer in milliseconds. */
723    private long m_offlineUpdateFrequency;
724
725    /** The maximal time to wait for re-indexing after a content is edited (in milliseconds). */
726    private long m_maxIndexWaitTime;
727
728    /** Path to index files below WEB-INF/. */
729    private String m_path;
730
731    /** The Solr configuration. */
732    private CmsSolrConfiguration m_solrConfig;
733
734    /** Timeout for abandoning indexing thread. */
735    private long m_timeout;
736
737    /**
738     * Default constructor when called as cron job.<p>
739     */
740    public CmsSearchManager() {
741
742        m_documentTypes = new HashMap<String, I_CmsDocumentFactory>();
743        m_documentTypeConfigs = new ArrayList<CmsSearchDocumentType>();
744        m_analyzers = new HashMap<Locale, CmsSearchAnalyzer>();
745        m_indexes = new ArrayList<I_CmsSearchIndex>();
746        m_indexSources = new TreeMap<String, CmsSearchIndexSource>();
747        m_offlineHandler = new CmsSearchOfflineHandler();
748        m_extractionCacheMaxAge = DEFAULT_EXTRACTION_CACHE_MAX_AGE;
749        m_maxExcerptLength = DEFAULT_EXCERPT_LENGTH;
750        m_offlineUpdateFrequency = DEFAULT_OFFLINE_UPDATE_FREQNENCY;
751        m_maxIndexWaitTime = DEFAULT_MAX_INDEX_WAITTIME;
752        m_maxModificationsBeforeCommit = DEFAULT_MAX_MODIFICATIONS_BEFORE_COMMIT;
753
754        m_fieldConfigurations = new HashMap<String, I_CmsSearchFieldConfiguration>();
755        // make sure we have a "standard" field configuration
756        addFieldConfiguration(CmsLuceneFieldConfiguration.DEFAULT_STANDARD);
757
758        if (CmsLog.INIT.isInfoEnabled()) {
759            CmsLog.INIT.info(Messages.get().getBundle().key(Messages.INIT_START_SEARCH_CONFIG_0));
760        }
761    }
762
763    /**
764     * Returns an analyzer for the given class name.<p>
765     *
766     * @param className the class name of the analyzer
767     *
768     * @return the appropriate lucene analyzer
769     *
770     * @throws Exception if something goes wrong
771     */
772    public static Analyzer getAnalyzer(String className) throws Exception {
773
774        Analyzer analyzer = null;
775        Class<?> analyzerClass;
776        try {
777            analyzerClass = Class.forName(className);
778        } catch (ClassNotFoundException e) {
779            // allow Lucene standard classes to be written in a short form
780            analyzerClass = Class.forName(LUCENE_ANALYZER + className);
781        }
782
783        // since Lucene 3.0 most analyzers need a "version" parameter and don't support an empty constructor
784        if (StandardAnalyzer.class.equals(analyzerClass)) {
785            // the Lucene standard analyzer is used - but without any stopwords.
786            analyzer = new StandardAnalyzer(new CharArraySet(0, false));
787        } else {
788            analyzer = (Analyzer)analyzerClass.newInstance();
789        }
790        return analyzer;
791    }
792
793    /**
794     * Returns the Solr index configured with the parameters name.
795     * The parameters must contain a key/value pair with an existing
796     * Solr index, otherwise <code>null</code> is returned.<p>
797     *
798     * @param cms the current context
799     * @param params the parameter map
800     *
801     * @return the best matching Solr index
802     */
803    public static final CmsSolrIndex getIndexSolr(CmsObject cms, Map<String, String[]> params) {
804
805        String indexName = null;
806        CmsSolrIndex index = null;
807        // try to get the index name from the parameters: 'core' or 'index'
808        if (params != null) {
809            indexName = params.get(OpenCmsSolrHandler.PARAM_CORE) != null
810            ? params.get(OpenCmsSolrHandler.PARAM_CORE)[0]
811            : (params.get(OpenCmsSolrHandler.PARAM_INDEX) != null
812            ? params.get(OpenCmsSolrHandler.PARAM_INDEX)[0]
813            : null);
814        }
815        if (indexName == null) {
816            // if no parameter is specified try to use the default online/offline indexes by context
817            indexName = cms.getRequestContext().getCurrentProject().isOnlineProject()
818            ? CmsSolrIndex.DEFAULT_INDEX_NAME_ONLINE
819            : CmsSolrIndex.DEFAULT_INDEX_NAME_OFFLINE;
820        }
821        // try to get the index
822        index = indexName != null ? OpenCms.getSearchManager().getIndexSolr(indexName) : null;
823        if (index == null) {
824            // if there is exactly one index, a missing core / index parameter doesn't matter, since there is no choice.
825            List<CmsSolrIndex> solrs = OpenCms.getSearchManager().getAllSolrIndexes();
826            if ((solrs != null) && !solrs.isEmpty() && (solrs.size() == 1)) {
827                index = solrs.get(0);
828            }
829        }
830        return index;
831    }
832
833    /**
834     * Returns <code>true</code> if the index for the given name is a Lucene index, <code>false</code> otherwise.<p>
835     *
836     * @param indexName the name of the index to check
837     *
838     * @return <code>true</code> if the index for the given name is a Lucene index
839     */
840    public static boolean isLuceneIndex(String indexName) {
841
842        I_CmsSearchIndex i = OpenCms.getSearchManager().getIndex(indexName);
843        return (i instanceof CmsSearchIndex) && (!(i instanceof CmsSolrIndex));
844    }
845
846    /**
847     * Adds an analyzer.<p>
848     *
849     * @param analyzer an analyzer
850     */
851    public void addAnalyzer(CmsSearchAnalyzer analyzer) {
852
853        m_analyzers.put(analyzer.getLocale(), analyzer);
854
855        if (CmsLog.INIT.isInfoEnabled()) {
856            CmsLog.INIT.info(
857                Messages.get().getBundle().key(
858                    Messages.INIT_ADD_ANALYZER_2,
859                    analyzer.getLocale(),
860                    analyzer.getClassName()));
861        }
862    }
863
864    /**
865     * Adds a document type.<p>
866     *
867     * @param documentType a document type
868     */
869    public void addDocumentTypeConfig(CmsSearchDocumentType documentType) {
870
871        m_documentTypeConfigs.add(documentType);
872
873        if (CmsLog.INIT.isInfoEnabled()) {
874            CmsLog.INIT.info(
875                Messages.get().getBundle().key(
876                    Messages.INIT_SEARCH_DOC_TYPES_2,
877                    documentType.getName(),
878                    documentType.getClassName()));
879        }
880    }
881
882    /**
883     * Adds a search field configuration to the search manager.<p>
884     *
885     * @param fieldConfiguration the search field configuration to add
886     */
887    public void addFieldConfiguration(I_CmsSearchFieldConfiguration fieldConfiguration) {
888
889        m_fieldConfigurations.put(fieldConfiguration.getName(), fieldConfiguration);
890    }
891
892    /**
893     * Adds a search index to the configuration.<p>
894     *
895     * @param searchIndex the search index to add
896     */
897    public void addSearchIndex(I_CmsSearchIndex searchIndex) {
898
899        if (!searchIndex.isInitialized()) {
900            if (OpenCms.getRunLevel() > OpenCms.RUNLEVEL_2_INITIALIZING) {
901                try {
902                    searchIndex.initialize();
903                } catch (CmsException e) {
904                    // should never happen
905                    LOG.error(e.getMessage(), e);
906                }
907            }
908        }
909
910        // name: not null or emtpy and unique
911        String name = searchIndex.getName();
912        if (CmsStringUtil.isEmptyOrWhitespaceOnly(name)) {
913            throw new CmsIllegalArgumentException(
914                Messages.get().container(Messages.ERR_SEARCHINDEX_CREATE_MISSING_NAME_0));
915        }
916        if (m_indexSources.keySet().contains(name)) {
917            throw new CmsIllegalArgumentException(
918                Messages.get().container(Messages.ERR_SEARCHINDEX_CREATE_INVALID_NAME_1, name));
919        }
920
921        m_indexes.add(searchIndex);
922        if (m_adminCms != null) {
923            initOfflineIndexes();
924        }
925
926        if (CmsLog.INIT.isInfoEnabled()) {
927            CmsLog.INIT.info(
928                Messages.get().getBundle().key(
929                    Messages.INIT_ADD_SEARCH_INDEX_2,
930                    searchIndex.getName(),
931                    searchIndex.getProject()));
932        }
933    }
934
935    /**
936     * Adds a search index source configuration.<p>
937     *
938     * @param searchIndexSource a search index source configuration
939     */
940    public void addSearchIndexSource(CmsSearchIndexSource searchIndexSource) {
941
942        m_indexSources.put(searchIndexSource.getName(), searchIndexSource);
943
944        if (CmsLog.INIT.isInfoEnabled()) {
945            CmsLog.INIT.info(
946                Messages.get().getBundle().key(
947                    Messages.INIT_SEARCH_INDEX_SOURCE_2,
948                    searchIndexSource.getName(),
949                    searchIndexSource.getIndexerClassName()));
950        }
951    }
952
953    /**
954     * Implements the event listener of this class.<p>
955     *
956     * @see org.opencms.main.I_CmsEventListener#cmsEvent(org.opencms.main.CmsEvent)
957     */
958    public void cmsEvent(CmsEvent event) {
959
960        switch (event.getType()) {
961            case I_CmsEventListener.EVENT_REBUILD_SEARCHINDEXES:
962                List<String> indexNames = null;
963                if ((event.getData() != null)
964                    && CmsStringUtil.isNotEmptyOrWhitespaceOnly(
965                        (String)event.getData().get(I_CmsEventListener.KEY_INDEX_NAMES))) {
966                    indexNames = CmsStringUtil.splitAsList(
967                        (String)event.getData().get(I_CmsEventListener.KEY_INDEX_NAMES),
968                        ",",
969                        true);
970                }
971                try {
972                    if (LOG.isDebugEnabled()) {
973                        LOG.debug(
974                            Messages.get().getBundle().key(
975                                Messages.LOG_EVENT_REBUILD_SEARCHINDEX_1,
976                                indexNames == null ? "" : CmsStringUtil.collectionAsString(indexNames, ",")),
977                            new Exception());
978                    }
979                    if (indexNames == null) {
980                        rebuildAllIndexes(getEventReport(event));
981                    } else {
982                        rebuildIndexes(indexNames, getEventReport(event));
983                    }
984                } catch (CmsException e) {
985                    if (LOG.isErrorEnabled()) {
986                        LOG.error(
987                            Messages.get().getBundle().key(
988                                Messages.ERR_EVENT_REBUILD_SEARCHINDEX_1,
989                                indexNames == null ? "" : CmsStringUtil.collectionAsString(indexNames, ",")),
990                            e);
991                    }
992                }
993                break;
994            case I_CmsEventListener.EVENT_CLEAR_CACHES:
995                if (LOG.isDebugEnabled()) {
996                    LOG.debug(Messages.get().getBundle().key(Messages.LOG_EVENT_CLEAR_CACHES_0), new Exception());
997                }
998                break;
999            case I_CmsEventListener.EVENT_PUBLISH_PROJECT:
1000                // event data contains a list of the published resources
1001                CmsUUID publishHistoryId = new CmsUUID((String)event.getData().get(I_CmsEventListener.KEY_PUBLISHID));
1002                if (LOG.isDebugEnabled()) {
1003                    LOG.debug(Messages.get().getBundle().key(Messages.LOG_EVENT_PUBLISH_PROJECT_1, publishHistoryId));
1004                }
1005                updateAllIndexes(m_adminCms, publishHistoryId, getEventReport(event));
1006                if (LOG.isDebugEnabled()) {
1007                    LOG.debug(
1008                        Messages.get().getBundle().key(
1009                            Messages.LOG_EVENT_PUBLISH_PROJECT_FINISHED_1,
1010                            publishHistoryId));
1011                }
1012                break;
1013            default:
1014                // no operation
1015        }
1016    }
1017
1018    /**
1019     * Returns all Solr index.<p>
1020     *
1021     * @return all Solr indexes
1022     */
1023    public List<CmsSolrIndex> getAllSolrIndexes() {
1024
1025        List<CmsSolrIndex> result = new ArrayList<CmsSolrIndex>();
1026        for (String indexName : getIndexNames()) {
1027            CmsSolrIndex index = getIndexSolr(indexName);
1028            if (index != null) {
1029                result.add(index);
1030            }
1031        }
1032        return result;
1033    }
1034
1035    /**
1036     * Returns an analyzer for the given language.<p>
1037     *
1038     * The analyzer is selected according to the analyzer configuration.<p>
1039     *
1040     * @param locale the locale to get the analyzer for
1041     * @return the appropriate lucene analyzer
1042     *
1043     * @throws CmsSearchException if something goes wrong
1044     */
1045    public Analyzer getAnalyzer(Locale locale) throws CmsSearchException {
1046
1047        Analyzer analyzer = null;
1048        String className = null;
1049
1050        CmsSearchAnalyzer analyzerConf = m_analyzers.get(locale);
1051        if (analyzerConf == null) {
1052            throw new CmsSearchException(Messages.get().container(Messages.ERR_ANALYZER_NOT_FOUND_1, locale));
1053        }
1054
1055        try {
1056            analyzer = getAnalyzer(analyzerConf.getClassName());
1057        } catch (Exception e) {
1058            throw new CmsSearchException(Messages.get().container(Messages.ERR_LOAD_ANALYZER_1, className), e);
1059        }
1060
1061        return analyzer;
1062    }
1063
1064    /**
1065     * Returns an unmodifiable view of the map that contains the {@link CmsSearchAnalyzer} list.<p>
1066     *
1067     * The keys in the map are {@link Locale} objects, and the values are {@link CmsSearchAnalyzer} objects.
1068     *
1069     * @return an unmodifiable view of the Analyzers Map
1070     */
1071    public Map<Locale, CmsSearchAnalyzer> getAnalyzers() {
1072
1073        return Collections.unmodifiableMap(m_analyzers);
1074    }
1075
1076    /**
1077     * Returns the search analyzer for the given locale.<p>
1078     *
1079     * @param locale the locale to get the analyzer for
1080     *
1081     * @return the search analyzer for the given locale
1082     */
1083    public CmsSearchAnalyzer getCmsSearchAnalyzer(Locale locale) {
1084
1085        return m_analyzers.get(locale);
1086    }
1087
1088    /**
1089     * Returns the name of the directory below WEB-INF/ where the search indexes are stored.<p>
1090     *
1091     * @return the name of the directory below WEB-INF/ where the search indexes are stored
1092     */
1093    public String getDirectory() {
1094
1095        return m_path;
1096    }
1097
1098    /**
1099     * Returns the configured Solr home directory <code>null</code> if not set.<p>
1100     *
1101     * @return the Solr home directory
1102     */
1103    public String getDirectorySolr() {
1104
1105        return m_solrConfig != null ? m_solrConfig.getHome() : null;
1106    }
1107
1108    /**
1109     * Returns a lucene document factory for given resource.<p>
1110     *
1111     * The type of the document factory is selected by the type of the resource
1112     * and the MIME type of the resource content, according to the configuration in <code>opencms-search.xml</code>.<p>
1113     *
1114     * @param resource a cms resource
1115     * @return a lucene document factory or null
1116     */
1117    public I_CmsDocumentFactory getDocumentFactory(CmsResource resource) {
1118
1119        // first get the MIME type of the resource
1120        String mimeType = OpenCms.getResourceManager().getMimeType(resource.getRootPath(), null, "unknown");
1121        String resourceType = null;
1122        try {
1123            resourceType = OpenCms.getResourceManager().getResourceType(resource.getTypeId()).getTypeName();
1124        } catch (CmsLoaderException e) {
1125            // ignore, unknown resource type, resource can not be indexed
1126            LOG.info(e.getLocalizedMessage(), e);
1127        }
1128        return getDocumentFactory(resourceType, mimeType);
1129    }
1130
1131    /**
1132     * Returns a lucene document factory for given resource type and MIME type.<p>
1133     *
1134     * The type of the document factory is selected  according to the configuration
1135     * in <code>opencms-search.xml</code>.<p>
1136     *
1137     * @param resourceType the resource type name
1138     * @param mimeType the MIME type
1139     *
1140     * @return a lucene document factory or null in case no matching factory was found
1141     */
1142    public I_CmsDocumentFactory getDocumentFactory(String resourceType, String mimeType) {
1143
1144        I_CmsDocumentFactory result = null;
1145        if (resourceType != null) {
1146            // create the factory lookup key for the document
1147            String documentTypeKey = A_CmsVfsDocument.getDocumentKey(resourceType, mimeType);
1148            // check if a setting is available for this specific MIME type
1149            result = m_documentTypes.get(documentTypeKey);
1150            if (result == null) {
1151                // no setting is available, try to use a generic setting without MIME type
1152                result = m_documentTypes.get(A_CmsVfsDocument.getDocumentKey(resourceType, null));
1153                // please note: the result may still be null
1154            }
1155        }
1156        return result;
1157    }
1158
1159    /**
1160     * Returns a document type config.<p>
1161     *
1162     * @param name the name of the document type config
1163     * @return the document type config.
1164     */
1165    public CmsSearchDocumentType getDocumentTypeConfig(String name) {
1166
1167        // this is really used only for the search manager GUI,
1168        // so performance is not an issue and no lookup map is generated
1169        for (int i = 0; i < m_documentTypeConfigs.size(); i++) {
1170            CmsSearchDocumentType type = m_documentTypeConfigs.get(i);
1171            if (type.getName().equals(name)) {
1172                return type;
1173            }
1174        }
1175        return null;
1176    }
1177
1178    /**
1179     * Returns an unmodifiable view (read-only) of the DocumentTypeConfigs Map.<p>
1180     *
1181     * @return an unmodifiable view (read-only) of the DocumentTypeConfigs Map
1182     */
1183    public List<CmsSearchDocumentType> getDocumentTypeConfigs() {
1184
1185        return Collections.unmodifiableList(m_documentTypeConfigs);
1186    }
1187
1188    /**
1189     * Returns the maximum age a text extraction result is kept in the cache (in hours).<p>
1190     *
1191     * @return the maximum age a text extraction result is kept in the cache (in hours)
1192     */
1193    public float getExtractionCacheMaxAge() {
1194
1195        return m_extractionCacheMaxAge;
1196    }
1197
1198    /**
1199     * Returns the search field configuration with the given name.<p>
1200     *
1201     * In case no configuration is available with the given name, <code>null</code> is returned.<p>
1202     *
1203     * @param name the name to get the search field configuration for
1204     *
1205     * @return the search field configuration with the given name
1206     */
1207    public I_CmsSearchFieldConfiguration getFieldConfiguration(String name) {
1208
1209        return m_fieldConfigurations.get(name);
1210    }
1211
1212    /**
1213     * Returns the unmodifieable List of configured {@link I_CmsSearchFieldConfiguration} entries.<p>
1214     *
1215     * @return the unmodifieable List of configured {@link I_CmsSearchFieldConfiguration} entries
1216     */
1217    public List<I_CmsSearchFieldConfiguration> getFieldConfigurations() {
1218
1219        List<I_CmsSearchFieldConfiguration> result = new ArrayList<I_CmsSearchFieldConfiguration>(
1220            m_fieldConfigurations.values());
1221        Collections.sort(result);
1222        return Collections.unmodifiableList(result);
1223    }
1224
1225    /**
1226     * Returns the Lucene search field configurations only.<p>
1227     *
1228     * @return the Lucene search field configurations
1229     */
1230    public List<CmsLuceneFieldConfiguration> getFieldConfigurationsLucene() {
1231
1232        List<CmsLuceneFieldConfiguration> result = new ArrayList<CmsLuceneFieldConfiguration>();
1233        for (I_CmsSearchFieldConfiguration conf : m_fieldConfigurations.values()) {
1234            if (conf instanceof CmsLuceneFieldConfiguration) {
1235                result.add((CmsLuceneFieldConfiguration)conf);
1236            }
1237        }
1238        Collections.sort(result);
1239        return Collections.unmodifiableList(result);
1240    }
1241
1242    /**
1243     * Returns the Solr search field configurations only.<p>
1244     *
1245     * @return the Solr search field configurations
1246     */
1247    public List<CmsSolrFieldConfiguration> getFieldConfigurationsSolr() {
1248
1249        List<CmsSolrFieldConfiguration> result = new ArrayList<CmsSolrFieldConfiguration>();
1250        for (I_CmsSearchFieldConfiguration conf : m_fieldConfigurations.values()) {
1251            if (conf instanceof CmsSolrFieldConfiguration) {
1252                result.add((CmsSolrFieldConfiguration)conf);
1253            }
1254        }
1255        Collections.sort(result);
1256        return Collections.unmodifiableList(result);
1257    }
1258
1259    /**
1260     * Returns the force unlock mode during indexing.<p>
1261     *
1262     * @return the force unlock mode during indexing
1263     */
1264    public CmsSearchForceUnlockMode getForceunlock() {
1265
1266        return m_forceUnlockMode;
1267    }
1268
1269    /**
1270     * Returns the highlighter.<p>
1271     *
1272     * @return the highlighter
1273     */
1274    public I_CmsTermHighlighter getHighlighter() {
1275
1276        return m_highlighter;
1277    }
1278
1279    /**
1280     * Returns the Lucene search index configured with the given name.<p>
1281     * The index must exist, otherwise <code>null</code> is returned.
1282     *
1283     * @param indexName then name of the requested search index
1284     *
1285     * @return the Lucene search index configured with the given name
1286     */
1287    public I_CmsSearchIndex getIndex(String indexName) {
1288
1289        for (I_CmsSearchIndex index : m_indexes) {
1290            if (indexName.equalsIgnoreCase(index.getName())) {
1291                return index;
1292            }
1293        }
1294        return null;
1295    }
1296
1297    /**
1298     * Returns the seconds to wait for an index lock during an update operation.<p>
1299     *
1300     * @return the seconds to wait for an index lock during an update operation
1301     */
1302    public int getIndexLockMaxWaitSeconds() {
1303
1304        return m_indexLockMaxWaitSeconds;
1305    }
1306
1307    /**
1308     * Returns the names of all configured indexes.<p>
1309     *
1310     * @return list of names
1311     */
1312    public List<String> getIndexNames() {
1313
1314        List<String> indexNames = new ArrayList<String>();
1315        for (int i = 0, n = m_indexes.size(); i < n; i++) {
1316            indexNames.add((m_indexes.get(i)).getName());
1317        }
1318
1319        return indexNames;
1320    }
1321
1322    /**
1323     * Returns the Solr index configured with the given name.<p>
1324     * The index must exist, otherwise <code>null</code> is returned.
1325     *
1326     * @param indexName then name of the requested Solr index
1327     * @return the Solr index configured with the given name
1328     */
1329    public CmsSolrIndex getIndexSolr(String indexName) {
1330
1331        I_CmsSearchIndex index = getIndex(indexName);
1332        if (index instanceof CmsSolrIndex) {
1333            return (CmsSolrIndex)index;
1334        }
1335        return null;
1336    }
1337
1338    /**
1339     * Returns a search index source for a specified source name.<p>
1340     *
1341     * @param sourceName the name of the index source
1342     * @return a search index source
1343     */
1344    public CmsSearchIndexSource getIndexSource(String sourceName) {
1345
1346        return m_indexSources.get(sourceName);
1347    }
1348
1349    /**
1350     * Returns the max. excerpt length.<p>
1351     *
1352     * @return the max excerpt length
1353     */
1354    public int getMaxExcerptLength() {
1355
1356        return m_maxExcerptLength;
1357    }
1358
1359    /**
1360     * Returns the maximal time to wait for re-indexing after a content is edited (in milliseconds).<p>
1361     *
1362     * @return the maximal time to wait for re-indexing after a content is edited (in milliseconds)
1363     */
1364    public long getMaxIndexWaitTime() {
1365
1366        return m_maxIndexWaitTime;
1367    }
1368
1369    /**
1370     * Returns the maximum number of modifications before a commit in the search index is triggered.<p>
1371     *
1372     * @return the maximum number of modifications before a commit in the search index is triggered
1373     */
1374    public int getMaxModificationsBeforeCommit() {
1375
1376        return m_maxModificationsBeforeCommit;
1377    }
1378
1379    /**
1380     * Returns the update frequency of the offline indexer in milliseconds.<p>
1381     *
1382     * @return the update frequency of the offline indexer in milliseconds
1383     */
1384    public long getOfflineUpdateFrequency() {
1385
1386        return m_offlineUpdateFrequency;
1387    }
1388
1389    /**
1390     * Returns an unmodifiable list of all configured <code>{@link I_CmsSearchIndex}</code> instances.<p>
1391     *
1392     * @return an unmodifiable list of all configured <code>{@link I_CmsSearchIndex}</code> instances
1393     */
1394    public List<I_CmsSearchIndex> getSearchIndexes() {
1395
1396        return Collections.unmodifiableList(m_indexes);
1397    }
1398
1399    /**
1400     * Returns an unmodifiable list of all configured <code>{@link I_CmsSearchIndex}</code> instances.<p>
1401     *
1402     * @return an unmodifiable list of all configured <code>{@link I_CmsSearchIndex}</code> instances
1403     */
1404    public List<I_CmsSearchIndex> getSearchIndexesAll() {
1405
1406        return Collections.unmodifiableList(m_indexes);
1407    }
1408
1409    /**
1410     * Returns an unmodifiable list of all configured <code>{@link I_CmsSearchIndex}</code> instances.<p>
1411     *
1412     * @return an unmodifiable list of all configured <code>{@link I_CmsSearchIndex}</code> instances
1413     */
1414    public List<CmsSolrIndex> getSearchIndexesSolr() {
1415
1416        List<CmsSolrIndex> indexes = new ArrayList<CmsSolrIndex>();
1417        for (I_CmsSearchIndex index : m_indexes) {
1418            if (index instanceof CmsSolrIndex) {
1419                indexes.add((CmsSolrIndex)index);
1420            }
1421        }
1422        return Collections.unmodifiableList(indexes);
1423    }
1424
1425    /**
1426     * Returns an unmodifiable view (read-only) of the SearchIndexSources Map.<p>
1427     *
1428     * @return an unmodifiable view (read-only) of the SearchIndexSources Map
1429     */
1430    public Map<String, CmsSearchIndexSource> getSearchIndexSources() {
1431
1432        return Collections.unmodifiableMap(m_indexSources);
1433    }
1434
1435    /**
1436     * Return singleton instance of the OpenCms spellchecker.<p>
1437     *
1438     * @return instance of CmsSolrSpellchecker.
1439     */
1440    public CmsSolrSpellchecker getSolrDictionary() {
1441
1442        // get the core container that contains one core for each configured index
1443        if (m_coreContainer == null) {
1444            m_coreContainer = createCoreContainer();
1445        }
1446        return CmsSolrSpellchecker.getInstance(m_coreContainer);
1447    }
1448
1449    /**
1450     * Returns the Solr configuration.<p>
1451     *
1452     * @return the Solr configuration
1453     */
1454    public CmsSolrConfiguration getSolrServerConfiguration() {
1455
1456        return m_solrConfig;
1457    }
1458
1459    /**
1460     * Returns the timeout to abandon threads indexing a resource.<p>
1461     *
1462     * @return the timeout to abandon threads indexing a resource
1463     */
1464    public long getTimeout() {
1465
1466        return m_timeout;
1467    }
1468
1469    /**
1470     * Initializes the search manager.<p>
1471     *
1472     * @param cms the cms object
1473     *
1474     * @throws CmsRoleViolationException in case the given opencms object does not have <code>{@link CmsRole#WORKPLACE_MANAGER}</code> permissions
1475     */
1476    public void initialize(CmsObject cms) throws CmsRoleViolationException {
1477
1478        OpenCms.getRoleManager().checkRole(cms, CmsRole.WORKPLACE_MANAGER);
1479        try {
1480            // store the Admin cms to index Cms resources
1481            m_adminCms = OpenCms.initCmsObject(cms);
1482        } catch (CmsException e) {
1483            // this should never happen
1484            LOG.error(e.getLocalizedMessage(), e);
1485        }
1486        // make sure the site root is the root site
1487        m_adminCms.getRequestContext().setSiteRoot("/");
1488
1489        // create the extraction result cache
1490        m_extractionResultCache = new CmsExtractionResultCache(
1491            OpenCms.getSystemInfo().getAbsoluteRfsPathRelativeToWebInf(getDirectory()),
1492            "/extractCache");
1493        initializeFieldConfigurations();
1494        initializeIndexes();
1495        initOfflineIndexes();
1496
1497        // register this object as event listener
1498        OpenCms.addCmsEventListener(
1499            this,
1500            new int[] {
1501                I_CmsEventListener.EVENT_CLEAR_CACHES,
1502                I_CmsEventListener.EVENT_PUBLISH_PROJECT,
1503                I_CmsEventListener.EVENT_REBUILD_SEARCHINDEXES});
1504    }
1505
1506    /**
1507     * Calls {@link I_CmsSearchFieldConfiguration#init()} for all registered field configurations.
1508     */
1509    public void initializeFieldConfigurations() {
1510
1511        for (I_CmsSearchFieldConfiguration config : m_fieldConfigurations.values()) {
1512            config.init();
1513        }
1514
1515    }
1516
1517    /**
1518     * Initializes all configured document types and search indexes.<p>
1519     *
1520     * This methods needs to be called if after a change in the index configuration has been made.
1521     */
1522    public void initializeIndexes() {
1523
1524        initAvailableDocumentTypes();
1525        initSearchIndexes();
1526    }
1527
1528    /**
1529     * Initialize the offline index handler, require after an offline index has been added.<p>
1530     */
1531    public void initOfflineIndexes() {
1532
1533        // check which indexes are configured as offline indexes
1534        List<I_CmsSearchIndex> offlineIndexes = new ArrayList<I_CmsSearchIndex>();
1535        Iterator<I_CmsSearchIndex> i = m_indexes.iterator();
1536        while (i.hasNext()) {
1537            I_CmsSearchIndex index = i.next();
1538            if (I_CmsSearchIndex.REBUILD_MODE_OFFLINE.equals(index.getRebuildMode())) {
1539                // this is an offline index
1540                offlineIndexes.add(index);
1541            }
1542        }
1543        m_offlineIndexes = offlineIndexes;
1544        m_offlineHandler.initialize();
1545
1546    }
1547
1548    /**
1549     * Initializes the spell check index.<p>
1550     *
1551     * @param adminCms the ROOT_ADMIN cms context
1552     */
1553    public void initSpellcheckIndex(CmsObject adminCms) {
1554
1555        if (CmsSpellcheckDictionaryIndexer.updatingIndexNecessesary(adminCms)) {
1556            final CmsSolrSpellchecker spellchecker = OpenCms.getSearchManager().getSolrDictionary();
1557            if (spellchecker != null) {
1558
1559                Runnable initRunner = new Runnable() {
1560
1561                    public void run() {
1562
1563                        try {
1564                            spellchecker.parseAndAddDictionaries(adminCms);
1565                        } catch (CmsRoleViolationException e) {
1566                            LOG.error(e.getLocalizedMessage(), e);
1567                        }
1568                    }
1569                };
1570                new Thread(initRunner).start();
1571            }
1572        }
1573    }
1574
1575    /**
1576     * Returns if the offline indexing is paused.<p>
1577     *
1578     * @return <code>true</code> if the offline indexing is paused
1579     */
1580    public boolean isOfflineIndexingPaused() {
1581
1582        return m_offlineUpdateFrequency == Long.MAX_VALUE;
1583    }
1584
1585    /**
1586     * Updates the indexes from as a scheduled job.<p>
1587     *
1588     * @param cms the OpenCms user context to use when reading resources from the VFS
1589     * @param parameters the parameters for the scheduled job
1590     *
1591     * @throws Exception if something goes wrong
1592     *
1593     * @return the String to write in the scheduler log
1594     *
1595     * @see org.opencms.scheduler.I_CmsScheduledJob#launch(CmsObject, Map)
1596     */
1597    public String launch(CmsObject cms, Map<String, String> parameters) throws Exception {
1598
1599        CmsSearchManager manager = OpenCms.getSearchManager();
1600
1601        I_CmsReport report = null;
1602        boolean writeLog = Boolean.valueOf(parameters.get(JOB_PARAM_WRITELOG)).booleanValue();
1603
1604        if (writeLog) {
1605            report = new CmsLogReport(cms.getRequestContext().getLocale(), CmsSearchManager.class);
1606        }
1607
1608        List<String> updateList = null;
1609        String indexList = parameters.get(JOB_PARAM_INDEXLIST);
1610        if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(indexList)) {
1611            // index list has been provided as job parameter
1612            updateList = new ArrayList<String>();
1613            String[] indexNames = CmsStringUtil.splitAsArray(indexList, '|');
1614            for (int i = 0; i < indexNames.length; i++) {
1615                // check if the index actually exists
1616                if (manager.getIndex(indexNames[i]) != null) {
1617                    updateList.add(indexNames[i]);
1618                } else {
1619                    if (LOG.isWarnEnabled()) {
1620                        LOG.warn(Messages.get().getBundle().key(Messages.LOG_NO_INDEX_WITH_NAME_1, indexNames[i]));
1621                    }
1622                }
1623            }
1624        }
1625
1626        long startTime = System.currentTimeMillis();
1627
1628        if (updateList == null) {
1629            // all indexes need to be updated
1630            manager.rebuildAllIndexes(report);
1631        } else {
1632            // rebuild only the selected indexes
1633            manager.rebuildIndexes(updateList, report);
1634        }
1635
1636        long runTime = System.currentTimeMillis() - startTime;
1637
1638        String finishMessage = Messages.get().getBundle().key(
1639            Messages.LOG_REBUILD_INDEXES_FINISHED_1,
1640            CmsStringUtil.formatRuntime(runTime));
1641
1642        if (LOG.isInfoEnabled()) {
1643            LOG.info(finishMessage);
1644        }
1645        return finishMessage;
1646    }
1647
1648    /**
1649     * Pauses the offline indexing.<p>
1650     * May take some time, because the indexes are updated first.<p>
1651     */
1652    public void pauseOfflineIndexing() {
1653
1654        if (m_offlineUpdateFrequency != Long.MAX_VALUE) {
1655            m_configuredOfflineIndexingFrequency = m_offlineUpdateFrequency;
1656            m_offlineUpdateFrequency = Long.MAX_VALUE;
1657            updateOfflineIndexes(0);
1658        }
1659    }
1660
1661    /**
1662     * Rebuilds (if required creates) all configured indexes.<p>
1663     *
1664     * @param report the report object to write messages (or <code>null</code>)
1665     *
1666     * @throws CmsException if something goes wrong
1667     */
1668    public void rebuildAllIndexes(I_CmsReport report) throws CmsException {
1669
1670        try {
1671            SEARCH_MANAGER_LOCK.lock();
1672
1673            CmsMessageContainer container = null;
1674            for (int i = 0, n = m_indexes.size(); i < n; i++) {
1675                // iterate all configured search indexes
1676                I_CmsSearchIndex searchIndex = m_indexes.get(i);
1677                try {
1678                    // update the index
1679                    updateIndex(searchIndex, report, null);
1680                } catch (CmsException e) {
1681                    container = new CmsMessageContainer(
1682                        Messages.get(),
1683                        Messages.ERR_INDEX_REBUILD_ALL_1,
1684                        new Object[] {searchIndex.getName()});
1685                    LOG.error(
1686                        Messages.get().getBundle().key(Messages.ERR_INDEX_REBUILD_ALL_1, searchIndex.getName()),
1687                        e);
1688                }
1689            }
1690            // clean up the extraction result cache
1691            cleanExtractionCache();
1692            if (container != null) {
1693                // throw stored exception
1694                throw new CmsSearchException(container);
1695            }
1696        } finally {
1697            SEARCH_MANAGER_LOCK.unlock();
1698        }
1699    }
1700
1701    /**
1702     * Rebuilds (if required creates) the index with the given name.<p>
1703     *
1704     * @param indexName the name of the index to rebuild
1705     * @param report the report object to write messages (or <code>null</code>)
1706     *
1707     * @throws CmsException if something goes wrong
1708     */
1709    public void rebuildIndex(String indexName, I_CmsReport report) throws CmsException {
1710
1711        try {
1712            SEARCH_MANAGER_LOCK.lock();
1713            // get the search index by name
1714            I_CmsSearchIndex index = getIndex(indexName);
1715            // update the index
1716            updateIndex(index, report, null);
1717            // clean up the extraction result cache
1718            cleanExtractionCache();
1719        } finally {
1720            SEARCH_MANAGER_LOCK.unlock();
1721        }
1722    }
1723
1724    /**
1725     * Rebuilds (if required creates) the List of indexes with the given name.<p>
1726     *
1727     * @param indexNames the names (String) of the index to rebuild
1728     * @param report the report object to write messages (or <code>null</code>)
1729     *
1730     * @throws CmsException if something goes wrong
1731     */
1732    public void rebuildIndexes(List<String> indexNames, I_CmsReport report) throws CmsException {
1733
1734        try {
1735            SEARCH_MANAGER_LOCK.lock();
1736            Iterator<String> i = indexNames.iterator();
1737            while (i.hasNext()) {
1738                String indexName = i.next();
1739                // get the search index by name
1740                I_CmsSearchIndex index = getIndex(indexName);
1741                if (index != null) {
1742                    // update the index
1743                    updateIndex(index, report, null);
1744                } else {
1745                    if (LOG.isWarnEnabled()) {
1746                        LOG.warn(Messages.get().getBundle().key(Messages.LOG_NO_INDEX_WITH_NAME_1, indexName));
1747                    }
1748                }
1749            }
1750            // clean up the extraction result cache
1751            cleanExtractionCache();
1752        } finally {
1753            SEARCH_MANAGER_LOCK.unlock();
1754        }
1755    }
1756
1757    /**
1758     * Registers a new Solr core for the given index.<p>
1759     *
1760     * @param index the index to register a new Solr core for
1761     *
1762     * @throws CmsConfigurationException if no Solr server is configured
1763     */
1764    @SuppressWarnings("resource")
1765    public void registerSolrIndex(CmsSolrIndex index) throws CmsConfigurationException {
1766
1767        if ((m_solrConfig == null) || !m_solrConfig.isEnabled()) {
1768            // No solr server configured
1769            throw new CmsConfigurationException(Messages.get().container(Messages.ERR_SOLR_NOT_ENABLED_0));
1770        }
1771
1772        if (m_solrConfig.getServerUrl() != null) {
1773            // HTTP Server configured
1774            // TODO Implement multi core support for HTTP server
1775            // @see http://lucidworks.lucidimagination.com/display/solr/Configuring+solr.xml
1776            index.setSolrServer(new Builder().withBaseSolrUrl(m_solrConfig.getServerUrl()).build());
1777        }
1778
1779        // get the core container that contains one core for each configured index
1780        if (m_coreContainer == null) {
1781            m_coreContainer = createCoreContainer();
1782        }
1783
1784        // unload the existing core if it exists to avoid problems with forced unlock.
1785        if (m_coreContainer.getAllCoreNames().contains(index.getCoreName())) {
1786            m_coreContainer.unload(index.getCoreName(), false, false, true);
1787        }
1788        // ensure that all locks on the index are gone
1789        ensureIndexIsUnlocked(index.getPath());
1790
1791        // load the core to the container
1792        File dataDir = new File(index.getPath());
1793        if (!dataDir.exists()) {
1794            dataDir.mkdirs();
1795            if (CmsLog.INIT.isInfoEnabled()) {
1796                CmsLog.INIT.info(
1797                    Messages.get().getBundle().key(
1798                        Messages.INIT_SOLR_INDEX_DIR_CREATED_2,
1799                        index.getName(),
1800                        index.getPath()));
1801            }
1802        }
1803        File instanceDir = new File(m_solrConfig.getHome() + FileSystems.getDefault().getSeparator() + index.getName());
1804        if (!instanceDir.exists()) {
1805            instanceDir.mkdirs();
1806            if (CmsLog.INIT.isInfoEnabled()) {
1807                CmsLog.INIT.info(
1808                    Messages.get().getBundle().key(
1809                        Messages.INIT_SOLR_INDEX_DIR_CREATED_2,
1810                        index.getName(),
1811                        index.getPath()));
1812            }
1813        }
1814
1815        // create the core
1816        // TODO: suboptimal - forces always the same schema
1817        SolrCore core = null;
1818        try {
1819            // creation includes registration.
1820            // TODO: this was the old code: core = m_coreContainer.create(descriptor, false);
1821            Map<String, String> properties = new HashMap<String, String>(3);
1822            properties.put(CoreDescriptor.CORE_DATADIR, dataDir.getAbsolutePath());
1823            properties.put(CoreDescriptor.CORE_CONFIGSET, "default");
1824            core = m_coreContainer.create(index.getCoreName(), instanceDir.toPath(), properties, false);
1825        } catch (NullPointerException e) {
1826            if (core != null) {
1827                core.close();
1828            }
1829            throw new CmsConfigurationException(
1830                Messages.get().container(
1831                    Messages.ERR_SOLR_SERVER_NOT_CREATED_3,
1832                    index.getName() + " (" + index.getCoreName() + ")",
1833                    index.getPath(),
1834                    m_solrConfig.getSolrConfigFile().getAbsolutePath()),
1835                e);
1836        }
1837
1838        if (index.isNoSolrServerSet()) {
1839            index.setSolrServer(new EmbeddedSolrServer(m_coreContainer, index.getCoreName()));
1840        }
1841        if (CmsLog.INIT.isInfoEnabled()) {
1842            CmsLog.INIT.info(
1843                Messages.get().getBundle().key(
1844                    Messages.INIT_SOLR_SERVER_CREATED_1,
1845                    index.getName() + " (" + index.getCoreName() + ")"));
1846        }
1847    }
1848
1849    /**
1850     * Removes this field configuration from the OpenCms configuration (if it is not used any more).<p>
1851     *
1852     * @param fieldConfiguration the field configuration to remove from the configuration
1853     *
1854     * @return true if remove was successful, false if preconditions for removal are ok but the given
1855     *         field configuration was unknown to the manager.
1856     *
1857     * @throws CmsIllegalStateException if the given field configuration is still used by at least one
1858     *         <code>{@link I_CmsSearchIndex}</code>.
1859     *
1860     */
1861    public boolean removeSearchFieldConfiguration(I_CmsSearchFieldConfiguration fieldConfiguration)
1862    throws CmsIllegalStateException {
1863
1864        // never remove the standard field configuration
1865        if (fieldConfiguration.getName().equals(CmsSearchFieldConfiguration.STR_STANDARD)) {
1866            throw new CmsIllegalStateException(
1867                Messages.get().container(
1868                    Messages.ERR_INDEX_CONFIGURATION_DELETE_STANDARD_1,
1869                    fieldConfiguration.getName()));
1870        }
1871        // validation if removal will be granted
1872        Iterator<I_CmsSearchIndex> itIndexes = m_indexes.iterator();
1873        I_CmsSearchIndex idx;
1874        // the list for collecting indexes that use the given field configuration
1875        List<I_CmsSearchIndex> referrers = new ArrayList<I_CmsSearchIndex>();
1876        I_CmsSearchFieldConfiguration refFieldConfig;
1877        while (itIndexes.hasNext()) {
1878            idx = itIndexes.next();
1879            refFieldConfig = idx.getFieldConfiguration();
1880            if (refFieldConfig.equals(fieldConfiguration)) {
1881                referrers.add(idx);
1882            }
1883        }
1884        if (referrers.size() > 0) {
1885            throw new CmsIllegalStateException(
1886                Messages.get().container(
1887                    Messages.ERR_INDEX_CONFIGURATION_DELETE_2,
1888                    fieldConfiguration.getName(),
1889                    referrers.toString()));
1890        }
1891
1892        // remove operation (no exception)
1893        return m_fieldConfigurations.remove(fieldConfiguration.getName()) != null;
1894
1895    }
1896
1897    /**
1898     * Removes a search field from the field configuration.<p>
1899     *
1900     * @param fieldConfiguration the field configuration
1901     * @param field field to remove from the field configuration
1902     *
1903     * @return true if remove was successful, false if preconditions for removal are ok but the given
1904     *         field was unknown.
1905     */
1906    public boolean removeSearchFieldConfigurationField(
1907        I_CmsSearchFieldConfiguration fieldConfiguration,
1908        CmsSearchField field) {
1909
1910        if (LOG.isInfoEnabled()) {
1911            LOG.info(
1912                Messages.get().getBundle().key(
1913                    Messages.LOG_REMOVE_FIELDCONFIGURATION_FIELD_INDEX_2,
1914                    field.getName(),
1915                    fieldConfiguration.getName()));
1916        }
1917
1918        return fieldConfiguration.getFields().remove(field);
1919    }
1920
1921    /**
1922     * Removes a search field mapping from the given field.<p>
1923     *
1924     * @param field the field
1925     * @param mapping mapping to remove from the field
1926     *
1927     * @return true if remove was successful, false if preconditions for removal are ok but the given
1928     *         mapping was unknown.
1929     *
1930     * @throws CmsIllegalStateException if the given mapping is the last mapping inside the given field.
1931     */
1932    public boolean removeSearchFieldMapping(CmsLuceneField field, CmsSearchFieldMapping mapping)
1933    throws CmsIllegalStateException {
1934
1935        if (field.getMappings().size() < 2) {
1936            throw new CmsIllegalStateException(
1937                Messages.get().container(
1938                    Messages.ERR_FIELD_MAPPING_DELETE_2,
1939                    mapping.getType().toString(),
1940                    field.getName()));
1941        } else {
1942
1943            if (LOG.isInfoEnabled()) {
1944                LOG.info(
1945                    Messages.get().getBundle().key(
1946                        Messages.LOG_REMOVE_FIELD_MAPPING_INDEX_2,
1947                        mapping.toString(),
1948                        field.getName()));
1949            }
1950            return field.getMappings().remove(mapping);
1951        }
1952    }
1953
1954    /**
1955     * Removes a search index from the configuration.<p>
1956     *
1957     * @param searchIndex the search index to remove
1958     */
1959    public void removeSearchIndex(I_CmsSearchIndex searchIndex) {
1960
1961        // shut down index to remove potential config files of Solr indexes
1962        searchIndex.shutDown();
1963        if (searchIndex instanceof CmsSolrIndex) {
1964            CmsSolrIndex solrIndex = (CmsSolrIndex)searchIndex;
1965            m_coreContainer.unload(solrIndex.getCoreName(), true, true, true);
1966        }
1967        m_indexes.remove(searchIndex);
1968        initOfflineIndexes();
1969
1970        if (LOG.isInfoEnabled()) {
1971            LOG.info(
1972                Messages.get().getBundle().key(
1973                    Messages.LOG_REMOVE_SEARCH_INDEX_2,
1974                    searchIndex.getName(),
1975                    searchIndex.getProject()));
1976        }
1977    }
1978
1979    /**
1980     * Removes all indexes included in the given list (which must contain the name of an index to remove).<p>
1981     *
1982     * @param indexNames the names of the index to remove
1983     */
1984    public void removeSearchIndexes(List<String> indexNames) {
1985
1986        Iterator<String> i = indexNames.iterator();
1987        while (i.hasNext()) {
1988            String indexName = i.next();
1989            // get the search index by name
1990            I_CmsSearchIndex index = getIndex(indexName);
1991            if (index != null) {
1992                // remove the index
1993                removeSearchIndex(index);
1994            } else {
1995                if (LOG.isWarnEnabled()) {
1996                    LOG.warn(Messages.get().getBundle().key(Messages.LOG_NO_INDEX_WITH_NAME_1, indexName));
1997                }
1998            }
1999        }
2000    }
2001
2002    /**
2003     * Removes this indexsource from the OpenCms configuration (if it is not used any more).<p>
2004     *
2005     * @param indexsource the indexsource to remove from the configuration
2006     *
2007     * @return true if remove was successful, false if preconditions for removal are ok but the given
2008     *         searchindex was unknown to the manager.
2009     *
2010     * @throws CmsIllegalStateException if the given indexsource is still used by at least one
2011     *         <code>{@link I_CmsSearchIndex}</code>.
2012     *
2013     */
2014    public boolean removeSearchIndexSource(CmsSearchIndexSource indexsource) throws CmsIllegalStateException {
2015
2016        // validation if removal will be granted
2017        Iterator<I_CmsSearchIndex> itIndexes = m_indexes.iterator();
2018        I_CmsSearchIndex idx;
2019        // the list for collecting indexes that use the given index source
2020        List<I_CmsSearchIndex> referrers = new ArrayList<I_CmsSearchIndex>();
2021        // the current list of referred index sources of the iterated index
2022        List<CmsSearchIndexSource> refsources;
2023        while (itIndexes.hasNext()) {
2024            idx = itIndexes.next();
2025            refsources = idx.getSources();
2026            if (refsources != null) {
2027                if (refsources.contains(indexsource)) {
2028                    referrers.add(idx);
2029                }
2030            }
2031        }
2032        if (referrers.size() > 0) {
2033            throw new CmsIllegalStateException(
2034                Messages.get().container(
2035                    Messages.ERR_INDEX_SOURCE_DELETE_2,
2036                    indexsource.getName(),
2037                    referrers.toString()));
2038        }
2039
2040        // remove operation (no exception)
2041        return m_indexSources.remove(indexsource.getName()) != null;
2042
2043    }
2044
2045    /**
2046     * Resumes offline indexing if it was paused.<p>
2047     */
2048    public void resumeOfflineIndexing() {
2049
2050        if (m_offlineUpdateFrequency == Long.MAX_VALUE) {
2051            setOfflineUpdateFrequency(
2052                m_configuredOfflineIndexingFrequency > 0
2053                ? m_configuredOfflineIndexingFrequency
2054                : DEFAULT_OFFLINE_UPDATE_FREQNENCY);
2055        }
2056    }
2057
2058    /**
2059     * Sets the name of the directory below WEB-INF/ where the search indexes are stored.<p>
2060     *
2061     * @param value the name of the directory below WEB-INF/ where the search indexes are stored
2062     */
2063    public void setDirectory(String value) {
2064
2065        m_path = value;
2066    }
2067
2068    /**
2069     * Sets the maximum age a text extraction result is kept in the cache (in hours).<p>
2070     *
2071     * @param extractionCacheMaxAge the maximum age for a text extraction result to set
2072     */
2073    public void setExtractionCacheMaxAge(float extractionCacheMaxAge) {
2074
2075        m_extractionCacheMaxAge = extractionCacheMaxAge;
2076    }
2077
2078    /**
2079     * Sets the maximum age a text extraction result is kept in the cache (in hours) as a String.<p>
2080     *
2081     * @param extractionCacheMaxAge the maximum age for a text extraction result to set
2082     */
2083    public void setExtractionCacheMaxAge(String extractionCacheMaxAge) {
2084
2085        try {
2086            setExtractionCacheMaxAge(Float.parseFloat(extractionCacheMaxAge));
2087        } catch (NumberFormatException e) {
2088            LOG.error(
2089                Messages.get().getBundle().key(
2090                    Messages.LOG_PARSE_EXTRACTION_CACHE_AGE_FAILED_2,
2091                    extractionCacheMaxAge,
2092                    new Float(DEFAULT_EXTRACTION_CACHE_MAX_AGE)),
2093                e);
2094            setExtractionCacheMaxAge(DEFAULT_EXTRACTION_CACHE_MAX_AGE);
2095        }
2096    }
2097
2098    /**
2099     * Sets the unlock mode during indexing.<p>
2100     *
2101     * @param value the value
2102     */
2103    public void setForceunlock(String value) {
2104
2105        m_forceUnlockMode = CmsSearchForceUnlockMode.valueOf(value);
2106    }
2107
2108    /**
2109     * Sets the highlighter.<p>
2110     *
2111     * A highlighter is a class implementing org.opencms.search.documents.I_TermHighlighter.<p>
2112     *
2113     * @param highlighter the package/class name of the highlighter
2114     */
2115    public void setHighlighter(String highlighter) {
2116
2117        try {
2118            m_highlighter = (I_CmsTermHighlighter)Class.forName(highlighter).newInstance();
2119        } catch (Exception e) {
2120            m_highlighter = null;
2121            LOG.error(e.getLocalizedMessage(), e);
2122        }
2123    }
2124
2125    /**
2126     * Sets the seconds to wait for an index lock during an update operation.<p>
2127     *
2128     * @param value the seconds to wait for an index lock during an update operation
2129     */
2130    public void setIndexLockMaxWaitSeconds(int value) {
2131
2132        m_indexLockMaxWaitSeconds = value;
2133    }
2134
2135    /**
2136     * Sets the max. excerpt length.<p>
2137     *
2138     * @param maxExcerptLength the max. excerpt length to set
2139     */
2140    public void setMaxExcerptLength(int maxExcerptLength) {
2141
2142        m_maxExcerptLength = maxExcerptLength;
2143    }
2144
2145    /**
2146     * Sets the max. excerpt length as a String.<p>
2147     *
2148     * @param maxExcerptLength the max. excerpt length to set
2149     */
2150    public void setMaxExcerptLength(String maxExcerptLength) {
2151
2152        try {
2153            setMaxExcerptLength(Integer.parseInt(maxExcerptLength));
2154        } catch (Exception e) {
2155            LOG.error(
2156                Messages.get().getBundle().key(
2157                    Messages.LOG_PARSE_EXCERPT_LENGTH_FAILED_2,
2158                    maxExcerptLength,
2159                    new Integer(DEFAULT_EXCERPT_LENGTH)),
2160                e);
2161            setMaxExcerptLength(DEFAULT_EXCERPT_LENGTH);
2162        }
2163    }
2164
2165    /**
2166     * Sets the maximal wait time for offline index updates after edit operations.<p>
2167     *
2168     * @param maxIndexWaitTime  the maximal wait time to set in milliseconds
2169     */
2170    public void setMaxIndexWaitTime(long maxIndexWaitTime) {
2171
2172        m_maxIndexWaitTime = maxIndexWaitTime;
2173    }
2174
2175    /**
2176     * Sets the maximal wait time for offline index updates after edit operations.<p>
2177     *
2178     * @param maxIndexWaitTime the maximal wait time to set in milliseconds
2179     */
2180    public void setMaxIndexWaitTime(String maxIndexWaitTime) {
2181
2182        try {
2183            setMaxIndexWaitTime(Long.parseLong(maxIndexWaitTime));
2184        } catch (Exception e) {
2185            LOG.error(
2186                Messages.get().getBundle().key(
2187                    Messages.LOG_PARSE_MAX_INDEX_WAITTIME_FAILED_2,
2188                    maxIndexWaitTime,
2189                    new Long(DEFAULT_MAX_INDEX_WAITTIME)),
2190                e);
2191            setMaxIndexWaitTime(DEFAULT_MAX_INDEX_WAITTIME);
2192        }
2193    }
2194
2195    /**
2196     * Sets the maximum number of modifications before a commit in the search index is triggered.<p>
2197     *
2198     * @param maxModificationsBeforeCommit the maximum number of modifications to set
2199     */
2200    public void setMaxModificationsBeforeCommit(int maxModificationsBeforeCommit) {
2201
2202        m_maxModificationsBeforeCommit = maxModificationsBeforeCommit;
2203    }
2204
2205    /**
2206     * Sets the maximum number of modifications before a commit in the search index is triggered as a string.<p>
2207     *
2208     * @param value the maximum number of modifications to set
2209     */
2210    public void setMaxModificationsBeforeCommit(String value) {
2211
2212        try {
2213            setMaxModificationsBeforeCommit(Integer.parseInt(value));
2214        } catch (Exception e) {
2215            LOG.error(
2216                Messages.get().getBundle().key(
2217                    Messages.LOG_PARSE_MAXCOMMIT_FAILED_2,
2218                    value,
2219                    new Integer(DEFAULT_MAX_MODIFICATIONS_BEFORE_COMMIT)),
2220                e);
2221            setMaxModificationsBeforeCommit(DEFAULT_MAX_MODIFICATIONS_BEFORE_COMMIT);
2222        }
2223    }
2224
2225    /**
2226     * Sets the update frequency of the offline indexer in milliseconds.<p>
2227     *
2228     * @param offlineUpdateFrequency the update frequency in milliseconds to set
2229     */
2230    public void setOfflineUpdateFrequency(long offlineUpdateFrequency) {
2231
2232        m_offlineUpdateFrequency = offlineUpdateFrequency;
2233        updateOfflineIndexes(0);
2234    }
2235
2236    /**
2237     * Sets the update frequency of the offline indexer in milliseconds.<p>
2238     *
2239     * @param offlineUpdateFrequency the update frequency in milliseconds to set
2240     */
2241    public void setOfflineUpdateFrequency(String offlineUpdateFrequency) {
2242
2243        try {
2244            setOfflineUpdateFrequency(Long.parseLong(offlineUpdateFrequency));
2245        } catch (Exception e) {
2246            LOG.error(
2247                Messages.get().getBundle().key(
2248                    Messages.LOG_PARSE_OFFLINE_UPDATE_FAILED_2,
2249                    offlineUpdateFrequency,
2250                    new Long(DEFAULT_OFFLINE_UPDATE_FREQNENCY)),
2251                e);
2252            setOfflineUpdateFrequency(DEFAULT_OFFLINE_UPDATE_FREQNENCY);
2253        }
2254    }
2255
2256    /**
2257     * Sets the Solr configuration.<p>
2258     *
2259     * @param config the Solr configuration
2260     */
2261    public void setSolrServerConfiguration(CmsSolrConfiguration config) {
2262
2263        m_solrConfig = config;
2264    }
2265
2266    /**
2267     * Sets the timeout to abandon threads indexing a resource.<p>
2268     *
2269     * @param value the timeout in milliseconds
2270     */
2271    public void setTimeout(long value) {
2272
2273        m_timeout = value;
2274    }
2275
2276    /**
2277     * Sets the timeout to abandon threads indexing a resource as a String.<p>
2278     *
2279     * @param value the timeout in milliseconds
2280     */
2281    public void setTimeout(String value) {
2282
2283        try {
2284            setTimeout(Long.parseLong(value));
2285        } catch (Exception e) {
2286            LOG.error(
2287                Messages.get().getBundle().key(Messages.LOG_PARSE_TIMEOUT_FAILED_2, value, new Long(DEFAULT_TIMEOUT)),
2288                e);
2289            setTimeout(DEFAULT_TIMEOUT);
2290        }
2291    }
2292
2293    /**
2294     * Shuts down the search manager.<p>
2295     *
2296     * This will cause all search indices to be shut down.<p>
2297     */
2298    public void shutDown() {
2299
2300        if (m_offlineIndexThread != null) {
2301            m_offlineIndexThread.shutDown();
2302        }
2303
2304        if (m_offlineHandler != null) {
2305            OpenCms.removeCmsEventListener(m_offlineHandler);
2306        }
2307
2308        Iterator<I_CmsSearchIndex> i = m_indexes.iterator();
2309        while (i.hasNext()) {
2310            I_CmsSearchIndex index = i.next();
2311            index.shutDown();
2312            index = null;
2313        }
2314        m_indexes.clear();
2315
2316        shutDownSolrContainer();
2317
2318        if (CmsLog.INIT.isInfoEnabled()) {
2319            CmsLog.INIT.info(Messages.get().getBundle().key(Messages.INIT_SHUTDOWN_MANAGER_0));
2320        }
2321    }
2322
2323    /**
2324     * Updates all offline indexes.<p>
2325     *
2326     * Can be used to force an index update when it's not convenient to wait until the
2327     * offline update interval has eclipsed.<p>
2328     *
2329     * Since the offline indexes still need some time to update the new resources,
2330     * the method waits for at most the configurable <code>maxIndexWaitTime</code>
2331     * to ensure that updating is finished.
2332     *
2333     * @see #updateOfflineIndexes(long)
2334     *
2335     */
2336    public void updateOfflineIndexes() {
2337
2338        updateOfflineIndexes(getMaxIndexWaitTime());
2339    }
2340
2341    /**
2342     * Updates all offline indexes.<p>
2343     *
2344     * Can be used to force an index update when it's not convenient to wait until the
2345     * offline update interval has eclipsed.<p>
2346     *
2347     * Since the offline index will still need some time to update the new resources even if it runs directly,
2348     * a wait time of 2500 or so should be given in order to make sure the index finished updating.
2349     *
2350     * @param waitTime milliseconds to wait after the offline update index was notified of the changes
2351     */
2352    public void updateOfflineIndexes(long waitTime) {
2353
2354        if ((m_offlineIndexThread != null) && m_offlineIndexThread.isAlive()) {
2355            // notify existing thread of update frequency change
2356            if (LOG.isDebugEnabled()) {
2357                LOG.debug(Messages.get().getBundle().key(Messages.LOG_OI_UPDATE_INTERRUPT_0));
2358            }
2359            m_offlineIndexThread.interrupt();
2360            if (waitTime > 0) {
2361                m_offlineIndexThread.getWaitHandle().enter(waitTime);
2362            }
2363        }
2364    }
2365
2366    /**
2367     * Collects the resources whose indexed document depends on one of the updated resources.<p>
2368     * We take transitive dependencies into account and handle cyclic dependencies correctly as well.
2369     *
2370     * @param adminCms an OpenCms user context with Admin permissions
2371     * @param updateResources the resources to be re-indexed
2372     *
2373     * @return the updated list of resource to re-index
2374     */
2375    protected List<CmsPublishedResource> addAdditionallyAffectedResources(
2376        CmsObject adminCms,
2377        List<CmsPublishedResource> updateResources) {
2378
2379        Set<CmsPublishedResource> updateResourceSet = new HashSet<>(updateResources);
2380        Collection<CmsPublishedResource> resourcesToCheck = updateResourceSet;
2381        Collection<CmsPublishedResource> additionalResources = Collections.emptySet();
2382        do {
2383            additionalResources = findRelatedContainerPages(adminCms, updateResourceSet, resourcesToCheck);
2384            additionalResources.addAll(addIndexContentRelatedResources(adminCms, updateResourceSet, resourcesToCheck));
2385            updateResources.addAll(additionalResources);
2386            updateResourceSet.addAll(additionalResources);
2387            resourcesToCheck = additionalResources;
2388        } while (resourcesToCheck.size() > 0);
2389        return updateResources;
2390    }
2391
2392    /**
2393     * Collects the resources whose indexed document depends on one of the updated resources.<p>
2394     *
2395     * @param adminCms an OpenCms user context with Admin permissions
2396     * @param updateResources the resources to be re-indexed
2397     * @param updateResourcesToCheck the resources to check additionally affected resources for, subset of updateResources
2398     *
2399     * @return the list of resources that need to be additionally re-index
2400     */
2401    protected Collection<CmsPublishedResource> addIndexContentRelatedResources(
2402        CmsObject adminCms,
2403        Collection<CmsPublishedResource> updateResources,
2404        Collection<CmsPublishedResource> updateResourcesToCheck) {
2405
2406        Collection<CmsPublishedResource> additionalUpdateResources = new HashSet<>();
2407        for (CmsPublishedResource checkedRes : updateResourcesToCheck) {
2408            try {
2409                CmsRelationFilter filter = CmsRelationFilter.relationsToStructureId(checkedRes.getStructureId());
2410                filter = filter.filterType(CmsRelationType.INDEX_CONTENT);
2411                List<CmsRelation> relations = adminCms.readRelations(filter);
2412                for (CmsRelation relation : relations) {
2413                    CmsResource res = relation.getSource(adminCms, CmsResourceFilter.ALL);
2414                    CmsPublishedResource additionalPubRes = new CmsPublishedResource(res);
2415                    if (!updateResources.contains(additionalPubRes)) {
2416                        additionalUpdateResources.add(additionalPubRes);
2417                    }
2418                }
2419            } catch (CmsException e) {
2420                LOG.error(e.getLocalizedMessage(), e);
2421            }
2422        }
2423        return additionalUpdateResources;
2424    }
2425
2426    /**
2427     * Cleans up the extraction result cache.<p>
2428     */
2429    protected void cleanExtractionCache() {
2430
2431        // clean up the extraction result cache
2432        m_extractionResultCache.cleanCache(m_extractionCacheMaxAge);
2433    }
2434
2435    /**
2436     * Collects the related containerpages to the resources that have been published.<p>
2437     *
2438     * @param adminCms an OpenCms user context with Admin permissions
2439     * @param updateResources the resources to be re-indexed
2440     * @param updateResourcesToCheck the resources to check additionally affected resources for, subset of updateResources
2441     *
2442     * @return the list of resources that need to be additionally re-index
2443     */
2444    protected Collection<CmsPublishedResource> findRelatedContainerPages(
2445        CmsObject adminCms,
2446        Collection<CmsPublishedResource> updateResources,
2447        Collection<CmsPublishedResource> updateResourcesToCheck) {
2448
2449        Collection<CmsPublishedResource> additionalUpdateResources = new HashSet<>();
2450
2451        Set<CmsResource> elementGroups = new HashSet<CmsResource>();
2452        Set<CmsResource> containerPages = new HashSet<CmsResource>();
2453        int containerPageTypeId = -1;
2454        try {
2455            containerPageTypeId = CmsResourceTypeXmlContainerPage.getContainerPageTypeId();
2456        } catch (CmsLoaderException e) {
2457            // will happen during setup, when container page type is not available yet
2458            LOG.info(e.getLocalizedMessage(), e);
2459        }
2460        if (containerPageTypeId != -1) {
2461            for (CmsPublishedResource pubRes : updateResourcesToCheck) {
2462                try {
2463                    if (OpenCms.getResourceManager().getResourceType(
2464                        pubRes.getType()) instanceof CmsResourceTypeXmlContent) {
2465                        CmsRelationFilter filter = CmsRelationFilter.relationsToStructureId(pubRes.getStructureId());
2466                        filter.filterStrong();
2467                        List<CmsRelation> relations = adminCms.readRelations(filter);
2468                        for (CmsRelation relation : relations) {
2469                            CmsResource res = relation.getSource(adminCms, CmsResourceFilter.ALL);
2470                            if (CmsResourceTypeXmlContainerPage.isContainerPage(res)) {
2471                                containerPages.add(res);
2472                                if (CmsDetailOnlyContainerUtil.isDetailContainersPage(
2473                                    adminCms,
2474                                    adminCms.getSitePath(res))) {
2475                                    addDetailContent(adminCms, containerPages, adminCms.getSitePath(res));
2476                                }
2477                            } else if (OpenCms.getResourceManager().getResourceType(
2478                                res.getTypeId()).getTypeName().equals(
2479                                    CmsResourceTypeXmlContainerPage.GROUP_CONTAINER_TYPE_NAME)) {
2480                                elementGroups.add(res);
2481                            }
2482                        }
2483                    }
2484                    if (containerPageTypeId == pubRes.getType()) {
2485                        addDetailContent(
2486                            adminCms,
2487                            containerPages,
2488                            adminCms.getRequestContext().removeSiteRoot(pubRes.getRootPath()));
2489                    }
2490                } catch (CmsException e) {
2491                    LOG.error(e.getLocalizedMessage(), e);
2492                }
2493            }
2494            for (CmsResource pubRes : elementGroups) {
2495                try {
2496                    CmsRelationFilter filter = CmsRelationFilter.relationsToStructureId(pubRes.getStructureId());
2497                    filter.filterStrong();
2498                    List<CmsRelation> relations = adminCms.readRelations(filter);
2499                    for (CmsRelation relation : relations) {
2500                        CmsResource res = relation.getSource(adminCms, CmsResourceFilter.ALL);
2501                        if (CmsResourceTypeXmlContainerPage.isContainerPage(res)) {
2502                            containerPages.add(res);
2503                            if (CmsDetailOnlyContainerUtil.isDetailContainersPage(
2504                                adminCms,
2505                                adminCms.getSitePath(res))) {
2506                                addDetailContent(adminCms, containerPages, adminCms.getSitePath(res));
2507                            }
2508                        }
2509                    }
2510                } catch (CmsException e) {
2511                    LOG.error(e.getLocalizedMessage(), e);
2512                }
2513            }
2514            // add all found container pages as published resource objects to the list
2515            for (CmsResource page : containerPages) {
2516                CmsPublishedResource pubCont = new CmsPublishedResource(page);
2517                if (!updateResources.contains(pubCont)) {
2518                    // ensure container page is added only once
2519                    additionalUpdateResources.add(pubCont);
2520                }
2521            }
2522        }
2523        return additionalUpdateResources;
2524    }
2525
2526    /**
2527     * Returns the set of names of all configured document types.<p>
2528     *
2529     * @return the set of names of all configured document types
2530     */
2531    protected List<String> getDocumentTypes() {
2532
2533        List<String> names = new ArrayList<String>();
2534        for (Iterator<I_CmsDocumentFactory> i = m_documentTypes.values().iterator(); i.hasNext();) {
2535            I_CmsDocumentFactory factory = i.next();
2536            names.add(factory.getName());
2537        }
2538        return names;
2539    }
2540
2541    /**
2542     * Returns the a offline project used for offline indexing.<p>
2543     *
2544     * @return the offline project if available
2545     */
2546    protected CmsProject getOfflineIndexProject() {
2547
2548        CmsProject result = null;
2549        for (I_CmsSearchIndex index : m_offlineIndexes) {
2550            try {
2551                result = m_adminCms.readProject(index.getProject());
2552
2553                if (!result.isOnlineProject()) {
2554                    break;
2555                }
2556            } catch (Exception e) {
2557                // may be a missconfigured index, ignore
2558                LOG.error(e.getLocalizedMessage(), e);
2559            }
2560        }
2561        return result;
2562    }
2563
2564    /**
2565     * Returns a new thread manager for the indexing threads.<p>
2566     *
2567     * @return a new thread manager for the indexing threads
2568     */
2569    protected CmsIndexingThreadManager getThreadManager() {
2570
2571        return new CmsIndexingThreadManager(m_timeout, m_maxModificationsBeforeCommit);
2572    }
2573
2574    /**
2575     * Initializes the available Cms resource types to be indexed.<p>
2576     *
2577     * A map stores document factories keyed by a string representing
2578     * a colon separated list of Cms resource types and/or mimetypes.<p>
2579     *
2580     * The keys of this map are used to trigger a document factory to convert
2581     * a Cms resource into a Lucene index document.<p>
2582     *
2583     * A document factory is a class implementing the interface
2584     * {@link org.opencms.search.documents.I_CmsDocumentFactory}.<p>
2585     */
2586    protected void initAvailableDocumentTypes() {
2587
2588        CmsSearchDocumentType documenttype = null;
2589        String className = null;
2590        String name = null;
2591        I_CmsDocumentFactory documentFactory = null;
2592        List<String> resourceTypes = null;
2593        List<String> mimeTypes = null;
2594        Class<?> c = null;
2595
2596        m_documentTypes = new HashMap<String, I_CmsDocumentFactory>();
2597
2598        for (int i = 0, n = m_documentTypeConfigs.size(); i < n; i++) {
2599
2600            documenttype = m_documentTypeConfigs.get(i);
2601            name = documenttype.getName();
2602
2603            try {
2604                className = documenttype.getClassName();
2605                resourceTypes = documenttype.getResourceTypes();
2606                mimeTypes = documenttype.getMimeTypes();
2607
2608                if (name == null) {
2609                    throw new CmsIndexException(Messages.get().container(Messages.ERR_DOCTYPE_NO_NAME_0));
2610                }
2611                if (className == null) {
2612                    throw new CmsIndexException(Messages.get().container(Messages.ERR_DOCTYPE_NO_CLASS_DEF_0));
2613                }
2614                if (resourceTypes.size() == 0) {
2615                    throw new CmsIndexException(Messages.get().container(Messages.ERR_DOCTYPE_NO_RESOURCETYPE_DEF_0));
2616                }
2617
2618                try {
2619                    c = Class.forName(className);
2620                    documentFactory = (I_CmsDocumentFactory)c.getConstructor(new Class[] {String.class}).newInstance(
2621                        new Object[] {name});
2622                } catch (ClassNotFoundException exc) {
2623                    throw new CmsIndexException(
2624                        Messages.get().container(Messages.ERR_DOCCLASS_NOT_FOUND_1, className),
2625                        exc);
2626                } catch (Exception exc) {
2627                    throw new CmsIndexException(Messages.get().container(Messages.ERR_DOCCLASS_INIT_1, className), exc);
2628                }
2629
2630                if (documentFactory.isUsingCache()) {
2631                    // init cache if used by the factory
2632                    documentFactory.setCache(m_extractionResultCache);
2633                }
2634
2635                for (Iterator<String> key = documentFactory.getDocumentKeys(
2636                    resourceTypes,
2637                    mimeTypes).iterator(); key.hasNext();) {
2638                    m_documentTypes.put(key.next(), documentFactory);
2639                }
2640
2641            } catch (CmsException e) {
2642                if (LOG.isWarnEnabled()) {
2643                    LOG.warn(Messages.get().getBundle().key(Messages.LOG_DOCTYPE_CONFIG_FAILED_1, name), e);
2644                }
2645            }
2646        }
2647    }
2648
2649    /**
2650     * Initializes the configured search indexes.<p>
2651     *
2652     * This initializes also the list of Cms resources types
2653     * to be indexed by an index source.<p>
2654     */
2655    protected void initSearchIndexes() {
2656
2657        I_CmsSearchIndex index = null;
2658        for (int i = 0, n = m_indexes.size(); i < n; i++) {
2659            index = m_indexes.get(i);
2660            // reset disabled flag
2661            index.setEnabled(true);
2662            // check if the index has been configured correctly
2663            if (index.checkConfiguration(m_adminCms)) {
2664                // the index is configured correctly
2665                try {
2666                    index.initialize();
2667                } catch (Exception e) {
2668                    if (CmsLog.INIT.isWarnEnabled()) {
2669                        // in this case the index will be disabled
2670                        CmsLog.INIT.warn(Messages.get().getBundle().key(Messages.INIT_SEARCH_INIT_FAILED_1, index), e);
2671                    }
2672                }
2673            }
2674            // output a log message if the index was successfully configured or not
2675            if (CmsLog.INIT.isInfoEnabled()) {
2676                if (index.isEnabled()) {
2677                    CmsLog.INIT.info(
2678                        Messages.get().getBundle().key(Messages.INIT_INDEX_CONFIGURED_2, index, index.getProject()));
2679                } else {
2680                    CmsLog.INIT.warn(
2681                        Messages.get().getBundle().key(
2682                            Messages.INIT_INDEX_NOT_CONFIGURED_2,
2683                            index,
2684                            index.getProject()));
2685                }
2686            }
2687        }
2688    }
2689
2690    /**
2691     * Checks, if the index should be rebuilt/updated at all by the search manager.
2692     * @param index the index to check.
2693     * @return a flag, indicating if the index should be rebuilt/updated at all.
2694     */
2695    protected boolean shouldUpdateAtAll(I_CmsSearchIndex index) {
2696
2697        if (I_CmsSearchIndex.REBUILD_MODE_NEVER.equals(index.getRebuildMode())) {
2698            LOG.debug(Messages.get().getBundle().key(Messages.LOG_SKIP_REBUILD_FOR_MODE_NEVER_1, index.getName()));
2699            return false;
2700        } else {
2701            return true;
2702        }
2703
2704    }
2705
2706    /**
2707     * Incrementally updates all indexes that have their rebuild mode set to <code>"auto"</code>
2708     * after resources have been published.<p>
2709     *
2710     * @param adminCms an OpenCms user context with Admin permissions
2711     * @param publishHistoryId the history ID of the published project
2712     * @param report the report to write the output to
2713     */
2714    protected void updateAllIndexes(CmsObject adminCms, CmsUUID publishHistoryId, I_CmsReport report) {
2715
2716        int oldPriority = Thread.currentThread().getPriority();
2717        try {
2718            SEARCH_MANAGER_LOCK.lock();
2719            Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
2720            List<CmsPublishedResource> publishedResources;
2721            try {
2722                // read the list of all published resources
2723                publishedResources = adminCms.readPublishedResources(publishHistoryId);
2724            } catch (CmsException e) {
2725                LOG.error(
2726                    Messages.get().getBundle().key(Messages.LOG_READING_CHANGED_RESOURCES_FAILED_1, publishHistoryId),
2727                    e);
2728                return;
2729            }
2730            Set<CmsUUID> bothNewAndDeleted = getIdsOfPublishResourcesWhichAreBothNewAndDeleted(publishedResources);
2731            // When published resources with both states 'new' and 'deleted' exist in the same publish job history, the resource has been moved
2732
2733            List<CmsPublishedResource> updateResources = new ArrayList<CmsPublishedResource>();
2734            for (CmsPublishedResource res : publishedResources) {
2735                if (res.isFolder() || res.getState().isUnchanged()) {
2736                    // folders and unchanged resources don't need to be indexed after publish
2737                    continue;
2738                }
2739                if (res.getState().isDeleted() || res.getState().isNew() || res.getState().isChanged()) {
2740                    if (updateResources.contains(res)) {
2741                        // resource may have been added as a sibling of another resource
2742                        // in this case we make sure to use the value from the publish list because of the "deleted" flag
2743                        boolean hasMoved = bothNewAndDeleted.contains(res.getStructureId())
2744                            || (res.getMovedState() == CmsPublishedResource.STATE_MOVED_DESTINATION)
2745                            || (res.getMovedState() == CmsPublishedResource.STATE_MOVED_SOURCE);
2746                        // check it this is a moved resource with source / target info, in this case we need both entries
2747                        if (!hasMoved) {
2748                            // if the resource was moved, we must contain both entries
2749                            updateResources.remove(res);
2750                        }
2751                        // "equals()" implementation of published resource checks for id,
2752                        // so the removed value may have a different "deleted" or "modified" status value
2753                        updateResources.add(res);
2754                    } else {
2755                        // resource not yet contained in the list
2756                        updateResources.add(res);
2757                        // check for the siblings (not for deleted resources, these are already gone)
2758                        if (!res.getState().isDeleted() && (res.getSiblingCount() > 1)) {
2759                            // this resource has siblings
2760                            try {
2761                                // read siblings from the online project
2762                                List<CmsResource> siblings = adminCms.readSiblings(
2763                                    res.getRootPath(),
2764                                    CmsResourceFilter.ALL);
2765                                Iterator<CmsResource> itSib = siblings.iterator();
2766                                while (itSib.hasNext()) {
2767                                    // check all siblings
2768                                    CmsResource sibling = itSib.next();
2769                                    CmsPublishedResource sib = new CmsPublishedResource(sibling);
2770                                    if (!updateResources.contains(sib)) {
2771                                        // ensure sibling is added only once
2772                                        updateResources.add(sib);
2773                                    }
2774                                }
2775                            } catch (CmsException e) {
2776                                // ignore, just use the original resource
2777                                if (LOG.isWarnEnabled()) {
2778                                    LOG.warn(
2779                                        Messages.get().getBundle().key(
2780                                            Messages.LOG_UNABLE_TO_READ_SIBLINGS_1,
2781                                            res.getRootPath()),
2782                                        e);
2783                                }
2784                            }
2785                        }
2786                    }
2787                }
2788            }
2789
2790            addAdditionallyAffectedResources(adminCms, updateResources);
2791            if (!updateResources.isEmpty()) {
2792                // sort the resource to update
2793                Collections.sort(updateResources);
2794                // only update the indexes if the list of remaining published resources is not empty
2795                Iterator<I_CmsSearchIndex> i = m_indexes.iterator();
2796                while (i.hasNext()) {
2797                    I_CmsSearchIndex index = i.next();
2798                    if (I_CmsSearchIndex.REBUILD_MODE_AUTO.equals(index.getRebuildMode())) {
2799                        // only update indexes which have the rebuild mode set to "auto"
2800                        try {
2801                            updateIndex(index, report, updateResources);
2802                        } catch (CmsException e) {
2803                            LOG.error(
2804                                Messages.get().getBundle().key(Messages.LOG_UPDATE_INDEX_FAILED_1, index.getName()),
2805                                e);
2806                        }
2807                    }
2808                }
2809            }
2810            // clean up the extraction result cache
2811            cleanExtractionCache();
2812        } finally {
2813            SEARCH_MANAGER_LOCK.unlock();
2814            Thread.currentThread().setPriority(oldPriority);
2815        }
2816    }
2817
2818    /**
2819     * Updates (if required creates) the index with the given name.<p>
2820     *
2821     * If the optional List of <code>{@link CmsPublishedResource}</code> instances is provided, the index will be
2822     * incrementally updated for these resources only. If this List is <code>null</code> or empty,
2823     * the index will be fully rebuild.<p>
2824     *
2825     * @param index the index to update or rebuild
2826     * @param report the report to write output messages to
2827     * @param resourcesToIndex an (optional) list of <code>{@link CmsPublishedResource}</code> objects to update in the index
2828     *
2829     * @throws CmsException if something goes wrong
2830     */
2831    protected void updateIndex(I_CmsSearchIndex index, I_CmsReport report, List<CmsPublishedResource> resourcesToIndex)
2832    throws CmsException {
2833
2834        if (shouldUpdateAtAll(index)) {
2835            try {
2836                SEARCH_MANAGER_LOCK.lock();
2837
2838                // copy the stored admin context for the indexing
2839                CmsObject cms = OpenCms.initCmsObject(m_adminCms);
2840                // make sure a report is available
2841                if (report == null) {
2842                    report = new CmsLogReport(cms.getRequestContext().getLocale(), CmsSearchManager.class);
2843                }
2844
2845                // check if the index has been configured correctly
2846                if (!index.checkConfiguration(cms)) {
2847                    // the index is disabled
2848                    return;
2849                }
2850
2851                // set site root and project for this index
2852                cms.getRequestContext().setSiteRoot("/");
2853                // switch to the index project
2854                cms.getRequestContext().setCurrentProject(cms.readProject(index.getProject()));
2855
2856                if ((resourcesToIndex == null) || resourcesToIndex.isEmpty()) {
2857                    // rebuild the complete index
2858
2859                    updateIndexCompletely(cms, index, report);
2860                } else {
2861                    updateIndexIncremental(cms, index, report, resourcesToIndex);
2862                }
2863            } finally {
2864                SEARCH_MANAGER_LOCK.unlock();
2865            }
2866        }
2867    }
2868
2869    /**
2870     * The method updates all OpenCms documents that are indexed.
2871     * @param cms the OpenCms user context to use for accessing the VFS
2872     * @param index the index to update
2873     * @param report the report to write output messages to
2874     * @throws CmsIndexException thrown if indexing fails for some reason
2875     */
2876    @SuppressWarnings("null")
2877    protected void updateIndexCompletely(CmsObject cms, I_CmsSearchIndex index, I_CmsReport report)
2878    throws CmsIndexException {
2879
2880        // create a new thread manager for the indexing threads
2881        CmsIndexingThreadManager threadManager = getThreadManager();
2882
2883        boolean isOfflineIndex = false;
2884        if (I_CmsSearchIndex.REBUILD_MODE_OFFLINE.equals(index.getRebuildMode())) {
2885            // disable offline indexing while the complete index is rebuild
2886            isOfflineIndex = true;
2887            index.setRebuildMode(I_CmsSearchIndex.REBUILD_MODE_MANUAL);
2888            // re-initialize the offline indexes, this will disable this offline index
2889            initOfflineIndexes();
2890        }
2891
2892        I_CmsIndexWriter writer = null;
2893        try {
2894            // create a backup of the existing index
2895            CmsSearchIndex indexInternal = null;
2896            String backup = null;
2897            if (index instanceof CmsSearchIndex) {
2898                indexInternal = (CmsSearchIndex)index;
2899                backup = indexInternal.createIndexBackup();
2900                if (backup != null) {
2901                    indexInternal.indexSearcherOpen(backup);
2902                }
2903            }
2904
2905            // create a new index writer
2906            writer = index.getIndexWriter(report, true);
2907            if (writer instanceof I_CmsSolrIndexWriter) {
2908                try {
2909                    ((I_CmsSolrIndexWriter)writer).deleteAllDocuments();
2910                } catch (IOException e) {
2911                    LOG.error(e.getMessage(), e);
2912                }
2913            }
2914
2915            // output start information on the report
2916            report.println(
2917                Messages.get().container(Messages.RPT_SEARCH_INDEXING_REBUILD_BEGIN_1, index.getName()),
2918                I_CmsReport.FORMAT_HEADLINE);
2919
2920            // iterate all configured index sources of this index
2921            Iterator<CmsSearchIndexSource> sources = index.getSources().iterator();
2922            while (sources.hasNext()) {
2923                // get the next index source
2924                CmsSearchIndexSource source = sources.next();
2925                // create the indexer
2926                I_CmsIndexer indexer = source.getIndexer().newInstance(cms, report, index);
2927                // new index creation, use all resources from the index source
2928                indexer.rebuildIndex(writer, threadManager, source);
2929
2930                // wait for indexing threads to finish
2931                while (threadManager.isRunning()) {
2932                    try {
2933                        Thread.sleep(500);
2934                    } catch (InterruptedException e) {
2935                        // just continue with the loop after interruption
2936                        LOG.info(e.getLocalizedMessage(), e);
2937                    }
2938                }
2939
2940                // commit and optimize the index after each index source has been finished
2941                try {
2942                    writer.commit();
2943                } catch (IOException e) {
2944                    if (LOG.isWarnEnabled()) {
2945                        LOG.warn(
2946                            Messages.get().getBundle().key(
2947                                Messages.LOG_IO_INDEX_WRITER_COMMIT_2,
2948                                index.getName(),
2949                                index.getPath()),
2950                            e);
2951                    }
2952                }
2953                try {
2954                    writer.optimize();
2955                } catch (IOException e) {
2956                    if (LOG.isWarnEnabled()) {
2957                        LOG.warn(
2958                            Messages.get().getBundle().key(
2959                                Messages.LOG_IO_INDEX_WRITER_OPTIMIZE_2,
2960                                index.getName(),
2961                                index.getPath()),
2962                            e);
2963                    }
2964                }
2965            }
2966
2967            // we are sure here that indexInternal is not null
2968            if (backup != null) {
2969                // remove the backup after the files have been re-indexed
2970                indexInternal.indexSearcherClose();
2971                indexInternal.removeIndexBackup(backup);
2972            }
2973
2974            // output finish information on the report
2975            report.println(
2976                Messages.get().container(Messages.RPT_SEARCH_INDEXING_REBUILD_END_1, index.getName()),
2977                I_CmsReport.FORMAT_HEADLINE);
2978
2979        } finally {
2980            if (writer != null) {
2981                try {
2982                    writer.close();
2983                } catch (IOException e) {
2984                    if (LOG.isWarnEnabled()) {
2985                        LOG.warn(
2986                            Messages.get().getBundle().key(
2987                                Messages.LOG_IO_INDEX_WRITER_CLOSE_2,
2988                                index.getPath(),
2989                                index.getName()),
2990                            e);
2991                    }
2992                }
2993            }
2994            if (isOfflineIndex) {
2995                // reset the mode of the offline index
2996                index.setRebuildMode(I_CmsSearchIndex.REBUILD_MODE_OFFLINE);
2997                // re-initialize the offline indexes, this will re-enable this index
2998                initOfflineIndexes();
2999            }
3000            // index has changed - initialize the index searcher instance
3001            index.onIndexChanged(true);
3002        }
3003
3004        // show information about indexing runtime
3005        threadManager.reportStatistics(report);
3006    }
3007
3008    /**
3009     * Incrementally updates the given index.<p>
3010     *
3011     * @param cms the OpenCms user context to use for accessing the VFS
3012     * @param index the index to update
3013     * @param report the report to write output messages to
3014     * @param resourcesToIndex a list of <code>{@link CmsPublishedResource}</code> objects to update in the index
3015     *
3016     * @throws CmsException if something goes wrong
3017     */
3018    protected void updateIndexIncremental(
3019        CmsObject cms,
3020        I_CmsSearchIndex index,
3021        I_CmsReport report,
3022        List<CmsPublishedResource> resourcesToIndex)
3023    throws CmsException {
3024
3025        try {
3026            SEARCH_MANAGER_LOCK.lock();
3027
3028            // update the existing index
3029            List<CmsSearchIndexUpdateData> updateCollections = new ArrayList<CmsSearchIndexUpdateData>();
3030
3031            boolean hasResourcesToDelete = false;
3032            boolean hasResourcesToUpdate = false;
3033
3034            // iterate all configured index sources of this index
3035            Iterator<CmsSearchIndexSource> sources = index.getSources().iterator();
3036            while (sources.hasNext()) {
3037                // get the next index source
3038                CmsSearchIndexSource source = sources.next();
3039                // create the indexer
3040                I_CmsIndexer indexer = source.getIndexer().newInstance(cms, report, index);
3041                // collect the resources to update
3042                CmsSearchIndexUpdateData updateData = indexer.getUpdateData(source, resourcesToIndex);
3043                if (!updateData.isEmpty()) {
3044                    // add the update collection to the internal pipeline
3045                    updateCollections.add(updateData);
3046                    hasResourcesToDelete = hasResourcesToDelete | updateData.hasResourcesToDelete();
3047                    hasResourcesToUpdate = hasResourcesToUpdate | updateData.hasResourceToUpdate();
3048                }
3049            }
3050
3051            // only start index modification if required
3052            if (hasResourcesToDelete || hasResourcesToUpdate) {
3053                // output start information on the report
3054                report.println(
3055                    Messages.get().container(Messages.RPT_SEARCH_INDEXING_UPDATE_BEGIN_1, index.getName()),
3056                    I_CmsReport.FORMAT_HEADLINE);
3057
3058                I_CmsIndexWriter writer = null;
3059                try {
3060                    // obtain an index writer that updates the current index
3061                    writer = index.getIndexWriter(report, false);
3062
3063                    if (hasResourcesToDelete) {
3064                        // delete the resource from the index
3065                        Iterator<CmsSearchIndexUpdateData> i = updateCollections.iterator();
3066                        while (i.hasNext()) {
3067                            CmsSearchIndexUpdateData updateCollection = i.next();
3068                            if (updateCollection.hasResourcesToDelete()) {
3069                                updateCollection.getIndexer().deleteResources(
3070                                    writer,
3071                                    updateCollection.getResourcesToDelete());
3072                            }
3073                        }
3074                    }
3075
3076                    if (hasResourcesToUpdate) {
3077                        // create a new thread manager
3078                        CmsIndexingThreadManager threadManager = getThreadManager();
3079
3080                        Iterator<CmsSearchIndexUpdateData> i = updateCollections.iterator();
3081                        while (i.hasNext()) {
3082                            CmsSearchIndexUpdateData updateCollection = i.next();
3083                            if (updateCollection.hasResourceToUpdate()) {
3084                                updateCollection.getIndexer().updateResources(
3085                                    writer,
3086                                    threadManager,
3087                                    updateCollection.getResourcesToUpdate());
3088                            }
3089                        }
3090
3091                        // wait for indexing threads to finish
3092                        while (threadManager.isRunning()) {
3093                            try {
3094                                Thread.sleep(500);
3095                            } catch (InterruptedException e) {
3096                                // just continue with the loop after interruption
3097                                LOG.info(e.getLocalizedMessage(), e);
3098                            }
3099                        }
3100                    }
3101                } finally {
3102                    // close the index writer
3103                    if (writer != null) {
3104                        try {
3105                            writer.commit();
3106                        } catch (IOException e) {
3107                            LOG.error(
3108                                Messages.get().getBundle().key(
3109                                    Messages.LOG_IO_INDEX_WRITER_COMMIT_2,
3110                                    index.getName(),
3111                                    index.getPath()),
3112                                e);
3113                        }
3114                    }
3115                    // index has changed - initialize the index searcher instance
3116                    index.onIndexChanged(false);
3117                }
3118
3119                // output finish information on the report
3120                report.println(
3121                    Messages.get().container(Messages.RPT_SEARCH_INDEXING_UPDATE_END_1, index.getName()),
3122                    I_CmsReport.FORMAT_HEADLINE);
3123            }
3124        } finally {
3125            SEARCH_MANAGER_LOCK.unlock();
3126        }
3127    }
3128
3129    /**
3130     * Updates the offline search indexes for the given list of resources.<p>
3131     *
3132     * @param report the report to write the index information to
3133     * @param resourcesToIndex the list of {@link CmsPublishedResource} objects to index
3134     */
3135    protected void updateIndexOffline(I_CmsReport report, List<CmsPublishedResource> resourcesToIndex) {
3136
3137        CmsObject cms = m_adminCms;
3138        try {
3139            // copy the administration context for the indexing
3140            cms = OpenCms.initCmsObject(m_adminCms);
3141            // set site root and project for this index
3142            cms.getRequestContext().setSiteRoot("/");
3143        } catch (CmsException e) {
3144            LOG.error(e.getLocalizedMessage(), e);
3145        }
3146
3147        Iterator<I_CmsSearchIndex> j = m_offlineIndexes.iterator();
3148        while (j.hasNext()) {
3149            I_CmsSearchIndex index = j.next();
3150            if (index.getSources() != null) {
3151                try {
3152                    // switch to the index project
3153                    cms.getRequestContext().setCurrentProject(cms.readProject(index.getProject()));
3154                    updateIndexIncremental(cms, index, report, resourcesToIndex);
3155                } catch (CmsException e) {
3156                    LOG.error(Messages.get().getBundle().key(Messages.LOG_UPDATE_INDEX_FAILED_1, index.getName()), e);
3157                }
3158            }
3159        }
3160    }
3161
3162    /**
3163     * Checks if the given containerpage is used as a detail containers and adds the related detail content to the resource set.<p>
3164     *
3165     * @param adminCms the cms context
3166     * @param containerPages the containerpages
3167     * @param containerPage the container page site path
3168     */
3169    private void addDetailContent(CmsObject adminCms, Set<CmsResource> containerPages, String containerPage) {
3170
3171        if (CmsDetailOnlyContainerUtil.isDetailContainersPage(adminCms, containerPage)) {
3172
3173            try {
3174                CmsResource detailRes = adminCms.readResource(
3175                    CmsDetailOnlyContainerUtil.getDetailContentPath(containerPage),
3176                    CmsResourceFilter.IGNORE_EXPIRATION);
3177                containerPages.add(detailRes);
3178            } catch (Throwable e) {
3179                if (LOG.isWarnEnabled()) {
3180                    LOG.warn(e.getLocalizedMessage(), e);
3181                }
3182            }
3183        }
3184    }
3185
3186    /**
3187     * Creates the Solr core container.<p>
3188     *
3189     * @return the created core container
3190     */
3191    private CoreContainer createCoreContainer() {
3192
3193        CoreContainer container = null;
3194        try {
3195            // get the core container
3196            // still no core container: create it
3197            container = CoreContainer.createAndLoad(
3198                Paths.get(m_solrConfig.getHome()),
3199                m_solrConfig.getSolrFile().toPath());
3200            if (CmsLog.INIT.isInfoEnabled()) {
3201                CmsLog.INIT.info(
3202                    Messages.get().getBundle().key(
3203                        Messages.INIT_SOLR_CORE_CONTAINER_CREATED_2,
3204                        m_solrConfig.getHome(),
3205                        m_solrConfig.getSolrFile().getName()));
3206            }
3207        } catch (Exception e) {
3208            LOG.error(
3209                Messages.get().getBundle().key(
3210                    Messages.ERR_SOLR_CORE_CONTAINER_NOT_CREATED_1,
3211                    m_solrConfig.getSolrFile().getAbsolutePath()),
3212                e);
3213        }
3214        return container;
3215
3216    }
3217
3218    /**
3219     * Remove write.lock file in the data directory to ensure the index is unlocked.
3220     * @param dataDir the data directory of the Solr index that should be unlocked.
3221     */
3222    private void ensureIndexIsUnlocked(String dataDir) {
3223
3224        Collection<File> lockFiles = new ArrayList<File>(2);
3225        lockFiles.add(
3226            new File(
3227                CmsFileUtil.addTrailingSeparator(CmsFileUtil.addTrailingSeparator(dataDir) + "index") + "write.lock"));
3228        lockFiles.add(
3229            new File(
3230                CmsFileUtil.addTrailingSeparator(CmsFileUtil.addTrailingSeparator(dataDir) + "spellcheck")
3231                    + "write.lock"));
3232        for (File lockFile : lockFiles) {
3233            if (lockFile.exists()) {
3234                lockFile.delete();
3235                LOG.warn(
3236                    "Forcely unlocking index with data dir \""
3237                        + dataDir
3238                        + "\" by removing file \""
3239                        + lockFile.getAbsolutePath()
3240                        + "\".");
3241            }
3242        }
3243    }
3244
3245    /**
3246     * Returns the report in the given event data, if <code>null</code>
3247     * a new log report is used.<p>
3248     *
3249     * @param event the event to get the report for
3250     *
3251     * @return the report
3252     */
3253    private I_CmsReport getEventReport(CmsEvent event) {
3254
3255        I_CmsReport report = null;
3256        if (event.getData() != null) {
3257            report = (I_CmsReport)event.getData().get(I_CmsEventListener.KEY_REPORT);
3258        }
3259        if (report == null) {
3260            report = new CmsLogReport(Locale.ENGLISH, getClass());
3261        }
3262        return report;
3263    }
3264
3265    /**
3266     * Gets all structure ids for which published resources of both states 'new' and 'deleted' exist in the given list.<p>
3267     *
3268     * @param publishedResources a list of published resources
3269     *
3270     * @return the set of structure ids that satisfy the condition above
3271     */
3272    private Set<CmsUUID> getIdsOfPublishResourcesWhichAreBothNewAndDeleted(
3273        List<CmsPublishedResource> publishedResources) {
3274
3275        Set<CmsUUID> result = new HashSet<CmsUUID>();
3276        Set<CmsUUID> deletedSet = new HashSet<CmsUUID>();
3277        for (CmsPublishedResource pubRes : publishedResources) {
3278            if (pubRes.getState().isNew()) {
3279                result.add(pubRes.getStructureId());
3280            }
3281            if (pubRes.getState().isDeleted()) {
3282                deletedSet.add(pubRes.getStructureId());
3283            }
3284        }
3285        result.retainAll(deletedSet);
3286        return result;
3287    }
3288
3289    /**
3290     * Shuts down the Solr core container.<p>
3291     */
3292    private void shutDownSolrContainer() {
3293
3294        if (m_coreContainer != null) {
3295            for (SolrCore core : m_coreContainer.getCores()) {
3296                // do not unload spellcheck core because otherwise the core.properties file is removed
3297                // even when calling m_coreContainer.unload(core.getName(), false, false, false);
3298                if (!core.getName().equals(CmsSolrSpellchecker.SPELLCHECKER_INDEX_CORE)) {
3299                    m_coreContainer.unload(core.getName(), false, false, true);
3300                }
3301            }
3302            m_coreContainer.shutdown();
3303            if (CmsLog.INIT.isInfoEnabled()) {
3304                CmsLog.INIT.info(Messages.get().getBundle().key(Messages.INIT_SOLR_SHUTDOWN_SUCCESS_0));
3305            }
3306            m_coreContainer = null;
3307        }
3308    }
3309
3310}