001/*
002 * File   : $Source$
003 * Date   : $Date$
004 * Version: $Revision$
005 *
006 * This library is part of OpenCms -
007 * the Open Source Content Management System
008 *
009 * Copyright (C) 2002 - 2009 Alkacon Software (http://www.alkacon.com)
010 *
011 * This library is free software; you can redistribute it and/or
012 * modify it under the terms of the GNU Lesser General Public
013 * License as published by the Free Software Foundation; either
014 * version 2.1 of the License, or (at your option) any later version.
015 *
016 * This library is distributed in the hope that it will be useful,
017 * but WITHOUT ANY WARRANTY; without even the implied warranty of
018 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
019 * Lesser General Public License for more details.
020 *
021 * For further information about Alkacon Software, please see the
022 * company website: http://www.alkacon.com
023 *
024 * For further information about OpenCms, please see the
025 * project website: http://www.opencms.org
026 *
027 * You should have received a copy of the GNU Lesser General Public
028 * License along with this library; if not, write to the Free Software
029 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
030 */
031
032package org.opencms.search;
033
034import org.opencms.configuration.CmsParameterConfiguration;
035import org.opencms.file.CmsObject;
036import org.opencms.file.CmsResource;
037import org.opencms.file.CmsResourceFilter;
038import org.opencms.main.CmsException;
039import org.opencms.main.CmsIllegalArgumentException;
040import org.opencms.main.CmsLog;
041import org.opencms.main.OpenCms;
042import org.opencms.report.I_CmsReport;
043import org.opencms.search.documents.I_CmsTermHighlighter;
044import org.opencms.search.extractors.CmsExtractionResult;
045import org.opencms.search.extractors.I_CmsExtractionResult;
046import org.opencms.search.fields.CmsLuceneFieldConfiguration;
047import org.opencms.search.fields.CmsSearchField;
048import org.opencms.search.fields.CmsSearchFieldConfiguration;
049import org.opencms.util.CmsFileUtil;
050import org.opencms.util.CmsStringUtil;
051
052import java.io.File;
053import java.io.IOException;
054import java.nio.file.Paths;
055import java.text.ParseException;
056import java.util.ArrayList;
057import java.util.Calendar;
058import java.util.Collections;
059import java.util.Date;
060import java.util.HashMap;
061import java.util.List;
062import java.util.Locale;
063import java.util.Map;
064import java.util.Set;
065
066import org.apache.commons.logging.Log;
067import org.apache.lucene.analysis.Analyzer;
068import org.apache.lucene.document.DateTools;
069import org.apache.lucene.document.Document;
070import org.apache.lucene.index.DirectoryReader;
071import org.apache.lucene.index.FieldInfo;
072import org.apache.lucene.index.IndexReader;
073import org.apache.lucene.index.IndexWriter;
074import org.apache.lucene.index.IndexWriterConfig;
075import org.apache.lucene.index.StoredFieldVisitor;
076import org.apache.lucene.index.Term;
077import org.apache.lucene.queryparser.classic.QueryParser;
078import org.apache.lucene.search.BooleanClause;
079import org.apache.lucene.search.BooleanClause.Occur;
080import org.apache.lucene.search.BooleanQuery;
081import org.apache.lucene.search.IndexSearcher;
082import org.apache.lucene.search.MatchAllDocsQuery;
083import org.apache.lucene.search.MultiTermQuery;
084import org.apache.lucene.search.Query;
085import org.apache.lucene.search.ScoreMode;
086import org.apache.lucene.search.Sort;
087import org.apache.lucene.search.SortField;
088import org.apache.lucene.search.TermQuery;
089import org.apache.lucene.search.TopDocs;
090import org.apache.lucene.search.similarities.Similarity;
091import org.apache.lucene.store.Directory;
092import org.apache.lucene.store.FSDirectory;
093import org.apache.lucene.store.IOContext;
094import org.apache.solr.uninverting.UninvertingReader;
095import org.apache.solr.uninverting.UninvertingReader.Type;
096
097/**
098 * Abstract search index implementation.<p>
099 */
100public class CmsSearchIndex extends A_CmsSearchIndex {
101
102    /** A constant for the full qualified name of the CmsSearchIndex class. */
103    public static final String A_PARAM_PREFIX = "org.opencms.search.CmsSearchIndex";
104
105    /** Constant for additional parameter to enable optimized full index regeneration (default: false). */
106    public static final String BACKUP_REINDEXING = A_PARAM_PREFIX + ".useBackupReindexing";
107
108    /** Look table to quickly zero-pad days / months in date Strings. */
109    public static final String[] DATES = new String[] {
110        "00",
111        "01",
112        "02",
113        "03",
114        "04",
115        "05",
116        "06",
117        "07",
118        "08",
119        "09",
120        "10",
121        "11",
122        "12",
123        "13",
124        "14",
125        "15",
126        "16",
127        "17",
128        "18",
129        "19",
130        "20",
131        "21",
132        "22",
133        "23",
134        "24",
135        "25",
136        "26",
137        "27",
138        "28",
139        "29",
140        "30",
141        "31"};
142
143    /** Constant for a field list that contains the "meta" field as well as the "content" field. */
144    public static final String[] DOC_META_FIELDS = new String[] {
145        CmsSearchField.FIELD_META,
146        CmsSearchField.FIELD_CONTENT};
147
148    /** Constant for additional parameter to enable excerpt creation (default: true). */
149    public static final String EXCERPT = A_PARAM_PREFIX + ".createExcerpt";
150
151    /** Constant for additional parameter for index content extraction. */
152    public static final String EXTRACT_CONTENT = A_PARAM_PREFIX + ".extractContent";
153
154    /** Constant for additional parameter to enable/disable language detection (default: false). */
155    public static final String IGNORE_EXPIRATION = A_PARAM_PREFIX + ".ignoreExpiration";
156
157    /** Constant for additional parameter to enable/disable language detection (default: false). */
158    public static final String LANGUAGEDETECTION = "search.solr.useLanguageDetection";
159
160    /** Constant for additional parameter for the Lucene index setting. */
161    public static final String LUCENE_AUTO_COMMIT = "lucene.AutoCommit";
162
163    /** Constant for additional parameter for the Lucene index setting. */
164    public static final String LUCENE_RAM_BUFFER_SIZE_MB = "lucene.RAMBufferSizeMB";
165
166    /** Constant for additional parameter for controlling how many hits are loaded at maximum (default: 1000). */
167    public static final String MAX_HITS = A_PARAM_PREFIX + ".maxHits";
168
169    /** Indicates how many hits are loaded at maximum by default. */
170    public static final int MAX_HITS_DEFAULT = 5000;
171
172    /** Constant for years max range span in document search. */
173    public static final int MAX_YEAR_RANGE = 25;
174
175    /** Constant for additional parameter to enable permission checks (default: true). */
176    public static final String PERMISSIONS = A_PARAM_PREFIX + ".checkPermissions";
177
178    /** Constant for additional parameter to set the thread priority during search. */
179    public static final String PRIORITY = A_PARAM_PREFIX + ".priority";
180
181    /** Constant for additional parameter to enable time range checks (default: true). */
182    public static final String TIME_RANGE = A_PARAM_PREFIX + ".checkTimeRange";
183
184    /**
185     * A stored field visitor, that does not return the large fields: "content" and "contentblob".<p>
186     */
187    protected static final StoredFieldVisitor VISITOR = new StoredFieldVisitor() {
188
189        /**
190         * @see org.apache.lucene.index.StoredFieldVisitor#needsField(org.apache.lucene.index.FieldInfo)
191         */
192        @Override
193        public Status needsField(FieldInfo fieldInfo) {
194
195            return !CmsSearchFieldConfiguration.LAZY_FIELDS.contains(fieldInfo.name) ? Status.YES : Status.NO;
196        }
197    };
198
199    /** The log object for this class. */
200    private static final Log LOG = CmsLog.getLog(CmsSearchIndex.class);
201
202    /** The serial version id. */
203    private static final long serialVersionUID = 8461682478204452718L;
204
205    /** The configured Lucene analyzer used for this index. */
206    private transient Analyzer m_analyzer;
207
208    /** Indicates if backup re-indexing is used by this index. */
209    private boolean m_backupReindexing;
210
211    /** The permission check mode for this index. */
212    private boolean m_checkPermissions;
213
214    /** The time range check mode for this index. */
215    private boolean m_checkTimeRange;
216
217    /** The excerpt mode for this index. */
218    private boolean m_createExcerpt;
219
220    /** Map of display query filters to use. */
221    private transient Map<String, Query> m_displayFilters;
222
223    /**
224     * Signals whether expiration dates should be ignored when checking permissions or not.<p>
225     * @see #IGNORE_EXPIRATION
226     */
227    private boolean m_ignoreExpiration;
228
229    /** The Lucene index searcher to use. */
230    private transient IndexSearcher m_indexSearcher;
231
232    /** The Lucene index RAM buffer size, see {@link IndexWriterConfig#setRAMBufferSizeMB(double)}. */
233    private Double m_luceneRAMBufferSizeMB;
234
235    /** Indicates how many hits are loaded at maximum. */
236    private int m_maxHits;
237
238    /** The thread priority for a search. */
239    private int m_priority;
240
241    /** Controls if a resource requires view permission to be displayed in the result list. */
242    private boolean m_requireViewPermission;
243
244    /** The cms specific Similarity implementation. */
245    private final transient Similarity m_sim = new CmsSearchSimilarity();
246
247    /**
248     * Default constructor only intended to be used by the XML configuration. <p>
249     *
250     * It is recommended to use the constructor <code>{@link #CmsSearchIndex(String)}</code>
251     * as it enforces the mandatory name argument. <p>
252     */
253    public CmsSearchIndex() {
254
255        super();
256        m_checkPermissions = true;
257        m_priority = -1;
258        m_createExcerpt = true;
259        m_maxHits = MAX_HITS_DEFAULT;
260        m_checkTimeRange = false;
261    }
262
263    /**
264     * Creates a new CmsSearchIndex with the given name.<p>
265     *
266     * @param name the system-wide unique name for the search index
267     *
268     * @throws CmsIllegalArgumentException if the given name is null, empty or already taken by another search index
269     */
270    public CmsSearchIndex(String name)
271    throws CmsIllegalArgumentException {
272
273        this();
274        setName(name);
275    }
276
277    /**
278     * Generates a list of date terms for the optimized date range search with "daily" granularity level.<p>
279     *
280     * How this works:<ul>
281     * <li>For each document, terms are added for the year, the month and the day the document
282     * was modified or created) in. So for example if a document is modified at February 02, 2009,
283     * then the following terms are stored for this document:
284     * "20090202", "200902" and "2009".</li>
285     * <li>In case a date range search is done, then all possible matches for the
286     * provided rage are created as search terms and matched with the document terms.</li>
287     * <li>Consider the following use case: You want to find out if a resource has been changed
288     * in the time between November 29, 2007 and March 01, 2009.
289     * One term to match is simply "2008" because if a document
290     * was modified in 2008, then it is clearly in the date range.
291     * Other terms are "200712", "200901" and "200902", because all documents
292     * modified in these months are also a certain matches.
293     * Finally we need to add terms for "20071129", "20071130" and "20090301" to match the days in the
294     * starting and final month.</li>
295     * </ul>
296     *
297     * @param startDate start date of the range to search in
298     * @param endDate end date of the range to search in
299     *
300     * @return a list of date terms for the optimized date range search
301     */
302    public static List<String> getDateRangeSpan(long startDate, long endDate) {
303
304        if (startDate > endDate) {
305            // switch so that the end is always before the start
306            long temp = endDate;
307            endDate = startDate;
308            startDate = temp;
309        }
310
311        List<String> result = new ArrayList<String>(100);
312
313        // initialize calendars from the time value
314        Calendar calStart = Calendar.getInstance(OpenCms.getLocaleManager().getTimeZone());
315        Calendar calEnd = Calendar.getInstance(calStart.getTimeZone());
316        calStart.setTimeInMillis(startDate);
317        calEnd.setTimeInMillis(endDate);
318
319        // get the required info to build the date range from the calendars
320        int startDay = calStart.get(Calendar.DAY_OF_MONTH);
321        int endDay = calEnd.get(Calendar.DAY_OF_MONTH);
322        int maxDayInStartMonth = calStart.getActualMaximum(Calendar.DAY_OF_MONTH);
323        int startMonth = calStart.get(Calendar.MONTH) + 1;
324        int endMonth = calEnd.get(Calendar.MONTH) + 1;
325        int startYear = calStart.get(Calendar.YEAR);
326        int endYear = calEnd.get(Calendar.YEAR);
327
328        // first add all full years in the date range
329        result.addAll(getYearSpan(startYear + 1, endYear - 1));
330
331        if (startYear != endYear) {
332            // different year, different month
333            result.addAll(getMonthSpan(startMonth + 1, 12, startYear));
334            result.addAll(getMonthSpan(1, endMonth - 1, endYear));
335            result.addAll(getDaySpan(startDay, maxDayInStartMonth, startMonth, startYear));
336            result.addAll(getDaySpan(1, endDay, endMonth, endYear));
337        } else {
338            if (startMonth != endMonth) {
339                // same year, different month
340                result.addAll(getMonthSpan(startMonth + 1, endMonth - 1, startYear));
341                result.addAll(getDaySpan(startDay, maxDayInStartMonth, startMonth, startYear));
342                result.addAll(getDaySpan(1, endDay, endMonth, endYear));
343            } else {
344                // same year, same month
345                result.addAll(getDaySpan(startDay, endDay, endMonth, endYear));
346            }
347        }
348
349        // sort the result, makes the range better readable in the debugger
350        Collections.sort(result);
351        return result;
352    }
353
354    /**
355     * Calculate a span of days in the given year and month for the optimized date range search.<p>
356     *
357     * The result will contain dates formatted like "yyyyMMDD", for example "20080131".<p>
358     *
359     * @param startDay the start day
360     * @param endDay the end day
361     * @param month the month
362     * @param year the year
363     *
364     * @return a span of days in the given year and month for the optimized date range search
365     */
366    private static List<String> getDaySpan(int startDay, int endDay, int month, int year) {
367
368        List<String> result = new ArrayList<String>();
369        String yearMonthStr = String.valueOf(year) + DATES[month];
370        for (int i = startDay; i <= endDay; i++) {
371            String dateStr = yearMonthStr + DATES[i];
372            result.add(dateStr);
373        }
374        return result;
375    }
376
377    /**
378     * Calculate a span of months in the given year for the optimized date range search.<p>
379     *
380     * The result will contain dates formatted like "yyyyMM", for example "200801".<p>
381     *
382     * @param startMonth the start month
383     * @param endMonth the end month
384     * @param year the year
385     *
386     * @return a span of months in the given year for the optimized date range search
387     */
388    private static List<String> getMonthSpan(int startMonth, int endMonth, int year) {
389
390        List<String> result = new ArrayList<String>();
391        String yearStr = String.valueOf(year);
392        for (int i = startMonth; i <= endMonth; i++) {
393            String dateStr = yearStr + DATES[i];
394            result.add(dateStr);
395        }
396        return result;
397    }
398
399    /**
400     * Calculate a span of years for the optimized date range search.<p>
401     *
402     * The result will contain dates formatted like "yyyy", for example "2008".<p>
403     *
404     * @param startYear the start year
405     * @param endYear the end year
406     *
407     * @return a span of years for the optimized date range search
408     */
409    private static List<String> getYearSpan(int startYear, int endYear) {
410
411        List<String> result = new ArrayList<String>();
412        for (int i = startYear; i <= endYear; i++) {
413            String dateStr = String.valueOf(i);
414            result.add(dateStr);
415        }
416        return result;
417    }
418
419    /**
420     * Adds a parameter.<p>
421     *
422     * @param key the key/name of the parameter
423     * @param value the value of the parameter
424     *
425     */
426    @Override
427    public void addConfigurationParameter(String key, String value) {
428
429        if (PERMISSIONS.equals(key)) {
430            m_checkPermissions = Boolean.valueOf(value).booleanValue();
431        } else if (EXTRACT_CONTENT.equals(key)) {
432            setExtractContent(Boolean.valueOf(value).booleanValue());
433        } else if (BACKUP_REINDEXING.equals(key)) {
434            m_backupReindexing = Boolean.valueOf(value).booleanValue();
435        } else if (LANGUAGEDETECTION.equals(key)) {
436            setLanguageDetection(Boolean.valueOf(value).booleanValue());
437        } else if (IGNORE_EXPIRATION.equals(key)) {
438            m_ignoreExpiration = Boolean.valueOf(value).booleanValue();
439        } else if (PRIORITY.equals(key)) {
440            m_priority = Integer.parseInt(value);
441            if (m_priority < Thread.MIN_PRIORITY) {
442                m_priority = Thread.MIN_PRIORITY;
443                LOG.error(
444                    Messages.get().getBundle().key(
445                        Messages.LOG_SEARCH_PRIORITY_TOO_LOW_2,
446                        value,
447                        new Integer(Thread.MIN_PRIORITY)));
448
449            } else if (m_priority > Thread.MAX_PRIORITY) {
450                m_priority = Thread.MAX_PRIORITY;
451                LOG.debug(
452                    Messages.get().getBundle().key(
453                        Messages.LOG_SEARCH_PRIORITY_TOO_HIGH_2,
454                        value,
455                        new Integer(Thread.MAX_PRIORITY)));
456            }
457        }
458
459        if (MAX_HITS.equals(key)) {
460            try {
461                m_maxHits = Integer.parseInt(value);
462            } catch (NumberFormatException e) {
463                LOG.error(Messages.get().getBundle().key(Messages.LOG_INVALID_PARAM_3, value, key, getName()));
464            }
465            if (m_maxHits < (MAX_HITS_DEFAULT / 100)) {
466                m_maxHits = MAX_HITS_DEFAULT;
467                LOG.error(Messages.get().getBundle().key(Messages.LOG_INVALID_PARAM_3, value, key, getName()));
468            }
469        } else if (TIME_RANGE.equals(key)) {
470            m_checkTimeRange = Boolean.valueOf(value).booleanValue();
471        } else if (CmsSearchIndex.EXCERPT.equals(key)) {
472            m_createExcerpt = Boolean.valueOf(value).booleanValue();
473
474        } else if (LUCENE_RAM_BUFFER_SIZE_MB.equals(key)) {
475            try {
476                m_luceneRAMBufferSizeMB = Double.valueOf(value);
477            } catch (NumberFormatException e) {
478                LOG.error(Messages.get().getBundle().key(Messages.LOG_INVALID_PARAM_3, value, key, getName()));
479            }
480        }
481    }
482
483    /**
484     * Creates an empty document that can be used by this search field configuration.<p>
485     *
486     * @param resource the resource to create the document for
487     *
488     * @return a new and empty document
489     */
490    public I_CmsSearchDocument createEmptyDocument(CmsResource resource) {
491
492        return new CmsLuceneDocument(new Document());
493    }
494
495    /**
496     * Returns the Lucene analyzer used for this index.<p>
497     *
498     * @return the Lucene analyzer used for this index
499     */
500    public Analyzer getAnalyzer() {
501
502        return m_analyzer;
503    }
504
505    /**
506     * @see org.opencms.configuration.I_CmsConfigurationParameterHandler#getConfiguration()
507     */
508    @Override
509    public CmsParameterConfiguration getConfiguration() {
510
511        CmsParameterConfiguration result = new CmsParameterConfiguration();
512        if (getPriority() > 0) {
513            result.put(PRIORITY, String.valueOf(m_priority));
514        }
515        if (!isExtractingContent()) {
516            result.put(EXTRACT_CONTENT, String.valueOf(isExtractingContent()));
517        }
518        if (!isCheckingPermissions()) {
519            result.put(PERMISSIONS, String.valueOf(m_checkPermissions));
520        }
521        if (isBackupReindexing()) {
522            result.put(BACKUP_REINDEXING, String.valueOf(m_backupReindexing));
523        }
524        if (isLanguageDetection()) {
525            result.put(LANGUAGEDETECTION, String.valueOf(isLanguageDetection()));
526        }
527        if (getMaxHits() != MAX_HITS_DEFAULT) {
528            result.put(MAX_HITS, String.valueOf(getMaxHits()));
529        }
530        if (!isCreatingExcerpt()) {
531            result.put(EXCERPT, String.valueOf(m_createExcerpt));
532        }
533        if (m_luceneRAMBufferSizeMB != null) {
534            result.put(LUCENE_RAM_BUFFER_SIZE_MB, String.valueOf(m_luceneRAMBufferSizeMB));
535        }
536        // always write time range check parameter because of logic change in OpenCms 8.0
537        result.put(TIME_RANGE, String.valueOf(m_checkTimeRange));
538        return result;
539    }
540
541    /**
542     * @see org.opencms.search.I_CmsSearchIndex#getContentIfUnchanged(org.opencms.file.CmsResource)
543     */
544    @Override
545    public I_CmsExtractionResult getContentIfUnchanged(CmsResource resource) {
546
547        // compare "date of last modification of content" from Lucene index and OpenCms VFS
548        // if this is identical, then the data from the Lucene index can be re-used
549        I_CmsSearchDocument oldDoc = getDocument(CmsSearchField.FIELD_PATH, resource.getRootPath());
550        // first check if the document is already in the index
551        if ((oldDoc != null) && (oldDoc.getFieldValueAsDate(CmsSearchField.FIELD_DATE_CONTENT) != null)) {
552            long contentDateIndex = oldDoc.getFieldValueAsDate(CmsSearchField.FIELD_DATE_CONTENT).getTime();
553            // now compare the date with the date stored in the resource
554            if (contentDateIndex == resource.getDateContent()) {
555                // extract stored content blob from index
556                return CmsExtractionResult.fromBytes(oldDoc.getContentBlob());
557            }
558        }
559        return null;
560    }
561
562    /**
563     * Returns a document by document ID.<p>
564     *
565     * @param docId the id to get the document for
566     *
567     * @return the CMS specific document
568     */
569    public I_CmsSearchDocument getDocument(int docId) {
570
571        try {
572            IndexSearcher searcher = getSearcher();
573            return new CmsLuceneDocument(searcher.doc(docId));
574        } catch (IOException e) {
575            // ignore, return null and assume document was not found
576        }
577        return null;
578    }
579
580    /**
581     * Returns the Lucene document with the given root path from the index.<p>
582     *
583     * @param rootPath the root path of the document to get
584     *
585     * @return the Lucene document with the given root path from the index
586     *
587     * @deprecated Use {@link #getDocument(String, String)} instead and provide {@link org.opencms.search.fields.CmsLuceneField#FIELD_PATH} as field to search in
588     */
589    @Deprecated
590    public Document getDocument(String rootPath) {
591
592        if (getDocument(CmsSearchField.FIELD_PATH, rootPath) != null) {
593            return (Document)getDocument(CmsSearchField.FIELD_PATH, rootPath).getDocument();
594        }
595        return null;
596    }
597
598    /**
599     * Returns the first document where the given term matches the selected index field.<p>
600     *
601     * Use this method to search for documents which have unique field values, like a unique id.<p>
602     *
603     * @param field the field to search in
604     * @param term the term to search for
605     *
606     * @return the first document where the given term matches the selected index field
607     */
608    public I_CmsSearchDocument getDocument(String field, String term) {
609
610        Document result = null;
611        IndexSearcher searcher = getSearcher();
612        if (searcher != null) {
613            // search for an exact match on the selected field
614            Term resultTerm = new Term(field, term);
615            try {
616                TopDocs hits = searcher.search(new TermQuery(resultTerm), 1);
617                if (hits.scoreDocs.length > 0) {
618                    result = searcher.doc(hits.scoreDocs[0].doc);
619                }
620            } catch (IOException e) {
621                // ignore, return null and assume document was not found
622            }
623        }
624        if (result != null) {
625            return new CmsLuceneDocument(result);
626        }
627        return null;
628    }
629
630    /**
631     * Returns the language locale for the given resource in this index.<p>
632     *
633     * @param cms the current OpenCms user context
634     * @param resource the resource to check
635     * @param availableLocales a list of locales supported by the resource
636     *
637     * @return the language locale for the given resource in this index
638     */
639    @Override
640    public Locale getLocaleForResource(CmsObject cms, CmsResource resource, List<Locale> availableLocales) {
641
642        Locale result;
643        List<Locale> defaultLocales = OpenCms.getLocaleManager().getDefaultLocales(cms, resource);
644        List<Locale> locales = availableLocales;
645        if ((locales == null) || (locales.size() == 0)) {
646            locales = defaultLocales;
647        }
648        result = OpenCms.getLocaleManager().getBestMatchingLocale(getLocale(), defaultLocales, locales);
649        return result;
650    }
651
652    /**
653    * Returns the language locale of the index as a String.<p>
654    *
655    * @return the language locale of the index as a String
656    *
657    * @see #getLocale()
658    */
659    public String getLocaleString() {
660
661        return getLocale().toString();
662    }
663
664    /**
665     * Indicates the number of how many hits are loaded at maximum.<p>
666     *
667     * The number of maximum documents to load from the index
668     * must be specified. The default of this setting is {@link CmsSearchIndex#MAX_HITS_DEFAULT} (5000).
669     * This means that at maximum 5000 results are returned from the index.
670     * Please note that this number may be reduced further because of OpenCms read permissions
671     * or per-user file visibility settings not controlled in the index.<p>
672     *
673     * @return the number of how many hits are loaded at maximum
674     *
675     * @since 7.5.1
676     */
677    public int getMaxHits() {
678
679        return m_maxHits;
680    }
681
682    /**
683     * Returns the path where this index stores it's data in the "real" file system.<p>
684     *
685     * @return the path where this index stores it's data in the "real" file system
686     */
687    @Override
688    public String getPath() {
689
690        if (super.getPath() == null) {
691            setPath(generateIndexDirectory());
692        }
693        return super.getPath();
694    }
695
696    /**
697     * Returns the Thread priority for this search index.<p>
698     *
699     * @return the Thread priority for this search index
700     */
701    public int getPriority() {
702
703        return m_priority;
704    }
705
706    /**
707     * Returns the Lucene index searcher used for this search index.<p>
708     *
709     * @return the Lucene index searcher used for this search index
710     */
711    public IndexSearcher getSearcher() {
712
713        return m_indexSearcher;
714    }
715
716    /**
717     * @see org.opencms.search.A_CmsSearchIndex#initialize()
718     */
719    @Override
720    public void initialize() throws CmsSearchException {
721
722        super.initialize();
723
724        // get the configured analyzer and apply the the field configuration analyzer wrapper
725        @SuppressWarnings("resource")
726        Analyzer baseAnalyzer = OpenCms.getSearchManager().getAnalyzer(getLocale());
727
728        if (getFieldConfiguration() instanceof CmsLuceneFieldConfiguration) {
729            CmsLuceneFieldConfiguration fc = (CmsLuceneFieldConfiguration)getFieldConfiguration();
730            setAnalyzer(fc.getAnalyzer(baseAnalyzer));
731        }
732    }
733
734    /**
735     * Returns <code>true</code> if backup re-indexing is done by this index.<p>
736     *
737     * This is an optimization method by which the old extracted content is
738     * reused in order to save performance when re-indexing.<p>
739     *
740     * @return  <code>true</code> if backup re-indexing is done by this index
741     *
742     * @since 7.5.1
743     */
744    public boolean isBackupReindexing() {
745
746        return m_backupReindexing;
747    }
748
749    /**
750     * Returns <code>true</code> if permissions are checked for search results by this index.<p>
751     *
752     * If permission checks are not required, they can be turned off in the index search configuration parameters
753     * in <code>opencms-search.xml</code>. Not checking permissions will improve performance.<p>
754     *
755     * This is can be of use in scenarios when you know that all search results are always readable,
756     * which is usually true for public websites that do not have personalized accounts.<p>
757     *
758     * Please note that even if a result is returned where the current user has no read permissions,
759     * the user can not actually access this document. It will only appear in the search result list,
760     * but if the user clicks the link to open the document he will get an error.<p>
761     *
762     *
763     * @return <code>true</code> if permissions are checked for search results by this index
764     */
765    public boolean isCheckingPermissions() {
766
767        return m_checkPermissions;
768    }
769
770    /**
771     * Returns <code>true</code> if the document time range is checked with a granularity level of seconds
772     * for search results by this index.<p>
773     *
774     * Since OpenCms 8.0, time range checks are always done if {@link CmsSearchParameters#setMinDateLastModified(long)}
775     * or any of the corresponding methods are used.
776     * This is done very efficiently using optimized Lucene filers.
777     * However, the granularity of these checks are done only on a daily
778     * basis, which means that you can only find "changes made yesterday" but not "changes made last hour".
779     * For normal limitation of search results, a daily granularity should be enough.<p>
780     *
781     * If time range checks with a granularity level of seconds are required,
782     * they can be turned on in the index search configuration parameters
783     * in <code>opencms-search.xml</code>.
784     * Not checking the time range  with a granularity level of seconds will improve performance.<p>
785     *
786     * By default the granularity level of seconds is turned off since OpenCms 8.0<p>
787     *
788     * @return <code>true</code> if the document time range is checked  with a granularity level of seconds for search results by this index
789     */
790    public boolean isCheckingTimeRange() {
791
792        return m_checkTimeRange;
793    }
794
795    /**
796     * Returns the checkPermissions.<p>
797     *
798     * @return the checkPermissions
799     */
800    public boolean isCheckPermissions() {
801
802        return m_checkPermissions;
803    }
804
805    /**
806     * Returns <code>true</code> if an excerpt is generated by this index.<p>
807     *
808     * If no except is required, generation can be turned off in the index search configuration parameters
809     * in <code>opencms-search.xml</code>. Not generating an excerpt will improve performance.<p>
810     *
811     * @return <code>true</code> if an excerpt is generated by this index
812     */
813    public boolean isCreatingExcerpt() {
814
815        return m_createExcerpt;
816    }
817
818    /**
819     * Returns the ignoreExpiration.<p>
820     *
821     * @return the ignoreExpiration
822     */
823    public boolean isIgnoreExpiration() {
824
825        return m_ignoreExpiration;
826    }
827
828    /**
829     * @see org.opencms.search.A_CmsSearchIndex#isInitialized()
830     */
831    @Override
832    public boolean isInitialized() {
833
834        return super.isInitialized() && (null != getPath());
835    }
836
837    /**
838     * Returns <code>true</code> if a resource requires read permission to be included in the result list.<p>
839     *
840     * @return <code>true</code> if a resource requires read permission to be included in the result list
841     */
842    public boolean isRequireViewPermission() {
843
844        return m_requireViewPermission;
845    }
846
847    /**
848     * @see org.opencms.search.A_CmsSearchIndex#onIndexChanged(boolean)
849     */
850    @Override
851    public void onIndexChanged(boolean force) {
852
853        if (force) {
854            indexSearcherOpen(getPath());
855        } else {
856            indexSearcherUpdate();
857        }
858    }
859
860    /**
861     * Performs a search on the index within the given fields.<p>
862     *
863     * The result is returned as List with entries of type I_CmsSearchResult.<p>
864     *
865     * @param cms the current user's Cms object
866     * @param params the parameters to use for the search
867     *
868     * @return the List of results found or an empty list
869     *
870     * @throws CmsSearchException if something goes wrong
871     */
872    public CmsSearchResultList search(CmsObject cms, CmsSearchParameters params) throws CmsSearchException {
873
874        long timeTotal = -System.currentTimeMillis();
875        long timeLucene;
876        long timeResultProcessing;
877
878        if (LOG.isDebugEnabled()) {
879            LOG.debug(Messages.get().getBundle().key(Messages.LOG_SEARCH_PARAMS_2, params, getName()));
880        }
881
882        // the hits found during the search
883        TopDocs hits;
884
885        // storage for the results found
886        CmsSearchResultList searchResults = new CmsSearchResultList();
887
888        int previousPriority = Thread.currentThread().getPriority();
889
890        try {
891            // copy the user OpenCms context
892            CmsObject searchCms = OpenCms.initCmsObject(cms);
893
894            if (getPriority() > 0) {
895                // change thread priority in order to reduce search impact on overall system performance
896                Thread.currentThread().setPriority(getPriority());
897            }
898
899            // change the project
900            searchCms.getRequestContext().setCurrentProject(searchCms.readProject(getProject()));
901
902            timeLucene = -System.currentTimeMillis();
903
904            // several search options are searched using filters
905            BooleanQuery.Builder builder = new BooleanQuery.Builder();
906            // append root path filter
907            builder = appendPathFilter(searchCms, builder, params.getRoots());
908            // append category filter
909            builder = appendCategoryFilter(searchCms, builder, params.getCategories());
910            // append resource type filter
911            builder = appendResourceTypeFilter(searchCms, builder, params.getResourceTypes());
912
913            // append date last modified filter
914            builder = appendDateLastModifiedFilter(
915                builder,
916                params.getMinDateLastModified(),
917                params.getMaxDateLastModified());
918            // append date created filter
919            builder = appendDateCreatedFilter(builder, params.getMinDateCreated(), params.getMaxDateCreated());
920
921            // the search query to use, will be constructed in the next lines
922            Query query = null;
923            // store separate fields query for excerpt highlighting
924            Query fieldsQuery = null;
925
926            // get an index searcher that is certainly up to date
927            indexSearcherUpdate();
928            IndexSearcher searcher = getSearcher();
929
930            if (!params.isIgnoreQuery()) {
931                // since OpenCms 8 the query can be empty in which case only filters are used for the result
932                if (params.getParsedQuery() != null) {
933                    // the query was already build, re-use it
934                    QueryParser p = new QueryParser(CmsSearchField.FIELD_CONTENT, getAnalyzer());
935                    fieldsQuery = p.parse(params.getParsedQuery());
936                } else if (params.getFieldQueries() != null) {
937                    // each field has an individual query
938                    BooleanQuery.Builder mustOccur = null;
939                    BooleanQuery.Builder shouldOccur = null;
940                    for (CmsSearchParameters.CmsSearchFieldQuery fq : params.getFieldQueries()) {
941                        // add one sub-query for each defined field
942                        QueryParser p = new QueryParser(fq.getFieldName(), getAnalyzer());
943                        // first generate the combined keyword query
944                        Query keywordQuery = null;
945                        if (fq.getSearchTerms().size() == 1) {
946                            // this is just a single size keyword list
947                            keywordQuery = p.parse(fq.getSearchTerms().get(0));
948                        } else {
949                            // multiple size keyword list
950                            BooleanQuery.Builder keywordListQuery = new BooleanQuery.Builder();
951                            for (String keyword : fq.getSearchTerms()) {
952                                keywordListQuery.add(p.parse(keyword), fq.getTermOccur());
953                            }
954                            keywordQuery = keywordListQuery.build();
955                        }
956                        if (BooleanClause.Occur.SHOULD.equals(fq.getOccur())) {
957                            if (shouldOccur == null) {
958                                shouldOccur = new BooleanQuery.Builder();
959                            }
960                            shouldOccur.add(keywordQuery, fq.getOccur());
961                        } else {
962                            if (mustOccur == null) {
963                                mustOccur = new BooleanQuery.Builder();
964                            }
965                            mustOccur.add(keywordQuery, fq.getOccur());
966                        }
967                    }
968                    BooleanQuery.Builder booleanFieldsQuery = new BooleanQuery.Builder();
969                    if (mustOccur != null) {
970                        booleanFieldsQuery.add(mustOccur.build(), BooleanClause.Occur.MUST);
971                    }
972                    if (shouldOccur != null) {
973                        booleanFieldsQuery.add(shouldOccur.build(), BooleanClause.Occur.MUST);
974                    }
975                    fieldsQuery = searcher.rewrite(booleanFieldsQuery.build());
976                } else if ((params.getFields() != null) && (params.getFields().size() > 0)) {
977                    // no individual field queries have been defined, so use one query for all fields
978                    BooleanQuery.Builder booleanFieldsQuery = new BooleanQuery.Builder();
979                    // this is a "regular" query over one or more fields
980                    // add one sub-query for each of the selected fields, e.g. "content", "title" etc.
981                    for (int i = 0; i < params.getFields().size(); i++) {
982                        QueryParser p = new QueryParser(params.getFields().get(i), getAnalyzer());
983                        p.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);
984                        booleanFieldsQuery.add(p.parse(params.getQuery()), BooleanClause.Occur.SHOULD);
985                    }
986                    fieldsQuery = searcher.rewrite(booleanFieldsQuery.build());
987                } else {
988                    // if no fields are provided, just use the "content" field by default
989                    QueryParser p = new QueryParser(CmsSearchField.FIELD_CONTENT, getAnalyzer());
990                    fieldsQuery = searcher.rewrite(p.parse(params.getQuery()));
991                }
992
993                // finally set the main query to the fields query
994                // please note that we still need both variables in case the query is a MatchAllDocsQuery - see below
995                query = fieldsQuery;
996            }
997
998            if (LOG.isDebugEnabled()) {
999                LOG.debug(Messages.get().getBundle().key(Messages.LOG_BASE_QUERY_1, query));
1000            }
1001
1002            if (query == null) {
1003                // if no text query is set, then we match all documents
1004                query = new MatchAllDocsQuery();
1005            } else {
1006                // store the parsed query for page browsing
1007                params.setParsedQuery(query.toString(CmsSearchField.FIELD_CONTENT));
1008            }
1009
1010            // build the final query
1011            final BooleanQuery.Builder finalQueryBuilder = new BooleanQuery.Builder();
1012            finalQueryBuilder.add(query, BooleanClause.Occur.MUST);
1013            finalQueryBuilder.add(builder.build(), BooleanClause.Occur.FILTER);
1014            final BooleanQuery finalQuery = finalQueryBuilder.build();
1015
1016            // collect the categories
1017            CmsSearchCategoryCollector categoryCollector;
1018            if (params.isCalculateCategories()) {
1019                // USE THIS OPTION WITH CAUTION
1020                // this may slow down searched by an order of magnitude
1021                categoryCollector = new CmsSearchCategoryCollector(searcher);
1022                // perform a first search to collect the categories
1023                searcher.search(finalQuery, categoryCollector);
1024                // store the result
1025                searchResults.setCategories(categoryCollector.getCategoryCountResult());
1026            }
1027
1028            // get maxScore first, since Lucene 8, it's not computed automatically anymore
1029            TopDocs scoreHits = searcher.search(query, 1);
1030            float maxScore = scoreHits.scoreDocs.length == 0 ? Float.NaN : scoreHits.scoreDocs[0].score;
1031            // perform the search operation
1032            if ((params.getSort() == null) || (params.getSort() == CmsSearchParameters.SORT_DEFAULT)) {
1033                // apparently scoring is always enabled by Lucene if no sort order is provided
1034                hits = searcher.search(finalQuery, getMaxHits());
1035            } else {
1036                // if  a sort order is provided, we must check if scoring must be calculated by the searcher
1037                boolean isSortScore = isSortScoring(searcher, params.getSort());
1038                hits = searcher.search(finalQuery, getMaxHits(), params.getSort(), isSortScore);
1039            }
1040
1041            timeLucene += System.currentTimeMillis();
1042            timeResultProcessing = -System.currentTimeMillis();
1043
1044            if (hits != null) {
1045                long hitCount = hits.totalHits.value > hits.scoreDocs.length
1046                ? hits.scoreDocs.length
1047                : hits.totalHits.value;
1048                int page = params.getSearchPage();
1049                long start = -1, end = -1;
1050                if ((params.getMatchesPerPage() > 0) && (page > 0) && (hitCount > 0)) {
1051                    // calculate the final size of the search result
1052                    start = params.getMatchesPerPage() * (page - 1);
1053                    end = start + params.getMatchesPerPage();
1054                    // ensure that both i and n are inside the range of foundDocuments.size()
1055                    start = (start > hitCount) ? hitCount : start;
1056                    end = (end > hitCount) ? hitCount : end;
1057                } else {
1058                    // return all found documents in the search result
1059                    start = 0;
1060                    end = hitCount;
1061                }
1062
1063                Set<String> returnFields = ((CmsLuceneFieldConfiguration)getFieldConfiguration()).getReturnFields();
1064                Set<String> excerptFields = ((CmsLuceneFieldConfiguration)getFieldConfiguration()).getExcerptFields();
1065
1066                long visibleHitCount = hitCount;
1067                for (int i = 0, cnt = 0; (i < hitCount) && (cnt < end); i++) {
1068                    try {
1069                        Document doc = searcher.doc(hits.scoreDocs[i].doc, returnFields);
1070                        I_CmsSearchDocument searchDoc = new CmsLuceneDocument(doc);
1071                        searchDoc.setScore(hits.scoreDocs[i].score);
1072                        if ((isInTimeRange(doc, params)) && (hasReadPermission(searchCms, searchDoc))) {
1073                            // user has read permission
1074                            if (cnt >= start) {
1075                                // do not use the resource to obtain the raw content, read it from the lucene document!
1076                                String excerpt = null;
1077                                if (isCreatingExcerpt() && (fieldsQuery != null)) {
1078                                    Document exDoc = searcher.doc(hits.scoreDocs[i].doc, excerptFields);
1079                                    I_CmsTermHighlighter highlighter = OpenCms.getSearchManager().getHighlighter();
1080                                    excerpt = highlighter.getExcerpt(exDoc, this, params, fieldsQuery, getAnalyzer());
1081                                }
1082                                int score = Math.round(
1083                                    (maxScore != Float.NaN ? (hits.scoreDocs[i].score / maxScore) * 100f : 0));
1084                                searchResults.add(new CmsSearchResult(score, doc, excerpt));
1085                            }
1086                            cnt++;
1087                        } else {
1088                            visibleHitCount--;
1089                        }
1090                    } catch (Exception e) {
1091                        // should not happen, but if it does we want to go on with the next result nevertheless
1092                        if (LOG.isWarnEnabled()) {
1093                            LOG.warn(Messages.get().getBundle().key(Messages.LOG_RESULT_ITERATION_FAILED_0), e);
1094                        }
1095                    }
1096                }
1097
1098                // save the total count of search results
1099                searchResults.setHitCount((int)visibleHitCount);
1100            } else {
1101                searchResults.setHitCount(0);
1102            }
1103
1104            timeResultProcessing += System.currentTimeMillis();
1105        } catch (RuntimeException e) {
1106            throw new CmsSearchException(Messages.get().container(Messages.ERR_SEARCH_PARAMS_1, params), e);
1107        } catch (Exception e) {
1108            throw new CmsSearchException(Messages.get().container(Messages.ERR_SEARCH_PARAMS_1, params), e);
1109        } finally {
1110
1111            // re-set thread to previous priority
1112            Thread.currentThread().setPriority(previousPriority);
1113        }
1114
1115        if (LOG.isDebugEnabled()) {
1116            timeTotal += System.currentTimeMillis();
1117            Object[] logParams = new Object[] {
1118                new Long(hits == null ? 0 : hits.totalHits.value),
1119                new Long(timeTotal),
1120                new Long(timeLucene),
1121                new Long(timeResultProcessing)};
1122            LOG.debug(Messages.get().getBundle().key(Messages.LOG_STAT_RESULTS_TIME_4, logParams));
1123        }
1124
1125        return searchResults;
1126    }
1127
1128    /**
1129     * Sets the Lucene analyzer used for this index.<p>
1130     *
1131     * @param analyzer the Lucene analyzer to set
1132     */
1133    public void setAnalyzer(Analyzer analyzer) {
1134
1135        m_analyzer = analyzer;
1136    }
1137
1138    /**
1139     * Sets the checkPermissions.<p>
1140     *
1141     * @param checkPermissions the checkPermissions to set
1142     */
1143    public void setCheckPermissions(boolean checkPermissions) {
1144
1145        m_checkPermissions = checkPermissions;
1146    }
1147
1148    /**
1149     * Sets the ignoreExpiration.<p>
1150     *
1151     * @param ignoreExpiration the ignoreExpiration to set
1152     */
1153    public void setIgnoreExpiration(boolean ignoreExpiration) {
1154
1155        m_ignoreExpiration = ignoreExpiration;
1156    }
1157
1158    /**
1159     * Sets the number of how many hits are loaded at maximum.<p>
1160     *
1161     * This must be set at least to 50, or this setting is ignored.<p>
1162     *
1163     * @param maxHits the number of how many hits are loaded at maximum to set
1164     *
1165     * @see #getMaxHits()
1166     *
1167     * @since 7.5.1
1168     */
1169    public void setMaxHits(int maxHits) {
1170
1171        if (m_maxHits >= (MAX_HITS_DEFAULT / 100)) {
1172            m_maxHits = maxHits;
1173        }
1174    }
1175
1176    /**
1177     * Controls if a resource requires view permission to be displayed in the result list.<p>
1178     *
1179     * By default this is <code>false</code>.<p>
1180     *
1181     * @param requireViewPermission controls if a resource requires view permission to be displayed in the result list
1182     */
1183    public void setRequireViewPermission(boolean requireViewPermission) {
1184
1185        m_requireViewPermission = requireViewPermission;
1186    }
1187
1188    /**
1189     * Shuts down the search index.<p>
1190     *
1191     * This will close the local Lucene index searcher instance.<p>
1192     */
1193    @Override
1194    public void shutDown() {
1195
1196        super.shutDown();
1197        indexSearcherClose();
1198        if (m_analyzer != null) {
1199            m_analyzer.close();
1200        }
1201        if (CmsLog.INIT.isInfoEnabled()) {
1202            CmsLog.INIT.info(Messages.get().getBundle().key(Messages.INIT_SHUTDOWN_INDEX_1, getName()));
1203        }
1204    }
1205
1206    /**
1207     * Returns the name (<code>{@link #getName()}</code>) of this search index.<p>
1208     *
1209     * @return the name (<code>{@link #getName()}</code>) of this search index
1210     *
1211     * @see java.lang.Object#toString()
1212     */
1213    @Override
1214    public String toString() {
1215
1216        return getName();
1217    }
1218
1219    /**
1220     * Appends the a category filter to the given filter clause that matches all given categories.<p>
1221     *
1222     * In case the provided List is null or empty, the original filter is left unchanged.<p>
1223     *
1224     * The original filter parameter is extended and also provided as return value.<p>
1225     *
1226     * @param cms the current OpenCms search context
1227     * @param filter the filter to extend
1228     * @param categories the categories that will compose the filter
1229     *
1230     * @return the extended filter clause
1231     */
1232    protected BooleanQuery.Builder appendCategoryFilter(
1233        CmsObject cms,
1234        BooleanQuery.Builder filter,
1235        List<String> categories) {
1236
1237        if ((categories != null) && (categories.size() > 0)) {
1238            // add query categories (if required)
1239
1240            // categories are indexed as lower-case strings
1241            // @see org.opencms.search.fields.CmsSearchFieldConfiguration#appendCategories
1242            List<String> lowerCaseCategories = new ArrayList<String>();
1243            for (String category : categories) {
1244                lowerCaseCategories.add(category.toLowerCase());
1245            }
1246            filter.add(
1247                new BooleanClause(
1248                    getMultiTermQueryFilter(CmsSearchField.FIELD_CATEGORY, lowerCaseCategories),
1249                    BooleanClause.Occur.MUST));
1250        }
1251
1252        return filter;
1253    }
1254
1255    /**
1256     * Appends a date of creation filter to the given filter clause that matches the
1257     * given time range.<p>
1258     *
1259     * If the start time is equal to {@link Long#MIN_VALUE} and the end time is equal to {@link Long#MAX_VALUE}
1260     * than the original filter is left unchanged.<p>
1261     *
1262     * The original filter parameter is extended and also provided as return value.<p>
1263     *
1264     * @param filter the filter to extend
1265     * @param startTime start time of the range to search in
1266     * @param endTime end time of the range to search in
1267     *
1268     * @return the extended filter clause
1269     */
1270    protected BooleanQuery.Builder appendDateCreatedFilter(BooleanQuery.Builder filter, long startTime, long endTime) {
1271
1272        // create special optimized sub-filter for the date last modified search
1273        Query dateFilter = createDateRangeFilter(CmsSearchField.FIELD_DATE_CREATED_LOOKUP, startTime, endTime);
1274        if (dateFilter != null) {
1275            // extend main filter with the created date filter
1276            filter.add(new BooleanClause(dateFilter, BooleanClause.Occur.MUST));
1277        }
1278
1279        return filter;
1280    }
1281
1282    /**
1283     * Appends a date of last modification filter to the given filter clause that matches the
1284     * given time range.<p>
1285     *
1286     * If the start time is equal to {@link Long#MIN_VALUE} and the end time is equal to {@link Long#MAX_VALUE}
1287     * than the original filter is left unchanged.<p>
1288     *
1289     * The original filter parameter is extended and also provided as return value.<p>
1290     *
1291     * @param filter the filter to extend
1292     * @param startTime start time of the range to search in
1293     * @param endTime end time of the range to search in
1294     *
1295     * @return the extended filter clause
1296     */
1297    protected BooleanQuery.Builder appendDateLastModifiedFilter(
1298        BooleanQuery.Builder filter,
1299        long startTime,
1300        long endTime) {
1301
1302        // create special optimized sub-filter for the date last modified search
1303        Query dateFilter = createDateRangeFilter(CmsSearchField.FIELD_DATE_LASTMODIFIED_LOOKUP, startTime, endTime);
1304        if (dateFilter != null) {
1305            // extend main filter with the created date filter
1306            filter.add(new BooleanClause(dateFilter, BooleanClause.Occur.MUST));
1307        }
1308
1309        return filter;
1310    }
1311
1312    /**
1313     * Appends the a VFS path filter to the given filter clause that matches all given root paths.<p>
1314     *
1315     * In case the provided List is null or empty, the current request context site root is appended.<p>
1316     *
1317     * The original filter parameter is extended and also provided as return value.<p>
1318     *
1319     * @param cms the current OpenCms search context
1320     * @param filter the filter to extend
1321     * @param roots the VFS root paths that will compose the filter
1322     *
1323     * @return the extended filter clause
1324     */
1325    protected BooleanQuery.Builder appendPathFilter(CmsObject cms, BooleanQuery.Builder filter, List<String> roots) {
1326
1327        // complete the search root
1328        List<Term> terms = new ArrayList<Term>();
1329        if ((roots != null) && (roots.size() > 0)) {
1330            // add the all configured search roots with will request context
1331            for (int i = 0; i < roots.size(); i++) {
1332                String searchRoot = cms.getRequestContext().addSiteRoot(roots.get(i));
1333                extendPathFilter(terms, searchRoot);
1334            }
1335        } else {
1336            // use the current site root as the search root
1337            extendPathFilter(terms, cms.getRequestContext().getSiteRoot());
1338            // also add the shared folder (v 8.0)
1339            if (OpenCms.getSiteManager().getSharedFolder() != null) {
1340                extendPathFilter(terms, OpenCms.getSiteManager().getSharedFolder());
1341            }
1342        }
1343
1344        // add the calculated path filter for the root path
1345        BooleanQuery.Builder build = new BooleanQuery.Builder();
1346        terms.forEach(term -> build.add(new TermQuery(term), Occur.SHOULD));
1347        filter.add(new BooleanClause(build.build(), BooleanClause.Occur.MUST));
1348        return filter;
1349    }
1350
1351    /**
1352     * Appends the a resource type filter to the given filter clause that matches all given resource types.<p>
1353     *
1354     * In case the provided List is null or empty, the original filter is left unchanged.<p>
1355     *
1356     * The original filter parameter is extended and also provided as return value.<p>
1357     *
1358     * @param cms the current OpenCms search context
1359     * @param filter the filter to extend
1360     * @param resourceTypes the resource types that will compose the filter
1361     *
1362     * @return the extended filter clause
1363     */
1364    protected BooleanQuery.Builder appendResourceTypeFilter(
1365        CmsObject cms,
1366        BooleanQuery.Builder filter,
1367        List<String> resourceTypes) {
1368
1369        if ((resourceTypes != null) && (resourceTypes.size() > 0)) {
1370            // add query resource types (if required)
1371            filter.add(
1372                new BooleanClause(
1373                    getMultiTermQueryFilter(CmsSearchField.FIELD_TYPE, resourceTypes),
1374                    BooleanClause.Occur.MUST));
1375        }
1376
1377        return filter;
1378    }
1379
1380    /**
1381     * Creates an optimized date range filter for the date of last modification or creation.<p>
1382     *
1383     * If the start date is equal to {@link Long#MIN_VALUE} and the end date is equal to {@link Long#MAX_VALUE}
1384     * than <code>null</code> is returned.<p>
1385     *
1386     * @param fieldName the name of the field to search
1387     * @param startTime start time of the range to search in
1388     * @param endTime end time of the range to search in
1389     *
1390     * @return an optimized date range filter for the date of last modification or creation
1391     */
1392    protected Query createDateRangeFilter(String fieldName, long startTime, long endTime) {
1393
1394        Query filter = null;
1395        if ((startTime != Long.MIN_VALUE) || (endTime != Long.MAX_VALUE)) {
1396            // a date range has been set for this document search
1397            if (startTime == Long.MIN_VALUE) {
1398                // default start will always be "yyyy1231" in order to reduce term size
1399                Calendar cal = Calendar.getInstance(OpenCms.getLocaleManager().getTimeZone());
1400                cal.setTimeInMillis(endTime);
1401                cal.set(cal.get(Calendar.YEAR) - MAX_YEAR_RANGE, 11, 31, 0, 0, 0);
1402                startTime = cal.getTimeInMillis();
1403            } else if (endTime == Long.MAX_VALUE) {
1404                // default end will always be "yyyy0101" in order to reduce term size
1405                Calendar cal = Calendar.getInstance(OpenCms.getLocaleManager().getTimeZone());
1406                cal.setTimeInMillis(startTime);
1407                cal.set(cal.get(Calendar.YEAR) + MAX_YEAR_RANGE, 0, 1, 0, 0, 0);
1408                endTime = cal.getTimeInMillis();
1409            }
1410
1411            // get the list of all possible date range options
1412            List<String> dateRange = getDateRangeSpan(startTime, endTime);
1413            List<Term> terms = new ArrayList<Term>();
1414            for (String range : dateRange) {
1415                terms.add(new Term(fieldName, range));
1416            }
1417            // create the filter for the date
1418            BooleanQuery.Builder build = new BooleanQuery.Builder();
1419            terms.forEach(term -> build.add(new TermQuery(term), Occur.SHOULD));
1420            filter = build.build();
1421        }
1422        return filter;
1423    }
1424
1425    /**
1426     * Creates a backup of this index for optimized re-indexing of the whole content.<p>
1427     *
1428     * @return the path to the backup folder, or <code>null</code> in case no backup was created
1429     */
1430    protected String createIndexBackup() {
1431
1432        if (!isBackupReindexing()) {
1433            // if no backup is generated we don't need to do anything
1434            return null;
1435        }
1436
1437        // check if the target directory already exists
1438        File file = new File(getPath());
1439        if (!file.exists()) {
1440            // index does not exist yet, so we can't backup it
1441            return null;
1442        }
1443        String backupPath = getPath() + "_backup";
1444        FSDirectory oldDir = null;
1445        FSDirectory newDir = null;
1446        try {
1447            // open file directory for Lucene
1448            oldDir = FSDirectory.open(file.toPath());
1449            newDir = FSDirectory.open(Paths.get(backupPath));
1450            for (String fileName : oldDir.listAll()) {
1451                newDir.copyFrom(oldDir, fileName, fileName, IOContext.DEFAULT);
1452            }
1453        } catch (Exception e) {
1454            LOG.error(
1455                Messages.get().getBundle().key(Messages.LOG_IO_INDEX_BACKUP_CREATE_3, getName(), getPath(), backupPath),
1456                e);
1457            backupPath = null;
1458        } finally {
1459            if (oldDir != null) {
1460                try {
1461                    oldDir.close();
1462                } catch (IOException e) {
1463                    e.printStackTrace();
1464                }
1465            }
1466            if (newDir != null) {
1467                try {
1468                    newDir.close();
1469                } catch (IOException e) {
1470                    e.printStackTrace();
1471                }
1472            }
1473        }
1474        return backupPath;
1475    }
1476
1477    /**
1478     * Creates a new index writer.<p>
1479     *
1480     * @param create if <code>true</code> a whole new index is created, if <code>false</code> an existing index is updated
1481     * @param report the report
1482     *
1483     * @return the created new index writer
1484     *
1485     * @throws CmsIndexException in case the writer could not be created
1486     *
1487     * @see #getIndexWriter(I_CmsReport, boolean)
1488     */
1489    @Override
1490    protected I_CmsIndexWriter createIndexWriter(boolean create, I_CmsReport report) throws CmsIndexException {
1491
1492        IndexWriter indexWriter = null;
1493        FSDirectory dir = null;
1494        try {
1495            File f = new File(getPath());
1496            if (!f.exists()) {
1497                f = f.getParentFile();
1498                if ((f != null) && (!f.exists())) {
1499                    f.mkdirs();
1500                }
1501
1502                create = true;
1503            }
1504
1505            dir = FSDirectory.open(Paths.get(getPath()));
1506            IndexWriterConfig indexConfig = new IndexWriterConfig(getAnalyzer());
1507            //indexConfig.setMergePolicy(mergePolicy);
1508
1509            if (m_luceneRAMBufferSizeMB != null) {
1510                indexConfig.setRAMBufferSizeMB(m_luceneRAMBufferSizeMB.doubleValue());
1511            }
1512            if (create) {
1513                indexConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
1514            } else {
1515                indexConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
1516            }
1517            // register the modified default similarity implementation
1518            indexConfig.setSimilarity(m_sim);
1519
1520            indexWriter = new IndexWriter(dir, indexConfig);
1521        } catch (Exception e) {
1522            if (dir != null) {
1523                try {
1524                    dir.close();
1525                } catch (IOException e1) {
1526                    // TODO Auto-generated catch block
1527                    e1.printStackTrace();
1528                }
1529            }
1530            if (indexWriter != null) {
1531                try {
1532                    indexWriter.close();
1533                } catch (IOException closeExeception) {
1534                    throw new CmsIndexException(
1535                        Messages.get().container(Messages.ERR_IO_INDEX_WRITER_OPEN_2, getPath(), getName()),
1536                        e);
1537                }
1538            }
1539            throw new CmsIndexException(
1540                Messages.get().container(Messages.ERR_IO_INDEX_WRITER_OPEN_2, getPath(), getName()),
1541                e);
1542        }
1543
1544        return new CmsLuceneIndexWriter(indexWriter, this);
1545    }
1546
1547    /**
1548     * Extends the given path query with another term for the given search root element.<p>
1549     *
1550     * @param terms the path filter to extend
1551     * @param searchRoot the search root to add to the path query
1552     */
1553    protected void extendPathFilter(List<Term> terms, String searchRoot) {
1554
1555        if (!CmsResource.isFolder(searchRoot)) {
1556            searchRoot += "/";
1557        }
1558        terms.add(new Term(CmsSearchField.FIELD_PARENT_FOLDERS, searchRoot));
1559    }
1560
1561    /**
1562     * Generates the directory on the RFS for this index.<p>
1563     *
1564     * @return the directory on the RFS for this index
1565     */
1566    protected String generateIndexDirectory() {
1567
1568        return OpenCms.getSystemInfo().getAbsoluteRfsPathRelativeToWebInf(
1569            OpenCms.getSearchManager().getDirectory() + "/" + getName());
1570    }
1571
1572    /**
1573     * Returns a cached Lucene term query filter for the given field and terms.<p>
1574     *
1575     * @param field the field to use
1576     * @param terms the term to use
1577     *
1578     * @return a cached Lucene term query filter for the given field and terms
1579     */
1580    protected Query getMultiTermQueryFilter(String field, List<String> terms) {
1581
1582        return getMultiTermQueryFilter(field, null, terms);
1583    }
1584
1585    /**
1586     * Returns a cached Lucene term query filter for the given field and terms.<p>
1587     *
1588     * @param field the field to use
1589     * @param terms the term to use
1590     *
1591     * @return a cached Lucene term query filter for the given field and terms
1592     */
1593    protected Query getMultiTermQueryFilter(String field, String terms) {
1594
1595        return getMultiTermQueryFilter(field, terms, null);
1596    }
1597
1598    /**
1599     * Returns a cached Lucene term query filter for the given field and terms.<p>
1600     *
1601     * @param field the field to use
1602     * @param termsStr the terms to use as a String separated by a space ' ' char
1603     * @param termsList the list of terms to use
1604     *
1605     * @return a cached Lucene term query filter for the given field and terms
1606     */
1607    protected Query getMultiTermQueryFilter(String field, String termsStr, List<String> termsList) {
1608
1609        if (termsStr == null) {
1610            StringBuffer buf = new StringBuffer(64);
1611            for (int i = 0; i < termsList.size(); i++) {
1612                if (i > 0) {
1613                    buf.append(' ');
1614                }
1615                buf.append(termsList.get(i));
1616            }
1617            termsStr = buf.toString();
1618        }
1619        Query result = m_displayFilters.get(
1620            (new StringBuffer(64)).append(field).append('|').append(termsStr).toString());
1621        if (result == null) {
1622            List<Term> terms = new ArrayList<Term>();
1623            if (termsList == null) {
1624                termsList = CmsStringUtil.splitAsList(termsStr, ' ');
1625            }
1626            for (int i = 0; i < termsList.size(); i++) {
1627                terms.add(new Term(field, termsList.get(i)));
1628            }
1629
1630            BooleanQuery.Builder build = new BooleanQuery.Builder();
1631            terms.forEach(term -> build.add(new TermQuery(term), Occur.SHOULD));
1632            Query termsQuery = build.build(); //termsFilter
1633
1634            try {
1635                result = termsQuery.createWeight(m_indexSearcher, ScoreMode.COMPLETE_NO_SCORES, 1).getQuery();
1636                m_displayFilters.put(field + termsStr, result);
1637            } catch (IOException e) {
1638                // TODO don't know what happend
1639                e.printStackTrace();
1640            }
1641        }
1642        return result;
1643    }
1644
1645    /**
1646     * Checks if the OpenCms resource referenced by the result document can be read
1647     * by the user of the given OpenCms context.
1648     *
1649     * Returns the referenced <code>CmsResource</code> or <code>null</code> if
1650     * the user is not permitted to read the resource.<p>
1651     *
1652     * @param cms the OpenCms user context to use for permission testing
1653     * @param doc the search result document to check
1654     *
1655     * @return the referenced <code>CmsResource</code> or <code>null</code> if the user is not permitted
1656     */
1657    protected CmsResource getResource(CmsObject cms, I_CmsSearchDocument doc) {
1658
1659        // check if the resource exits in the VFS,
1660        // this will implicitly check read permission and if the resource was deleted
1661        CmsResourceFilter filter = CmsResourceFilter.DEFAULT;
1662        if (isRequireViewPermission()) {
1663            filter = CmsResourceFilter.DEFAULT_ONLY_VISIBLE;
1664        } else if (isIgnoreExpiration()) {
1665            filter = CmsResourceFilter.IGNORE_EXPIRATION;
1666        }
1667
1668        return getResource(cms, doc, filter);
1669    }
1670
1671    /**
1672     * Checks if the OpenCms resource referenced by the result document can be read
1673     * by the user of the given OpenCms context.
1674     *
1675     * Returns the referenced <code>CmsResource</code> or <code>null</code> if
1676     * the user is not permitted to read the resource.<p>
1677     *
1678     * @param cms the OpenCms user context to use for permission testing
1679     * @param doc the search result document to check
1680     * @param filter the resource filter to apply
1681     *
1682     * @return the referenced <code>CmsResource</code> or <code>null</code> if the user is not permitted
1683     */
1684    protected CmsResource getResource(CmsObject cms, I_CmsSearchDocument doc, CmsResourceFilter filter) {
1685
1686        try {
1687            CmsObject clone = OpenCms.initCmsObject(cms);
1688            clone.getRequestContext().setSiteRoot("");
1689            return clone.readResource(doc.getPath(), filter);
1690        } catch (CmsException e) {
1691            // Do nothing
1692        }
1693
1694        return null;
1695    }
1696
1697    /**
1698     * Returns a cached Lucene term query filter for the given field and term.<p>
1699     *
1700     * @param field the field to use
1701     * @param term the term to use
1702     *
1703     * @return a cached Lucene term query filter for the given field and term
1704     */
1705    protected Query getTermQueryFilter(String field, String term) {
1706
1707        return getMultiTermQueryFilter(field, term, Collections.singletonList(term));
1708    }
1709
1710    /**
1711     * Checks if the OpenCms resource referenced by the result document can be read
1712     * be the user of the given OpenCms context.<p>
1713     *
1714     * @param cms the OpenCms user context to use for permission testing
1715     * @param doc the search result document to check
1716     * @return <code>true</code> if the user has read permissions to the resource
1717     */
1718    protected boolean hasReadPermission(CmsObject cms, I_CmsSearchDocument doc) {
1719
1720        // If no permission check is needed: the document can be read
1721        // Else try to read the resource if this is not possible the user does not have enough permissions
1722        return !needsPermissionCheck(doc) ? true : (null != getResource(cms, doc));
1723    }
1724
1725    /**
1726     * Closes the index searcher for this index.<p>
1727     *
1728     * @see #indexSearcherOpen(String)
1729     */
1730    protected synchronized void indexSearcherClose() {
1731
1732        indexSearcherClose(m_indexSearcher);
1733    }
1734
1735    /**
1736     * Closes the given Lucene index searcher.<p>
1737     *
1738     * @param searcher the searcher to close
1739     */
1740    protected synchronized void indexSearcherClose(IndexSearcher searcher) {
1741
1742        // in case there is an index searcher available close it
1743        if ((searcher != null) && (searcher.getIndexReader() != null)) {
1744            try {
1745                searcher.getIndexReader().close();
1746            } catch (Exception e) {
1747                LOG.error(Messages.get().getBundle().key(Messages.ERR_INDEX_SEARCHER_CLOSE_1, getName()), e);
1748            }
1749        }
1750    }
1751
1752    /**
1753     * Initializes the index searcher for this index.<p>
1754     *
1755     * In case there is an index searcher still open, it is closed first.<p>
1756     *
1757     * For performance reasons, one instance of the index searcher should be kept
1758     * for all searches. However, if the index is updated or changed
1759     * this searcher instance needs to be re-initialized.<p>
1760     *
1761     * @param path the path to the index directory
1762     */
1763    protected synchronized void indexSearcherOpen(String path) {
1764
1765        IndexSearcher oldSearcher = null;
1766        Directory indexDirectory = null;
1767        try {
1768            indexDirectory = FSDirectory.open(Paths.get(path));
1769            if (DirectoryReader.indexExists(indexDirectory)) {
1770                IndexReader reader = UninvertingReader.wrap(
1771                    DirectoryReader.open(indexDirectory),
1772                    createUninvertingMap());
1773                if (m_indexSearcher != null) {
1774                    // store old searcher instance to close it later
1775                    oldSearcher = m_indexSearcher;
1776                }
1777                m_indexSearcher = new IndexSearcher(reader);
1778                m_indexSearcher.setSimilarity(m_sim);
1779                m_displayFilters = new HashMap<>();
1780            }
1781        } catch (IOException e) {
1782            LOG.error(Messages.get().getBundle().key(Messages.ERR_INDEX_SEARCHER_1, getName()), e);
1783            if (indexDirectory != null) {
1784                try {
1785                    indexDirectory.close();
1786                } catch (IOException closeException) {
1787                    // do nothing
1788                }
1789            }
1790        }
1791        if (oldSearcher != null) {
1792            // close the old searcher if required
1793            indexSearcherClose(oldSearcher);
1794        }
1795    }
1796
1797    /**
1798     * Reopens the index search reader for this index, required after the index has been changed.<p>
1799     *
1800     * @see #indexSearcherOpen(String)
1801     */
1802    protected synchronized void indexSearcherUpdate() {
1803
1804        IndexSearcher oldSearcher = m_indexSearcher;
1805        if ((oldSearcher != null) && (oldSearcher.getIndexReader() != null)) {
1806            // in case there is an index searcher available close it
1807            try {
1808                if (oldSearcher.getIndexReader() instanceof DirectoryReader) {
1809                    IndexReader newReader = DirectoryReader.openIfChanged(
1810                        (DirectoryReader)oldSearcher.getIndexReader());
1811                    if (newReader != null) {
1812                        m_indexSearcher = new IndexSearcher(newReader);
1813                        m_indexSearcher.setSimilarity(m_sim);
1814                        indexSearcherClose(oldSearcher);
1815                    }
1816                }
1817            } catch (Exception e) {
1818                LOG.error(Messages.get().getBundle().key(Messages.ERR_INDEX_SEARCHER_REOPEN_1, getName()), e);
1819            }
1820        } else {
1821            // make sure we end up with an open index searcher / reader
1822            indexSearcherOpen(getPath());
1823        }
1824    }
1825
1826    /**
1827     * Checks if the document is in the time range specified in the search parameters.<p>
1828     *
1829     * The creation date and/or the last modification date are checked.<p>
1830     *
1831     * @param doc the document to check the dates against the given time range
1832     * @param params the search parameters where the time ranges are specified
1833     *
1834     * @return true if document is in time range or not time range set otherwise false
1835     */
1836    protected boolean isInTimeRange(Document doc, CmsSearchParameters params) {
1837
1838        if (!isCheckingTimeRange()) {
1839            // time range check disabled
1840            return true;
1841        }
1842
1843        try {
1844            // check the creation date of the document against the given time range
1845            Date dateCreated = DateTools.stringToDate(doc.getField(CmsSearchField.FIELD_DATE_CREATED).stringValue());
1846            if (dateCreated.getTime() < params.getMinDateCreated()) {
1847                return false;
1848            }
1849            if (dateCreated.getTime() > params.getMaxDateCreated()) {
1850                return false;
1851            }
1852
1853            // check the last modification date of the document against the given time range
1854            Date dateLastModified = DateTools.stringToDate(
1855                doc.getField(CmsSearchField.FIELD_DATE_LASTMODIFIED).stringValue());
1856            if (dateLastModified.getTime() < params.getMinDateLastModified()) {
1857                return false;
1858            }
1859            if (dateLastModified.getTime() > params.getMaxDateLastModified()) {
1860                return false;
1861            }
1862
1863        } catch (ParseException ex) {
1864            // date could not be parsed -> doc is in time range
1865        }
1866
1867        return true;
1868    }
1869
1870    /**
1871     * Checks if the score for the results must be calculated based on the provided sort option.<p>
1872     *
1873     * Since Lucene 3 apparently the score is no longer calculated by default, but only if the
1874     * searcher is explicitly told so. This methods checks if, based on the given sort,
1875     * the score must be calculated.<p>
1876     *
1877     * @param searcher the index searcher to prepare
1878     * @param sort the sort option to use
1879     *
1880     * @return true if the sort option should be used
1881     */
1882    protected boolean isSortScoring(IndexSearcher searcher, Sort sort) {
1883
1884        boolean doScoring = false;
1885        if (sort != null) {
1886            if ((sort == CmsSearchParameters.SORT_DEFAULT) || (sort == CmsSearchParameters.SORT_TITLE)) {
1887                // these default sorts do need score calculation
1888                doScoring = true;
1889            } else if ((sort == CmsSearchParameters.SORT_DATE_CREATED)
1890                || (sort == CmsSearchParameters.SORT_DATE_LASTMODIFIED)) {
1891                // these default sorts don't need score calculation
1892                doScoring = false;
1893            } else {
1894                // for all non-defaults: check if the score field is present, in that case we must calculate the score
1895                SortField[] fields = sort.getSort();
1896                for (SortField field : fields) {
1897                    if (field == SortField.FIELD_SCORE) {
1898                        doScoring = true;
1899                        break;
1900                    }
1901                }
1902            }
1903        }
1904        return doScoring;
1905    }
1906
1907    /**
1908     * Checks if the OpenCms resource referenced by the result document needs to be checked.<p>
1909     *
1910     * @param doc the search result document to check
1911     *
1912     * @return <code>true</code> if the document needs to be checked <code>false</code> otherwise
1913     */
1914    protected boolean needsPermissionCheck(I_CmsSearchDocument doc) {
1915
1916        if (!isCheckingPermissions()) {
1917            // no permission check is performed at all
1918            return false;
1919        }
1920
1921        if ((doc.getType() == null) || (doc.getPath() == null)) {
1922            // permission check needs only to be performed for VFS documents that contain both fields
1923            return false;
1924        }
1925
1926        if (!I_CmsSearchDocument.VFS_DOCUMENT_KEY_PREFIX.equals(doc.getType())
1927            && !OpenCms.getResourceManager().hasResourceType(doc.getType())) {
1928            // this is an unknown VFS resource type (also not the generic "VFS" type of OpenCms before 7.0)
1929            return false;
1930        }
1931        return true;
1932    }
1933
1934    /**
1935     * Removes the given backup folder of this index.<p>
1936     *
1937     * @param path the backup folder to remove
1938     */
1939    protected void removeIndexBackup(String path) {
1940
1941        if (!isBackupReindexing()) {
1942            // if no backup is generated we don't need to do anything
1943            return;
1944        }
1945
1946        // check if the target directory already exists
1947        File file = new File(path);
1948        if (!file.exists()) {
1949            // index does not exist yet
1950            return;
1951        }
1952        try {
1953            FSDirectory dir = FSDirectory.open(file.toPath());
1954            dir.close();
1955            CmsFileUtil.purgeDirectory(file);
1956        } catch (Exception e) {
1957            LOG.error(Messages.get().getBundle().key(Messages.LOG_IO_INDEX_BACKUP_REMOVE_2, getName(), path), e);
1958        }
1959    }
1960
1961    /**
1962     * Generates the uninverting map and adds it to the field configuration.
1963     * @return the generated uninverting map
1964     *
1965     * @see CmsSearchField#addUninvertingMappings(Map)
1966     */
1967    private Map<String, Type> createUninvertingMap() {
1968
1969        Map<String, UninvertingReader.Type> uninvertingMap = new HashMap<String, UninvertingReader.Type>();
1970        CmsSearchField.addUninvertingMappings(uninvertingMap);
1971        getFieldConfiguration().addUninvertingMappings(uninvertingMap);
1972        return uninvertingMap;
1973    }
1974
1975}