Source code

001/*
002 * This library is part of OpenCms -
003 * the Open Source Content Management System
004 *
005 * Copyright (c) Alkacon Software GmbH & Co. KG (https://www.alkacon.com)
006 *
007 * This library is free software; you can redistribute it and/or
008 * modify it under the terms of the GNU Lesser General Public
009 * License as published by the Free Software Foundation; either
010 * version 2.1 of the License, or (at your option) any later version.
011 *
012 * This library is distributed in the hope that it will be useful,
013 * but WITHOUT ANY WARRANTY; without even the implied warranty of
014 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
015 * Lesser General Public License for more details.
016 *
017 * For further information about Alkacon Software, please see the
018 * company website: https://www.alkacon.com
019 *
020 * For further information about OpenCms, please see the
021 * project website: https://www.opencms.org
022 *
023 * You should have received a copy of the GNU Lesser General Public
024 * License along with this library; if not, write to the Free Software
025 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
026 */
027
028package org.opencms.search;
029
030import org.opencms.ade.galleries.shared.CmsGallerySearchScope;
031import org.opencms.file.CmsObject;
032import org.opencms.file.CmsPropertyDefinition;
033import org.opencms.i18n.CmsEncoder;
034import org.opencms.main.CmsLog;
035import org.opencms.main.OpenCms;
036import org.opencms.search.galleries.CmsGallerySearchParameters;
037import org.opencms.site.CmsSiteManagerImpl;
038import org.opencms.util.CmsFileUtil;
039import org.opencms.util.CmsHtmlExtractor;
040import org.opencms.util.CmsStringUtil;
041
042import java.io.UnsupportedEncodingException;
043import java.text.DateFormat;
044import java.text.ParseException;
045import java.text.SimpleDateFormat;
046import java.util.ArrayList;
047import java.util.Arrays;
048import java.util.Calendar;
049import java.util.Collection;
050import java.util.Date;
051import java.util.Iterator;
052import java.util.List;
053import java.util.Locale;
054import java.util.TimeZone;
055import java.util.stream.Collectors;
056
057import org.apache.commons.logging.Log;
058import org.apache.solr.common.SolrDocument;
059import org.apache.solr.common.SolrInputDocument;
060import org.apache.solr.common.SolrInputField;
061import org.apache.solr.common.util.ContentStream;
062import org.apache.solr.common.util.ContentStreamBase;
063
064import org.htmlparser.util.ParserException;
065
066/**
067 * Provides common functions regarding searching.<p>
068 *
069 * @since 9.0.0
070 */
071public final class CmsSearchUtil {
072
073    /** Date format object that obeys ISO 8601 which is used by Solr. */
074    private static final DateFormat DATEFORMAT_ISO_8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
075
076    /** The log object for this class. */
077    private static final Log LOG = CmsLog.getLog(CmsSearchUtil.class);
078
079    /** Variable to hold an UTC timezone object. */
080    public static final TimeZone TIMEZONE_UTC = TimeZone.getTimeZone("UTC");
081
082    /** Variable to hold an GMT timezone object. */
083    public static final TimeZone TIMEZONE_GMT = TimeZone.getTimeZone("GMT");
084
085    //start HttpClient
086    /**
087     * Date format pattern used to parse HTTP date headers in RFC 1123 format.
088     */
089    public static final String PATTERN_RFC1123 = "EEE, dd MMM yyyy HH:mm:ss zzz";
090
091    /**
092     * Date format pattern used to parse HTTP date headers in RFC 1036 format.
093     */
094    public static final String PATTERN_RFC1036 = "EEEE, dd-MMM-yy HH:mm:ss zzz";
095
096    /**
097     * Date format pattern used to parse HTTP date headers in ANSI C
098     * <code>asctime()</code> format.
099     */
100    public static final String PATTERN_ASCTIME = "EEE MMM d HH:mm:ss yyyy";
101
102    //These are included for back compat
103    private static final Collection<String> DEFAULT_HTTP_CLIENT_PATTERNS = Arrays.asList(
104        PATTERN_ASCTIME,
105        PATTERN_RFC1036,
106        PATTERN_RFC1123);
107    //---------------------------------------------------------------------------------------
108
109    /**
110     * A suite of default date formats that can be parsed, and thus transformed to the Solr specific format
111     */
112    public static final Collection<String> DEFAULT_DATE_FORMATS = new ArrayList<>();
113
114    private static final Date DEFAULT_TWO_DIGIT_YEAR_START;
115
116    static {
117        DEFAULT_DATE_FORMATS.add("yyyy-MM-dd'T'HH:mm:ss'Z'");
118        DEFAULT_DATE_FORMATS.add("yyyy-MM-dd'T'HH:mm:ss");
119        DEFAULT_DATE_FORMATS.add("yyyy-MM-dd");
120        DEFAULT_DATE_FORMATS.add("yyyy-MM-dd hh:mm:ss");
121        DEFAULT_DATE_FORMATS.add("yyyy-MM-dd HH:mm:ss");
122        DEFAULT_DATE_FORMATS.add("EEE MMM d hh:mm:ss z yyyy");
123        DEFAULT_DATE_FORMATS.addAll(DEFAULT_HTTP_CLIENT_PATTERNS);
124
125        Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
126        calendar.set(2000, Calendar.JANUARY, 1, 0, 0);
127        DEFAULT_TWO_DIGIT_YEAR_START = calendar.getTime();
128    }
129
130    /**
131     * Private constructor of utitlity class.<p>
132     */
133    private CmsSearchUtil() {
134
135        // noop
136    }
137
138    /**
139     * Computes the search root folders for the given search parameters based on the search scope.<p>
140     *
141     * @param cms the current CMS context
142     * @param params the current search parameters
143     *
144     * @return the search root folders based on the search scope
145     */
146    public static List<String> computeScopeFolders(CmsObject cms, CmsGallerySearchParameters params) {
147
148        String subsite = null;
149        if (params.getReferencePath() != null) {
150            subsite = OpenCms.getADEManager().getSubSiteRoot(
151                cms,
152                cms.getRequestContext().addSiteRoot(params.getReferencePath()));
153            if (subsite != null) {
154                subsite = cms.getRequestContext().removeSiteRoot(subsite);
155            } else if (LOG.isWarnEnabled()) {
156                LOG.warn(
157                    Messages.get().getBundle().key(
158                        Messages.LOG_GALLERIES_COULD_NOT_EVALUATE_SUBSITE_1,
159                        params.getReferencePath()));
160            }
161        } else if (LOG.isWarnEnabled()) {
162            LOG.warn(Messages.get().getBundle().key(Messages.LOG_GALLERIES_NO_REFERENCE_PATH_PROVIDED_0));
163        }
164
165        List<String> scopeFolders = getSearchRootsForScope(
166            params.getScope(),
167            cms.getRequestContext().getSiteRoot(),
168            subsite);
169        return scopeFolders;
170    }
171
172    /**
173     * Returns a given date object in the ISO 8601 format.
174     *
175     * @param date that should be converted.
176     * @return string that represents the given date in the ISO 8601 format.
177     */
178    public static String getDateAsIso8601(Date date) {
179
180        synchronized (CmsSearchUtil.class) {
181            if (DATEFORMAT_ISO_8601.getTimeZone() != TIMEZONE_UTC) {
182                DATEFORMAT_ISO_8601.setTimeZone(TIMEZONE_UTC);
183            }
184            return DATEFORMAT_ISO_8601.format(date);
185        }
186    }
187
188    /**
189     * Returns a given date object in the ISO 8601 format.
190     *
191     * @param date that should be converted.
192     * @return string that represents the given date in the ISO 8601 format.
193     */
194    public static String getDateAsIso8601(long date) {
195
196        // Check if date is set
197        if ((date > Long.MIN_VALUE) && (date < Long.MAX_VALUE)) {
198            final Date d_date = new Date(date);
199            return getDateAsIso8601(d_date);
200        }
201        return null;
202    }
203
204    /**
205     * Returns a time interval as Solr compatible query string.
206     * @param searchField the field to search for.
207     * @param startTime the lower limit of the interval.
208     * @param endTime the upper limit of the interval.
209     * @return Solr compatible query string.
210     */
211    public static String getDateCreatedTimeRangeFilterQuery(String searchField, long startTime, long endTime) {
212
213        String sStartTime = null;
214        String sEndTime = null;
215
216        // Convert startTime to ISO 8601 format
217        if ((startTime > Long.MIN_VALUE) && (startTime < Long.MAX_VALUE)) {
218            sStartTime = CmsSearchUtil.getDateAsIso8601(new Date(startTime));
219        }
220
221        // Convert endTime to ISO 8601 format
222        if ((endTime > Long.MIN_VALUE) && (endTime < Long.MAX_VALUE)) {
223            sEndTime = CmsSearchUtil.getDateAsIso8601(new Date(endTime));
224        }
225
226        // Build Solr range string
227        final String rangeString = CmsSearchUtil.getSolrRangeString(sStartTime, sEndTime);
228
229        // Build Solr filter string
230        return String.format("%s:%s", searchField, rangeString);
231    }
232
233    /**
234     * Gets the search roots to use for the given site/subsite parameters.<p>
235     *
236     * @param scope the search scope
237     * @param siteParam the current site
238     * @param subSiteParam the current subsite
239     *
240     * @return the list of search roots for that option
241     */
242    public static List<String> getSearchRootsForScope(
243        CmsGallerySearchScope scope,
244        String siteParam,
245        String subSiteParam) {
246
247        List<String> result = new ArrayList<String>();
248        if (scope == CmsGallerySearchScope.everything) {
249            result.add("/");
250            return result;
251        }
252        if (scope.isIncludeSite()) {
253            result.add(siteParam);
254        }
255        if (scope.isIncludeSubSite()) {
256            if (subSiteParam == null) {
257                result.add(siteParam);
258            } else {
259                result.add(CmsStringUtil.joinPaths(siteParam, subSiteParam));
260            }
261        }
262        if (scope.isIncludeShared()) {
263            String sharedFolder = OpenCms.getSiteManager().getSharedFolder();
264            if (sharedFolder != null) {
265                result.add(sharedFolder);
266            }
267            result.add(CmsSiteManagerImpl.PATH_SYSTEM_SHARED_FOLDER);
268        }
269        // the site and sub-site roots arrive without a trailing slash, the shared folders with one;
270        // normalize all of them, since a search root is matched against the parent-folders index
271        // field, which stores every ancestor path with its trailing slash
272        return result.stream().filter(root -> root != null).map(CmsFileUtil::addTrailingSeparator).collect(
273            Collectors.toList());
274    }
275
276    /**
277     * Returns a string that represents a valid Solr query range.
278     *
279     * @param from Lower bound of the query range.
280     * @param to Upper bound of the query range.
281     * @return String that represents a Solr query range.
282     */
283    public static String getSolrRangeString(String from, String to) {
284
285        // If a parameter is not initialized, use the asterisk '*' operator
286        if (CmsStringUtil.isEmptyOrWhitespaceOnly(from)) {
287            from = "*";
288        }
289
290        if (CmsStringUtil.isEmptyOrWhitespaceOnly(to)) {
291            to = "*";
292        }
293
294        return String.format("[%s TO %s]", from, to);
295    }
296
297    /**
298     * Returns a formatter that can be use by the current thread if needed to
299     * convert Date objects to the Internal representation.
300     *
301     * @param d The input date to parse
302     * @return The parsed {@link java.util.Date}
303     * @throws java.text.ParseException If the input can't be parsed
304     */
305    public static Date parseDate(String d) throws ParseException {
306
307        return parseDate(d, DEFAULT_DATE_FORMATS);
308    }
309
310    public static Date parseDate(String d, Collection<String> fmts) throws ParseException {
311
312        // 2007-04-26T08:05:04Z
313        if (d.endsWith("Z") && (d.length() > 20)) {
314            return DATEFORMAT_ISO_8601.parse(d);
315        }
316        return parseDate(d, fmts, null);
317    }
318
319    /**
320     * Slightly modified from org.apache.commons.httpclient.util.DateUtil.parseDate
321     * <p>
322     * Parses the date value using the given date formats.
323     *
324     * @param dateValue   the date value to parse
325     * @param dateFormats the date formats to use
326     * @param startDate   During parsing, two digit years will be placed in the range
327     *                    <code>startDate</code> to <code>startDate + 100 years</code>. This value may
328     *                    be <code>null</code>. When <code>null</code> is given as a parameter, year
329     *                    <code>2000</code> will be used.
330     * @return the parsed date
331     * @throws ParseException if none of the dataFormats could parse the dateValue
332     */
333    public static Date parseDate(String dateValue, Collection<String> dateFormats, Date startDate)
334    throws ParseException {
335
336        if (dateValue == null) {
337            throw new IllegalArgumentException("dateValue is null");
338        }
339        if (dateFormats == null) {
340            dateFormats = DEFAULT_HTTP_CLIENT_PATTERNS;
341        }
342        if (startDate == null) {
343            startDate = DEFAULT_TWO_DIGIT_YEAR_START;
344        }
345        // trim single quotes around date if present
346        // see issue #5279
347        if ((dateValue.length() > 1) && dateValue.startsWith("'") && dateValue.endsWith("'")) {
348            dateValue = dateValue.substring(1, dateValue.length() - 1);
349        }
350
351        SimpleDateFormat dateParser = null;
352        Iterator formatIter = dateFormats.iterator();
353
354        while (formatIter.hasNext()) {
355            String format = (String)formatIter.next();
356            if (dateParser == null) {
357                dateParser = new SimpleDateFormat(format, Locale.ENGLISH);
358                dateParser.setTimeZone(TIMEZONE_GMT);
359                dateParser.set2DigitYearStart(startDate);
360            } else {
361                dateParser.applyPattern(format);
362            }
363            try {
364                return dateParser.parse(dateValue);
365            } catch (ParseException pe) {
366                // ignore this exception, we will try the next format
367            }
368        }
369
370        // we were unable to parse the date
371        throw new ParseException("Unable to parse the date " + dateValue, 0);
372    }
373
374    /**
375     * Strips of HTML of the value to map, if necessary (depending on the property name).
376     * @param propertyName name of the property.
377     * @param value the properties value (possibly with HTML)
378     * @return the value with HTML stripped of, or the original value, if stripping of the HTML fails.
379     */
380    public static String stripHtmlFromPropertyIfNecessary(String propertyName, String value) {
381
382        if (propertyName.equals(CmsPropertyDefinition.PROPERTY_DESCRIPTION_HTML)) {
383            try {
384                return CmsHtmlExtractor.extractText(value, CmsEncoder.ENCODING_UTF_8);
385            } catch (ParserException | UnsupportedEncodingException e) {
386                LOG.warn("Could not strip HTML from property value. Returning the original value.", e);
387            }
388        }
389        return value;
390
391    }
392
393    /**
394     * Take a string and make it an iterable ContentStream
395     */
396    public static Collection<ContentStream> toContentStreams(final String str, final String contentType) {
397
398        if (str == null) {
399            return null;
400        }
401
402        ArrayList<ContentStream> streams = new ArrayList<>(1);
403        ContentStreamBase ccc = new ContentStreamBase.StringStream(str);
404        ccc.setContentType(contentType);
405        streams.add(ccc);
406        return streams;
407    }
408
409    /**
410     * @param d SolrInputDocument to convert
411     * @return a SolrDocument with the same fields and values as the SolrInputDocument
412     * @deprecated This method will be removed in Solr 6.0
413     */
414    @Deprecated
415    public static SolrDocument toSolrDocument(SolrInputDocument d) {
416
417        SolrDocument doc = new SolrDocument();
418        for (SolrInputField field : d) {
419            doc.setField(field.getName(), field.getValue());
420        }
421        if (d.getChildDocuments() != null) {
422            for (SolrInputDocument in : d.getChildDocuments()) {
423                doc.addChildDocument(toSolrDocument(in));
424            }
425
426        }
427        return doc;
428    }
429
430    /**
431     * @param d SolrDocument to convert
432     * @return a SolrInputDocument with the same fields and values as the
433     *   SolrDocument.
434     * @deprecated This method will be removed in Solr 6.0
435     */
436    @Deprecated
437    public static SolrInputDocument toSolrInputDocument(SolrDocument d) {
438
439        SolrInputDocument doc = new SolrInputDocument();
440        d.getFieldNames().forEach(name -> doc.addField(name, d.getFieldValue(name)));
441        return doc;
442    }
443
444}