Source code

001/*
002 * File   : $Source$
003 * Date   : $Date$
004 * Version: $Revision$
005 *
006 * This library is part of OpenCms -
007 * the Open Source Content Management System
008 *
009 * Copyright (C) 2002 - 2008 Alkacon Software (https://www.alkacon.com)
010 *
011 * This library is free software; you can redistribute it and/or
012 * modify it under the terms of the GNU Lesser General Public
013 * License as published by the Free Software Foundation; either
014 * version 2.1 of the License, or (at your option) any later version.
015 *
016 * This library is distributed in the hope that it will be useful,
017 * but WITHOUT ANY WARRANTY; without even the implied warranty of
018 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
019 * Lesser General Public License for more details.
020 *
021 * For further information about Alkacon Software, please see the
022 * company website: https://www.alkacon.com
023 *
024 * For further information about OpenCms, please see the
025 * project website: https://www.opencms.org
026 *
027 * You should have received a copy of the GNU Lesser General Public
028 * License along with this library; if not, write to the Free Software
029 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
030 */
031
032package org.opencms.search.solr;
033
034import org.opencms.file.CmsObject;
035import org.opencms.file.CmsPropertyDefinition;
036import org.opencms.main.OpenCms;
037import org.opencms.search.fields.CmsSearchField;
038import org.opencms.util.CmsFileUtil;
039import org.opencms.util.CmsPair;
040import org.opencms.util.CmsRequestUtil;
041import org.opencms.util.CmsStringUtil;
042
043import java.util.ArrayList;
044import java.util.Arrays;
045import java.util.Collections;
046import java.util.Date;
047import java.util.HashMap;
048import java.util.List;
049import java.util.Locale;
050import java.util.Map;
051import java.util.stream.Collectors;
052
053import org.apache.solr.client.solrj.request.SolrQuery;
054import org.apache.solr.common.params.CommonParams;
055
056/**
057 * A Solr search query.<p>
058 */
059public class CmsSolrQuery extends SolrQuery {
060
061    /** A constant to add the score field to the result documents. */
062    public static final String ALL_RETURN_FIELDS = "*,score";
063
064    /** The default facet date gap. */
065    public static final String DEFAULT_FACET_DATE_GAP = "+1DAY";
066
067    /** The default query. */
068    public static final String DEFAULT_QUERY = "*:*";
069
070    /** The query type. */
071    public static final String DEFAULT_QUERY_TYPE = "edismax";
072
073    /** The default search result count. */
074    public static final Integer DEFAULT_ROWS = Integer.valueOf(10);
075
076    /** A constant to add the score field to the result documents. */
077    public static final String MINIMUM_FIELDS = CmsSearchField.FIELD_PATH
078        + ","
079        + CmsSearchField.FIELD_TYPE
080        + ","
081        + CmsSearchField.FIELD_SOLR_ID
082        + ","
083        + CmsSearchField.FIELD_ID;
084
085    /** A constant to add the score field to the result documents. */
086    public static final String STRUCTURE_FIELDS = CmsSearchField.FIELD_PATH
087        + ","
088        + CmsSearchField.FIELD_TYPE
089        + ","
090        + CmsSearchField.FIELD_ID
091        + ","
092        + CmsSearchField.FIELD_CATEGORY
093        + ","
094        + CmsSearchField.FIELD_DATE_CONTENT
095        + ","
096        + CmsSearchField.FIELD_DATE_CREATED
097        + ","
098        + CmsSearchField.FIELD_DATE_EXPIRED
099        + ","
100        + CmsSearchField.FIELD_DATE_LASTMODIFIED
101        + ","
102        + CmsSearchField.FIELD_DATE_RELEASED
103        + ","
104        + CmsSearchField.FIELD_SUFFIX
105        + ","
106        + CmsSearchField.FIELD_DEPENDENCY_TYPE
107        + ","
108        + CmsSearchField.FIELD_DESCRIPTION
109        + ","
110        + CmsPropertyDefinition.PROPERTY_TITLE
111        + CmsSearchField.FIELD_DYNAMIC_PROPERTIES
112        + ","
113        + CmsSearchField.FIELD_RESOURCE_LOCALES
114        + ","
115        + CmsSearchField.FIELD_CONTENT_LOCALES
116        + ","
117        + CmsSearchField.FIELD_SCORE
118        + ","
119        + CmsSearchField.FIELD_PARENT_FOLDERS;
120
121    /** The serial version UID. */
122    private static final long serialVersionUID = -2387357736597627703L;
123
124    /** The facet date gap to use for date facets. */
125    private String m_facetDateGap = DEFAULT_FACET_DATE_GAP;
126
127    /** Ignore expiration flag. */
128    private boolean m_ignoreExpiration;
129
130    /** The parameters given by the 'query string'.  */
131    private Map<String, String[]> m_queryParameters = new HashMap<String, String[]>();
132
133    /** The search words. */
134    private String m_text;
135
136    /** The name of the field to search the text in. */
137    private List<String> m_textSearchFields = new ArrayList<String>();
138
139    /**
140     * Default constructor.<p>
141     */
142    public CmsSolrQuery() {
143
144        this(null, null);
145    }
146
147    /**
148     * Public constructor.<p>
149     *
150     * @param cms the current OpenCms context
151     * @param queryParams the Solr query parameters
152     */
153    public CmsSolrQuery(CmsObject cms, Map<String, String[]> queryParams) {
154
155        setQuery(DEFAULT_QUERY);
156        setFields(ALL_RETURN_FIELDS);
157        setRequestHandler(DEFAULT_QUERY_TYPE);
158        setRows(DEFAULT_ROWS);
159
160        // set the values from the request context
161        if (cms != null) {
162            setLocales(Collections.singletonList(cms.getRequestContext().getLocale()));
163            setSearchRoots(Collections.singletonList(cms.getRequestContext().getSiteRoot() + "/"));
164        }
165        if (queryParams != null) {
166            m_queryParameters = queryParams;
167        }
168        ensureParameters();
169        ensureReturnFields();
170        ensureExpiration();
171    }
172
173    /**
174     * Returns the resource type if only one is set as filter query.<p>
175     *
176     * @param fqs the field queries to check
177     *
178     * @return the type or <code>null</code>
179     */
180    public static String getResourceType(String[] fqs) {
181
182        String ret = null;
183        int count = 0;
184        if (fqs != null) {
185            for (String fq : fqs) {
186                if (fq.startsWith(CmsSearchField.FIELD_TYPE + ":")) {
187                    String val = fq.substring((CmsSearchField.FIELD_TYPE + ":").length());
188                    val = val.replaceAll("\"", "");
189                    if (OpenCms.getResourceManager().hasResourceType(val)) {
190                        count++;
191                        ret = val;
192                    }
193                }
194            }
195        }
196        return (count == 1) ? ret : null;
197    }
198
199    /**
200     * Creates and adds a filter query.<p>
201     *
202     * @param fieldName the field name to create a filter query on
203     * @param vals the values that should match for the given field
204     * @param all <code>true</code> to combine the given values with 'AND', <code>false</code> for 'OR'
205     * @param useQuotes <code>true</code> to surround the given values with double quotes, <code>false</code> otherwise
206     */
207    public void addFilterQuery(String fieldName, List<String> vals, boolean all, boolean useQuotes) {
208
209        if (getFilterQueries() != null) {
210            for (String fq : getFilterQueries()) {
211                if (fq.startsWith(fieldName + ":")) {
212                    removeFilterQuery(fq);
213                }
214            }
215        }
216        addFilterQuery(createFilterQuery(fieldName, vals, all, useQuotes));
217    }
218
219    /**
220     * Adds the given fields/orders to the existing sort fields.<p>
221     *
222     * @param sortFields the sortFields to set
223     */
224    public void addSortFieldOrders(Map<String, ORDER> sortFields) {
225
226        if ((sortFields != null) && !sortFields.isEmpty()) {
227            // add the sort fields to the query
228            for (Map.Entry<String, ORDER> entry : sortFields.entrySet()) {
229                addSort(entry.getKey(), entry.getValue());
230            }
231        }
232    }
233
234    /**
235     * @see java.lang.Object#clone()
236     */
237    @Override
238    public CmsSolrQuery clone() {
239
240        CmsSolrQuery sq = new CmsSolrQuery(null, CmsRequestUtil.createParameterMap(toString(), true, null));
241        if (m_ignoreExpiration) {
242            sq.removeExpiration();
243        }
244        return sq;
245    }
246
247    /**
248     * Ensures that the initial request parameters will overwrite the member values.<p>
249     *
250     * You can initialize the query with an HTTP request parameter then make some method calls
251     * and finally re-ensure that the initial request parameters will overwrite the changes
252     * made in the meanwhile.<p>
253     */
254    public void ensureParameters() {
255
256        // overwrite already set values with values from query String
257        if ((m_queryParameters != null) && !m_queryParameters.isEmpty()) {
258            for (Map.Entry<String, String[]> entry : m_queryParameters.entrySet()) {
259                if (!entry.getKey().equals(CommonParams.FQ)) {
260                    // add or replace all parameters from the query String
261                    setParam(entry.getKey(), entry.getValue());
262                } else {
263                    // special handling for filter queries
264                    replaceFilterQueries(entry.getValue());
265                }
266            }
267        }
268    }
269
270    /**
271     * Removes the expiration flag.
272     */
273    public void removeExpiration() {
274
275        if (getFilterQueries() != null) {
276            for (String fq : getFilterQueries()) {
277                if (fq.startsWith(CmsSearchField.FIELD_DATE_EXPIRED + ":")
278                    || fq.startsWith(CmsSearchField.FIELD_DATE_RELEASED + ":")) {
279                    removeFilterQuery(fq);
280                }
281            }
282        }
283        m_ignoreExpiration = true;
284    }
285
286    /**
287     * Sets the categories only if not set in the query parameters.<p>
288     *
289     * @param categories the categories to set
290     */
291    public void setCategories(List<String> categories) {
292
293        if ((categories != null) && !categories.isEmpty()) {
294            addFilterQuery(CmsSearchField.FIELD_CATEGORY + CmsSearchField.FIELD_DYNAMIC_EXACT, categories, true, true);
295        }
296    }
297
298    /**
299     * Sets the categories only if not set in the query parameters.<p>
300     *
301     * @param categories the categories to set
302     */
303    public void setCategories(String... categories) {
304
305        setCategories(Arrays.asList(categories));
306    }
307
308    /**
309     * Sets date ranges.<p>
310     *
311     * This call will overwrite all existing date ranges for the given keys (name of the date facet field).<p>
312     *
313     * The parameter Map uses as:<p>
314     * <ul>
315     * <li><code>keys: </code>Solr field name {@link org.opencms.search.fields.CmsSearchField} and
316     * <li><code>values: </code> pairs with min date as first and max date as second {@link org.opencms.util.CmsPair}
317     * </ul>
318     * Alternatively you can use Solr standard query syntax like:<p>
319     * <ul>
320     * <li><code>+created:[* TO NOW]</code>
321     * <li><code>+lastmodified:[' + date + ' TO NOW]</code>
322     * </ul>
323     * whereby date is Solr formatted:
324     * {@link org.opencms.search.CmsSearchUtil#getDateAsIso8601(Date)}
325     * <p>
326     *
327     * @param dateRanges the ranges map with field name as key and a CmsPair with min date as first and max date as second
328     */
329    public void setDateRanges(Map<String, CmsPair<Date, Date>> dateRanges) {
330
331        if ((dateRanges != null) && !dateRanges.isEmpty()) {
332            // remove the date ranges
333            for (Map.Entry<String, CmsPair<Date, Date>> entry : dateRanges.entrySet()) {
334                removeFacetField(entry.getKey());
335            }
336            // add the date ranges
337            for (Map.Entry<String, CmsPair<Date, Date>> entry : dateRanges.entrySet()) {
338                addDateRangeFacet(
339                    entry.getKey(),
340                    entry.getValue().getFirst(),
341                    entry.getValue().getSecond(),
342                    m_facetDateGap);
343            }
344        }
345    }
346
347    /**
348     * Sets the facetDateGap.<p>
349     *
350     * @param facetDateGap the facetDateGap to set
351     */
352    public void setFacetDateGap(String facetDateGap) {
353
354        m_facetDateGap = facetDateGap;
355    }
356
357    /**
358     * Sets the Geo filter query if not exists.
359     * @param fieldName the field name storing the coordinates
360     * @param coordinates the coordinates string as a lat,lng pair
361     * @param radius the radius
362     * @param units the units of the search radius
363     */
364    public void setGeoFilterQuery(String fieldName, String coordinates, String radius, String units) {
365
366        String geoFilterQuery = CmsSolrQueryUtil.composeGeoFilterQuery(fieldName, coordinates, radius, units);
367        if (!Arrays.asList(getFilterQueries()).contains(geoFilterQuery)) {
368            addFilterQuery(geoFilterQuery);
369        }
370    }
371
372    /**
373     * Sets the highlightFields.<p>
374     *
375     * @param highlightFields the highlightFields to set
376     */
377    public void setHighlightFields(List<String> highlightFields) {
378
379        setParam("hl.fl", CmsStringUtil.listAsString(highlightFields, ","));
380    }
381
382    /**
383     * Sets the highlightFields.<p>
384     *
385     * @param highlightFields the highlightFields to set
386     */
387    public void setHighlightFields(String... highlightFields) {
388
389        setParam("hl.fl", CmsStringUtil.arrayAsString(highlightFields, ","));
390    }
391
392    /**
393     * Sets the locales only if not set in the query parameters.<p>
394     *
395     * @param locales the locales to set
396     */
397    public void setLocales(List<Locale> locales) {
398
399        m_textSearchFields = new ArrayList<String>();
400        if ((locales == null) || locales.isEmpty()) {
401            m_textSearchFields.add(CmsSearchField.FIELD_TEXT);
402            if (getFilterQueries() != null) {
403                for (String fq : getFilterQueries()) {
404                    if (fq.startsWith(CmsSearchField.FIELD_CONTENT_LOCALES + ":")) {
405                        removeFilterQuery(fq);
406                    }
407                }
408            }
409        } else {
410            List<String> localeStrings = new ArrayList<String>();
411            for (Locale locale : locales) {
412                localeStrings.add(locale.toString());
413                if (!m_textSearchFields.contains("text")
414                    && !OpenCms.getLocaleManager().getAvailableLocales().contains(locale)) {
415                    // if the locale is not configured in the opencms-system.xml
416                    // there will no localized text fields, so take the general one
417                    m_textSearchFields.add("text");
418                } else {
419                    m_textSearchFields.add("text_" + locale);
420                }
421            }
422            addFilterQuery(CmsSearchField.FIELD_CONTENT_LOCALES, localeStrings, false, false);
423        }
424        if (m_text != null) {
425            setText(m_text);
426        }
427    }
428
429    /**
430     * Sets the locales only if not set in the query parameters.<p>
431     *
432     * @param locales the locales to set
433     */
434    public void setLocales(Locale... locales) {
435
436        setLocales(Arrays.asList(locales));
437    }
438
439    /**
440     * Sets the request handler.<p>
441     *
442     * @param qt the request handler
443     *
444     * @return this query
445     */
446    @Override
447    public SolrQuery setRequestHandler(String qt) {
448
449        SolrQuery q = super.setRequestHandler(qt);
450        if (m_text != null) {
451            setText(m_text);
452        }
453        return q;
454    }
455
456    /**
457     * Sets the resource types only if not set in the query parameters.<p>
458     *
459     * @param resourceTypes the resourceTypes to set
460     */
461    public void setResourceTypes(List<String> resourceTypes) {
462
463        if ((resourceTypes != null) && !resourceTypes.isEmpty()) {
464            addFilterQuery(CmsSearchField.FIELD_TYPE, resourceTypes, false, false);
465        }
466    }
467
468    /**
469     * Sets the resource types only if not set in the query parameters.<p>
470     *
471     * @param resourceTypes the resourceTypes to set
472     */
473    public void setResourceTypes(String... resourceTypes) {
474
475        setResourceTypes(Arrays.asList(resourceTypes));
476    }
477
478    /**
479     * Sets the requested return fields, but ensures that at least the 'path' and the 'type', 'id' and 'solr_id'
480     * are part of the fields returned field list.<p>
481     *
482     * @param returnFields the really requested return fields.
483     *
484     * @see CommonParams#FL
485     */
486    public void setReturnFields(String returnFields) {
487
488        ensureReturnFields(new String[] {returnFields});
489    }
490
491    /**
492     * Sets the search roots only if not set as query parameter.<p>
493     *
494     * @param searchRoots the searchRoots to set
495     */
496    public void setSearchRoots(List<String> searchRoots) {
497
498        if (searchRoots != null) {
499            searchRoots = searchRoots.stream().filter(r -> r != null).map(CmsFileUtil::addTrailingSeparator).collect(
500                Collectors.toList());
501            if (!searchRoots.isEmpty()) {
502                addFilterQuery(CmsSearchField.FIELD_PARENT_FOLDERS, searchRoots, false, true);
503            }
504        }
505    }
506
507    /**
508     * Sets the search roots only if not set as query parameter.<p>
509     *
510     * @param searchRoots the searchRoots to set
511     */
512    public void setSearchRoots(String... searchRoots) {
513
514        setSearchRoots(Arrays.asList(searchRoots));
515    }
516
517    /**
518     * Sets the return fields 'fl' to a predefined set that does not contain content specific fields.<p>
519     *
520     * @param structureQuery the <code>true</code> to return only structural fields
521     */
522    public void setStructureQuery(boolean structureQuery) {
523
524        if (structureQuery) {
525            setFields(STRUCTURE_FIELDS);
526        }
527    }
528
529    /**
530     * Sets the text.<p>
531     *
532     * @param text the text to set
533     */
534    public void setText(String text) {
535
536        m_text = text;
537        if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(text)) {
538            setQuery(createTextQuery(text));
539        }
540    }
541
542    /**
543     * Sets the textSearchFields.<p>
544     *
545     * @param textSearchFields the textSearchFields to set
546     */
547    public void setTextSearchFields(List<String> textSearchFields) {
548
549        m_textSearchFields = textSearchFields;
550        if (m_text != null) {
551            setText(m_text);
552        }
553    }
554
555    /**
556     * Sets the textSearchFields.<p>
557     *
558     * @param textSearchFields the textSearchFields to set
559     */
560    public void setTextSearchFields(String... textSearchFields) {
561
562        setTextSearchFields(Arrays.asList(textSearchFields));
563    }
564
565    /**
566     * Creates a filter query on the given field name.<p>
567     *
568     * Creates and adds a filter query.<p>
569     *
570     * @param fieldName the field name to create a filter query on
571     * @param vals the values that should match for the given field
572     * @param all <code>true</code> to combine the given values with 'AND', <code>false</code> for 'OR'
573     * @param useQuotes <code>true</code> to surround the given values with double quotes, <code>false</code> otherwise
574     *
575     * @return a filter query String e.g. <code>fq=fieldname:val1</code>
576     */
577    private String createFilterQuery(String fieldName, List<String> vals, boolean all, boolean useQuotes) {
578
579        String filterQuery = null;
580        if ((vals != null)) {
581            if (vals.size() == 1) {
582                if (useQuotes) {
583                    filterQuery = fieldName + ":" + "\"" + vals.get(0) + "\"";
584                } else {
585                    filterQuery = fieldName + ":" + vals.get(0);
586                }
587            } else if (vals.size() > 1) {
588                filterQuery = fieldName + ":(";
589                for (int j = 0; j < vals.size(); j++) {
590                    String val;
591                    if (useQuotes) {
592                        val = "\"" + vals.get(j) + "\"";
593                    } else {
594                        val = vals.get(j);
595                    }
596                    filterQuery += val;
597                    if (vals.size() > (j + 1)) {
598                        if (all) {
599                            filterQuery += " AND ";
600                        } else {
601                            filterQuery += " OR ";
602                        }
603                    }
604                }
605                filterQuery += ")";
606            }
607        }
608        return filterQuery;
609    }
610
611    /**
612     * Creates a OR combined 'q' parameter.<p>
613     *
614     * @param text the query string.
615     *
616     * @return returns the 'q' parameter
617     */
618    private String createTextQuery(String text) {
619
620        if (m_textSearchFields.isEmpty()) {
621            m_textSearchFields.add(CmsSearchField.FIELD_TEXT);
622        }
623        String q = "{!q.op=OR type=" + getRequestHandler() + " qf=\"";
624        q += String.join(" ", m_textSearchFields);
625        q += "\"}" + text;
626        return q;
627    }
628
629    /**
630     * Ensures that expired and not yet released resources are not returned by default.<p>
631     */
632    private void ensureExpiration() {
633
634        boolean expirationDateSet = false;
635        boolean releaseDateSet = false;
636        if (getFilterQueries() != null) {
637            for (String fq : getFilterQueries()) {
638                if (fq.startsWith(CmsSearchField.FIELD_DATE_EXPIRED + ":")) {
639                    expirationDateSet = true;
640                }
641                if (fq.startsWith(CmsSearchField.FIELD_DATE_RELEASED + ":")) {
642                    releaseDateSet = true;
643                }
644            }
645        }
646        if (!expirationDateSet) {
647            addFilterQuery(CmsSearchField.FIELD_DATE_EXPIRED + ":[NOW TO *]");
648        }
649        if (!releaseDateSet) {
650            addFilterQuery(CmsSearchField.FIELD_DATE_RELEASED + ":[* TO NOW]");
651        }
652    }
653
654    /**
655     * Ensures that at least the 'path' and the 'type', 'id' and 'solr_id' are part of the fields returned field list.<p>
656     *
657     * @see CommonParams#FL
658     */
659    private void ensureReturnFields() {
660
661        ensureReturnFields(getParams(CommonParams.FL));
662    }
663
664    /**
665     * Ensures that at least the 'path' and the 'type', 'id' and 'solr_id' are part of the fields returned field list.<p>
666     *
667     * @param requestedReturnFields the really requested return fields.
668     *
669     * @see CommonParams#FL
670     */
671    private void ensureReturnFields(String[] requestedReturnFields) {
672
673        if ((requestedReturnFields != null) && (requestedReturnFields.length > 0)) {
674            List<String> result = new ArrayList<String>();
675            for (String field : requestedReturnFields) {
676                List<String> list = CmsStringUtil.splitAsList(field, ',');
677                list.forEach(e -> e.trim());
678                if (!list.contains("*")) {
679                    for (String reqField : CmsStringUtil.splitAsList(MINIMUM_FIELDS, ",")) {
680                        if (!list.contains(reqField)) {
681                            list.add(reqField);
682                        }
683                    }
684                }
685                result.addAll(list);
686            }
687            setParam(CommonParams.FL, CmsStringUtil.arrayAsString(result.toArray(new String[0]), ","));
688        }
689    }
690
691    /**
692     * Removes those filter queries that restrict the fields used in the given filter query Strings.<p>
693     *
694     * Searches in the given Strings for a ":", then takes the field name part
695     * and removes the already set filter queries queries that are matching the same field name.<p>
696     *
697     * @param fqs the filter query Strings in the format <code>fq=fieldname:value</code> that should be removed
698     */
699    private void removeFilterQueries(String[] fqs) {
700
701        // iterate over the given filter queries to remove
702        for (String fq : fqs) {
703            int idx = fq.indexOf(':');
704            if (idx != -1) {
705                // get the field name of the fq to remove
706                String fieldName = fq.substring(0, idx);
707                // iterate over the fqs of the already existing fqs from the solr query
708                if (getFilterQueries() != null) {
709                    for (String sfq : getFilterQueries()) {
710                        if (sfq.startsWith(fieldName + ":")) {
711                            // there exists a filter query for exact the same field,  remove it
712                            removeFilterQuery(sfq);
713                        }
714                    }
715                }
716            }
717        }
718    }
719
720    /**
721     * Removes the given filter queries, if already set and then adds the filter queries again.<p>
722     *
723     * @param fqs the filter queries to remove
724     */
725    private void replaceFilterQueries(String[] fqs) {
726
727        removeFilterQueries(fqs);
728        addFilterQuery(fqs);
729    }
730}