001/*
002 * This library is part of OpenCms -
003 * the Open Source Content Management System
004 *
005 * Copyright (c) Alkacon Software GmbH & Co. KG (https://www.alkacon.com)
006 *
007 * This library is free software; you can redistribute it and/or
008 * modify it under the terms of the GNU Lesser General Public
009 * License as published by the Free Software Foundation; either
010 * version 2.1 of the License, or (at your option) any later version.
011 *
012 * This library is distributed in the hope that it will be useful,
013 * but WITHOUT ANY WARRANTY; without even the implied warranty of
014 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
015 * Lesser General Public License for more details.
016 *
017 * For further information about Alkacon Software GmbH & Co. KG, please see the
018 * company website: https://www.alkacon.com
019 *
020 * For further information about OpenCms, please see the
021 * project website: https://www.opencms.org
022 *
023 * You should have received a copy of the GNU Lesser General Public
024 * License along with this library; if not, write to the Free Software
025 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
026 */
027
028package org.opencms.search.solr;
029
030import org.opencms.ade.configuration.CmsADEConfigData;
031import org.opencms.ade.configuration.CmsFormatterUtils;
032import org.opencms.file.CmsFile;
033import org.opencms.file.CmsObject;
034import org.opencms.file.CmsResource;
035import org.opencms.main.CmsException;
036import org.opencms.main.CmsLog;
037import org.opencms.main.OpenCms;
038import org.opencms.search.CmsIndexException;
039import org.opencms.search.I_CmsSearchIndex;
040import org.opencms.search.documents.Messages;
041import org.opencms.search.extractors.CmsExtractionResult;
042import org.opencms.search.extractors.I_CmsExtractionResult;
043import org.opencms.util.CmsStringUtil;
044import org.opencms.xml.containerpage.CmsContainerBean;
045import org.opencms.xml.containerpage.CmsContainerElementBean;
046import org.opencms.xml.containerpage.CmsContainerPageBean;
047import org.opencms.xml.containerpage.CmsFormatterConfiguration;
048import org.opencms.xml.containerpage.CmsXmlContainerPage;
049import org.opencms.xml.containerpage.CmsXmlContainerPageFactory;
050import org.opencms.xml.containerpage.I_CmsFormatterBean;
051
052import java.util.ArrayList;
053import java.util.Collection;
054import java.util.Collections;
055import java.util.HashMap;
056import java.util.LinkedHashMap;
057import java.util.List;
058import java.util.Locale;
059import java.util.Map;
060import java.util.Set;
061
062import org.apache.commons.logging.Log;
063
064/**
065 * Lucene document factory class to extract index data from a resource
066 * of type <code>CmsResourceTypeContainerPage</code>.<p>
067 *
068 * @since 8.5.0
069 */
070public class CmsSolrDocumentContainerPage extends CmsSolrDocumentXmlContent {
071
072    /** The log object for this class. */
073    private static final Log LOG = CmsLog.getLog(CmsSolrDocumentContainerPage.class);
074
075    /** The solr document type name for xml-contents. */
076    public static final String TYPE_CONTAINERPAGE_SOLR = "containerpage-solr";
077
078    /**
079     * Sitemap attribute to list (comma-separated) names of containers
080     * whose elements content should not be extracted for the content of the page.
081     * NOTE: For nested containers, we match against the names without prefix.
082     */
083    private static final String SITEMAP_ATTRIBUTE_EXCLUDE_CONTAINER_NAMES = "search.containers.exclude.names";
084
085    /**
086     * Sitemap attribute to list (comma-separated) types of containers
087     * whose elements content should not be extracted for the content of the page.
088     */
089    private static final String SITEMAP_ATTRIBUTE_EXCLUDE_CONTAINER_TYPES = "search.containers.exclude.types";
090
091    /**
092     * Creates a new instance of this lucene document factory.<p>
093     *
094     * @param name name of the document type
095     */
096    public CmsSolrDocumentContainerPage(String name) {
097
098        super(name);
099    }
100
101    /**
102     * Returns the raw text content of a VFS resource of type <code>CmsResourceTypeContainerPage</code>.<p>
103     *
104     * @see org.opencms.search.documents.I_CmsSearchExtractor#extractContent(CmsObject, CmsResource, I_CmsSearchIndex)
105     */
106    @Override
107    public I_CmsExtractionResult extractContent(CmsObject cms, CmsResource resource, I_CmsSearchIndex index)
108    throws CmsException {
109
110        return extractContent(cms, resource, index, null);
111    }
112
113    /**
114     * Extracts the content of a given index resource according to the resource file type and the
115     * configuration of the given index.<p>
116     *
117     * @param cms the cms object
118     * @param resource the resource to extract the content from
119     * @param index the index to extract the content for
120     * @param forceLocale if set, only the content values for the given locale will be extracted
121     *
122     * @return the extracted content of the resource
123     *
124     * @throws CmsException if something goes wrong
125     */
126    public I_CmsExtractionResult extractContent(
127        CmsObject cms,
128        CmsResource resource,
129        I_CmsSearchIndex index,
130        Locale forceLocale)
131    throws CmsException {
132
133        logContentExtraction(resource, index);
134        I_CmsExtractionResult ex = null;
135        try {
136            CmsFile file = readFile(cms, resource);
137            CmsXmlContainerPage containerPage = CmsXmlContainerPageFactory.unmarshal(cms, file);
138
139            List<I_CmsExtractionResult> all = new ArrayList<I_CmsExtractionResult>();
140            CmsContainerPageBean containerBean = containerPage.getContainerPage(cms);
141            if (containerBean != null) {
142                CmsADEConfigData adeConfig = OpenCms.getADEManager().lookupConfigurationWithCache(
143                    cms,
144                    file.getRootPath());
145
146                Collection<String> excludeContainerNames = CmsStringUtil.splitAsSet(
147                    adeConfig.getAttribute(SITEMAP_ATTRIBUTE_EXCLUDE_CONTAINER_NAMES, ""),
148                    ",",
149                    true);
150                Collection<String> excludeContainerTypes = CmsStringUtil.splitAsSet(
151                    adeConfig.getAttribute(SITEMAP_ATTRIBUTE_EXCLUDE_CONTAINER_TYPES, ""),
152                    ",",
153                    true);
154
155                for (Map.Entry<String, CmsContainerBean> entry : containerBean.getContainers().entrySet()) {
156                    String containerName = entry.getKey();
157                    if (includeContainerForSearch(
158                        entry.getValue(),
159                        containerBean,
160                        excludeContainerNames,
161                        excludeContainerTypes)) {
162                        for (CmsContainerElementBean element : entry.getValue().getElements()) {
163                            // check all elements in this container
164                            // get the formatter configuration for this element
165                            try {
166                                element.initResource(cms);
167                                CmsResource elementResource = element.getResource();
168                                if (!(cms.readProject(index.getProject()).isOnlineProject()
169                                    && elementResource.isExpired(System.currentTimeMillis()))) {
170                                    CmsFormatterConfiguration formatters = adeConfig.getFormatters(
171                                        cms,
172                                        element.getResource());
173                                    boolean shouldExtractElement = false;
174                                    if ((formatters != null)
175                                        && (element.getFormatterId() != null)
176                                        && (formatters.isSearchContent(element.getFormatterId())
177                                            || adeConfig.isSearchContentFormatter(element.getFormatterId()))) {
178                                        // the content of this element must be included for the container page
179                                        shouldExtractElement = true;
180                                    } else if (formatters != null) {
181                                        String key = CmsFormatterUtils.getFormatterKey(containerName, element);
182                                        I_CmsFormatterBean formatter = adeConfig.findFormatter(key);
183                                        if (formatter != null) {
184                                            shouldExtractElement = formatter.isSearchContent();
185                                        }
186                                    }
187                                    if (LOG.isDebugEnabled()) {
188                                        LOG.debug(
189                                            "Should extract element "
190                                                + element.getResource().getRootPath()
191                                                + ": "
192                                                + shouldExtractElement);
193                                    }
194                                    if (shouldExtractElement) {
195                                        all.add(
196                                            CmsSolrDocumentXmlContent.extractXmlContent(
197                                                cms,
198                                                elementResource,
199                                                index,
200                                                forceLocale));
201                                    }
202
203                                }
204                            } catch (Exception e) {
205                                LOG.debug(
206                                    Messages.get().getBundle().key(
207                                        Messages.LOG_SKIPPING_CONTAINERPAGE_ELEMENT_WITH_UNREADABLE_RESOURCE_2,
208                                        file.getRootPath(),
209                                        element.getId()),
210                                    e);
211                            }
212                        }
213                    } else if (LOG.isDebugEnabled()) {
214                        LOG.debug(
215                            "Skip content extraction for container "
216                                + containerName
217                                + " in  page "
218                                + file.getRootPath()
219                                + " because of type or name excludes configured via sitemap attribute "
220                                + SITEMAP_ATTRIBUTE_EXCLUDE_CONTAINER_NAMES
221                                + " or "
222                                + SITEMAP_ATTRIBUTE_EXCLUDE_CONTAINER_TYPES
223                                + ".");
224                    }
225                }
226            }
227            // we have to overwrite the resource and content locales with the one from this container page
228            // TODO: Is this really the wanted behavior? It seems to be done like this before.
229            Map<String, String> fieldMappings = new HashMap<String, String>(1);
230            // Add to each container page the contents in all available locales,
231            // in case one containerpage is used in multiple languages.
232            List<Locale> localesAvailable = OpenCms.getLocaleManager().getAvailableLocales(cms, resource);
233            Map<Locale, LinkedHashMap<String, String>> multilingualValues = new HashMap<Locale, LinkedHashMap<String, String>>(
234                localesAvailable.size());
235            for (Locale localeAvailable : localesAvailable) {
236                multilingualValues.put(localeAvailable, new LinkedHashMap<String, String>());
237            }
238            Locale locale = forceLocale != null
239            ? forceLocale
240            : index.getLocaleForResource(cms, resource, containerPage.getLocales());
241            ex = new CmsExtractionResult(locale, multilingualValues, fieldMappings);
242            ex = ex.merge(all);
243            return ex;
244        } catch (Exception e) {
245            throw new CmsIndexException(
246                Messages.get().container(Messages.ERR_TEXT_EXTRACTION_1, resource.getRootPath()),
247                e);
248        }
249    }
250
251    /**
252     * @see org.opencms.search.documents.I_CmsDocumentFactory#isLocaleDependend()
253     */
254    @Override
255    public boolean isLocaleDependend() {
256
257        return true;
258    }
259
260    /**
261     * Checks if the elements in the provided container should be added to the extracted content of the page.
262     * They are only added if they should be added for the container and all its parent containers.
263     *
264     * @param container the container to check
265     * @param containerPageBean the container page the container is on
266     * @param excludeContainerNames set of container names to skip content extraction for
267     * @param excludeContainerTypes set of container types to skip content extraction for
268     * @return flag, indicating if the container's elements content should be extracted for the page content.
269     */
270    private boolean includeContainerForSearch(
271        CmsContainerBean container,
272        CmsContainerPageBean containerPageBean,
273        Collection<String> excludeContainerNames,
274        Collection<String> excludeContainerTypes) {
275
276        if (container.isNestedContainer()) {
277            // The parent of a nested container is the element (identified by its element instance id)
278            // whose formatter renders this container. Find the container that holds that parent element.
279            String parentInstanceId = container.getParentInstanceId();
280            for (CmsContainerBean c : containerPageBean.getContainers().values()) {
281                boolean containsParentElement = c.getElements().stream().anyMatch(
282                    element -> parentInstanceId.equals(element.getInstanceId()));
283                if (containsParentElement) {
284                    if (!includeContainerForSearch(
285                        c,
286                        containerPageBean,
287                        excludeContainerNames,
288                        excludeContainerTypes)) {
289                        // the elements of the parent container are not included, so this container can't be included as well
290                        return false;
291                    }
292                    // the parent container's elements are included, so check the container itself
293                    break;
294                }
295            }
296        }
297        Set<String> containerTypes = CmsStringUtil.splitAsSet(container.getType(), ",", true);
298        return !excludeContainerNames.contains(container.getSimpleName())
299            && Collections.disjoint(excludeContainerTypes, containerTypes);
300    }
301
302}