001/*
002 * This library is part of OpenCms -
003 * the Open Source Content Management System
004 *
005 * Copyright (c) Alkacon Software GmbH & Co. KG (https://www.alkacon.com)
006 *
007 * This library is free software; you can redistribute it and/or
008 * modify it under the terms of the GNU Lesser General Public
009 * License as published by the Free Software Foundation; either
010 * version 2.1 of the License, or (at your option) any later version.
011 *
012 * This library is distributed in the hope that it will be useful,
013 * but WITHOUT ANY WARRANTY; without even the implied warranty of
014 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
015 * Lesser General Public License for more details.
016 *
017 * For further information about Alkacon Software GmbH & Co. KG, please see the
018 * company website: https://www.alkacon.com
019 *
020 * For further information about OpenCms, please see the
021 * project website: https://www.opencms.org
022 *
023 * You should have received a copy of the GNU Lesser General Public
024 * License along with this library; if not, write to the Free Software
025 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
026 */
027
028package org.opencms.search.extractors;
029
030import org.opencms.util.CmsFileUtil;
031import org.opencms.util.CmsStringUtil;
032
033import java.io.ByteArrayInputStream;
034import java.io.InputStream;
035import java.io.StringWriter;
036import java.util.LinkedHashMap;
037import java.util.Map;
038
039import org.apache.commons.lang3.StringUtils;
040import org.apache.tika.metadata.DublinCore;
041import org.apache.tika.metadata.Metadata;
042import org.apache.tika.metadata.Office;
043import org.apache.tika.metadata.OfficeOpenXMLCore;
044import org.apache.tika.metadata.OfficeOpenXMLExtended;
045import org.apache.tika.metadata.PDF;
046import org.apache.tika.parser.ParseContext;
047import org.apache.tika.parser.Parser;
048import org.apache.tika.sax.BodyContentHandler;
049
050/**
051 * Base utility class that allows extraction of the indexable "plain" text from a given document format.<p>
052 *
053 * @since 6.0.0
054 */
055public abstract class A_CmsTextExtractor implements I_CmsTextExtractor {
056
057    /**
058     * @see org.opencms.search.extractors.I_CmsTextExtractor#extractText(byte[])
059     */
060    public I_CmsExtractionResult extractText(byte[] content) throws Exception {
061
062        // call stream based method of extraction without encoding
063        return extractText(new ByteArrayInputStream(content));
064    }
065
066    /**
067     * @see org.opencms.search.extractors.I_CmsTextExtractor#extractText(byte[], java.lang.String)
068     */
069    public I_CmsExtractionResult extractText(byte[] content, String encoding) throws Exception {
070
071        // call stream based method of extraction with encoding
072        return extractText(new ByteArrayInputStream(content), encoding);
073    }
074
075    /**
076     * @see org.opencms.search.extractors.I_CmsTextExtractor#extractText(java.io.InputStream)
077     */
078    public I_CmsExtractionResult extractText(InputStream in) throws Exception {
079
080        // encoding is null
081        // (using cast to disambiguate method)
082        return extractText(in, (String)null);
083    }
084
085    /**
086     * @see org.opencms.search.extractors.I_CmsTextExtractor#extractText(java.io.InputStream, java.lang.String)
087     */
088    public I_CmsExtractionResult extractText(InputStream in, String encoding) throws Exception {
089
090        // read the byte content
091        byte[] text = CmsFileUtil.readFully(in);
092        // call byte array based method of extraction
093        return extractText(text, encoding);
094    }
095
096    /**
097     * Combines a meta information item extracted from the document with the main content buffer and
098     * also stores the individual information as item in the Map of content items.<p>
099     *
100     * @param itemValue the value of the item to store
101     * @param itemKey the key in the Map of content items
102     * @param content a buffer where to append the content item
103     * @param contentItems the Map of individual content items
104     */
105    protected void combineContentItem(
106        String itemValue,
107        String itemKey,
108        StringBuffer content,
109        Map<String, String> contentItems) {
110
111        if (CmsStringUtil.isNotEmpty(itemValue)) {
112            contentItems.put(itemKey, itemValue);
113            content.append('\n');
114            content.append(itemValue);
115        }
116    }
117
118    /**
119     * Parses the given input stream with the provided parser and returns the result as a map of content items.<p>
120     *
121     * @param in the input stream for the content to parse
122     * @param parser the parser to use
123     *
124     * @return the result of the parsing as a map of content items
125     *
126     * @throws Exception in case something goes wrong
127     */
128    @SuppressWarnings("deprecation")
129    protected CmsExtractionResult extractText(InputStream in, Parser parser) throws Exception {
130
131        LinkedHashMap<String, String> contentItems = new LinkedHashMap<String, String>();
132
133        StringWriter writer = new StringWriter();
134        BodyContentHandler handler = new BodyContentHandler(writer);
135        Metadata meta = new Metadata();
136        ParseContext context = new ParseContext();
137
138        parser.parse(in, handler, meta, context);
139        in.close();
140        String result = writer.toString();
141
142        // add the main document text
143        StringBuffer content = new StringBuffer(result);
144        if (CmsStringUtil.isNotEmpty(result)) {
145            contentItems.put(I_CmsExtractionResult.ITEM_RAW, result);
146        }
147
148        // appends all known document meta data as content items
149        combineContentItem(meta.get(DublinCore.TITLE), I_CmsExtractionResult.ITEM_TITLE, content, contentItems);
150        String keywords = meta.get(PDF.DOC_INFO_KEY_WORDS);
151        if (StringUtils.isBlank(keywords)) {
152            keywords = meta.get(Office.KEYWORDS);
153        }
154        combineContentItem(keywords, I_CmsExtractionResult.ITEM_KEYWORDS, content, contentItems);
155        String subject = null;
156        if (StringUtils.isBlank(subject)) {
157            subject = meta.get(PDF.DOC_INFO_SUBJECT);
158        }
159        if (StringUtils.isBlank(subject)) {
160            subject = meta.get(I_CmsExtractionResult.ITEM_SUBJECT);
161        }
162        if (StringUtils.isBlank(subject)) {
163            subject = meta.get(DublinCore.SUBJECT);
164        }
165        combineContentItem(subject, I_CmsExtractionResult.ITEM_SUBJECT, content, contentItems);
166        if (meta.get(Office.AUTHOR) != null) {
167            combineContentItem(meta.get(Office.AUTHOR), I_CmsExtractionResult.ITEM_AUTHOR, content, contentItems);
168        } else {
169            combineContentItem(meta.get(DublinCore.CREATOR), I_CmsExtractionResult.ITEM_AUTHOR, content, contentItems);
170        }
171        String creator = meta.get("xmp:CreatorTool");
172        if (StringUtils.isBlank(creator)) {
173            creator = meta.get(DublinCore.CREATOR);
174        }
175        if (StringUtils.isBlank(creator)) {
176            creator = meta.get(I_CmsExtractionResult.ITEM_CREATOR);
177        }
178        combineContentItem(creator, I_CmsExtractionResult.ITEM_CREATOR, content, contentItems);
179        //
180        combineContentItem(
181            meta.get(OfficeOpenXMLCore.CATEGORY),
182            I_CmsExtractionResult.ITEM_CATEGORY,
183            content,
184            contentItems);
185        //
186        if (meta.get(OfficeOpenXMLExtended.COMMENTS) != null) {
187            combineContentItem(
188                meta.get(OfficeOpenXMLExtended.COMMENTS),
189                I_CmsExtractionResult.ITEM_COMMENTS,
190                content,
191                contentItems);
192        } else {
193            combineContentItem(
194                meta.get(DublinCore.DESCRIPTION),
195                I_CmsExtractionResult.ITEM_COMMENTS,
196                content,
197                contentItems);
198
199        }
200        String company = meta.get(OfficeOpenXMLExtended.COMPANY);
201        combineContentItem(company, I_CmsExtractionResult.ITEM_COMPANY, content, contentItems);
202        //
203        combineContentItem(
204            meta.get(OfficeOpenXMLExtended.MANAGER),
205            I_CmsExtractionResult.ITEM_MANAGER,
206            content,
207            contentItems);
208
209        String producer = meta.get(PDF.DOC_INFO_PRODUCER);
210        if (StringUtils.isBlank(producer)) {
211            producer = meta.get(I_CmsExtractionResult.ITEM_PRODUCER);
212        }
213        combineContentItem(producer, I_CmsExtractionResult.ITEM_PRODUCER, content, contentItems);
214
215        // return the final result
216        return new CmsExtractionResult(content.toString(), contentItems);
217    }
218
219    /**
220     * Removes "unwanted" control chars from the given content.<p>
221     *
222     * @param content the content to remove the unwanted control chars from
223     *
224     * @return the content with the unwanted control chars removed
225     */
226    protected String removeControlChars(String content) {
227
228        if (CmsStringUtil.isEmptyOrWhitespaceOnly(content)) {
229            // to avoid later null pointer exceptions an empty String is returned
230            return "";
231        }
232
233        char[] chars = content.toCharArray();
234        StringBuffer result = new StringBuffer(chars.length);
235        boolean wasUnwanted = false;
236        for (int i = 0; i < chars.length; i++) {
237            char ch = chars[i];
238
239            int type = Character.getType(ch);
240            switch (type) {
241
242                // punctuation
243                case Character.CURRENCY_SYMBOL:
244                case Character.CONNECTOR_PUNCTUATION:
245                case Character.FINAL_QUOTE_PUNCTUATION:
246                case Character.INITIAL_QUOTE_PUNCTUATION:
247                case Character.DASH_PUNCTUATION:
248                case Character.START_PUNCTUATION:
249                case Character.END_PUNCTUATION:
250                case Character.OTHER_PUNCTUATION:
251                    // letters
252                case Character.OTHER_LETTER:
253                case Character.MODIFIER_LETTER:
254                case Character.UPPERCASE_LETTER:
255                case Character.TITLECASE_LETTER:
256                case Character.LOWERCASE_LETTER:
257                    // digits
258                case Character.DECIMAL_DIGIT_NUMBER:
259                    // spaces
260                case Character.SPACE_SEPARATOR:
261                    result.append(ch);
262                    wasUnwanted = false;
263                    break;
264
265                // line separators
266                case Character.LINE_SEPARATOR:
267                    result.append('\n');
268                    wasUnwanted = true;
269                    break;
270
271                // symbols
272                case Character.MATH_SYMBOL:
273                case Character.OTHER_SYMBOL:
274                    // other stuff:
275                case Character.CONTROL:
276                case Character.COMBINING_SPACING_MARK:
277                case Character.ENCLOSING_MARK:
278                case Character.FORMAT:
279                case Character.LETTER_NUMBER:
280                case Character.MODIFIER_SYMBOL:
281                case Character.NON_SPACING_MARK:
282                case Character.PARAGRAPH_SEPARATOR:
283                case Character.PRIVATE_USE:
284                case Character.SURROGATE:
285                case Character.UNASSIGNED:
286                case Character.OTHER_NUMBER:
287                default:
288                    if (!wasUnwanted) {
289                        result.append('\n');
290                        wasUnwanted = true;
291                    }
292            }
293        }
294
295        return result.toString();
296    }
297}