/*
 * This library is part of OpenCms -
 * the Open Source Content Management System
 *
 * Copyright (c) Alkacon Software GmbH & Co. KG (http://www.alkacon.com)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * For further information about Alkacon Software, please see the
 * company website: http://www.alkacon.com
 *
 * For further information about OpenCms, please see the
 * project website: http://www.opencms.org
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

package org.opencms.search.solr.spellchecking;

import org.opencms.file.CmsFile;
import org.opencms.file.CmsObject;
import org.opencms.file.CmsProject;
import org.opencms.file.CmsRequestContext;
import org.opencms.file.CmsResource;
import org.opencms.file.CmsResourceFilter;
import org.opencms.main.CmsException;
import org.opencms.main.CmsLog;
import org.opencms.main.OpenCms;
import org.opencms.main.OpenCmsServlet;
import org.opencms.util.CmsStringUtil;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import org.apache.commons.logging.Log;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.common.SolrInputDocument;

/**
 * Helping class for manipulating the Solr spellchecker indices.
 */
public final class CmsSpellcheckDictionaryIndexer {

    /** The log object for this class. */
    private static final Log LOG = CmsLog.getLog(OpenCmsServlet.class);

    /** The default directory that's holding the dictionary files. */
    public static final String DEFAULT_DICTIONARY_DIRECTORY = "/system/workplace/editors/spellcheck";

    /** A regex pattern that applies to the Solr spellcheck directories.
     * Matching string example: "spellchecker_en" */
    public static final String INDEXES_REGEX = "spellchecker_[a-z]{2}";

    /** A regex pattern that applies to custom dictionaries.
     * Matching string example: "custom_dict_en.txt" */
    public static final String CUSTOM_DICTIONARY = "custom_dict_[a-z]{2}.txt";

    /** A regex pattern that applies to the naming of the dictionary files.
     * Matching string example: "dict_en.txt" */
    public static final String DICTIONARY_NAME_REGEX = "dict_[a-z]{2}.txt";

    /** A regex pattern that applies to the naming of zipped dictionary files.
     * Matching string example: "dict_en.zip" */
    public static final String ZIP_NAME_REGEX = "dict_[a-z]{2}.zip";

    /** Maximum amount of entries while parsing the dictionary. This variable is needed
     * in order to prevent OutOfMemoryExceptions while parsing large dictionaries. If you
     * encounter such exceptions you can adjust its value to a smaller number. */
    private static final int MAX_LIST_SIZE = 100000;

    /**
     * FileFilter implementation that returns only directories whose name matches
     * the spellchecker indices regex.
     */
    private static final FileFilter SPELLCHECKING_DIRECTORY_NAME_FILTER = new FileFilter() {

        public boolean accept(File f) {

            return f.isDirectory() && f.getName().matches(INDEXES_REGEX);
        }
    };

    /**
     * Default constructor is private as each method is static.
     */
    private CmsSpellcheckDictionaryIndexer() {

    }

    /**
     * Adds all dictionaries that are available in the default directory. <p>
     *
     * @param client The SolrClient instance object.
     * @param cms the cms context
     */
    public static void parseAndAddDictionaries(SolrClient client, CmsObject cms) {

        if ((null == client) || (null == cms)) {
            return;
        }

        // Set the correct cms context
        setCmsOfflineProject(cms);

        try {
            // Get all file resources in the default dictionary directory
            final List<CmsResource> resources = cms.getResourcesInFolder(
                DEFAULT_DICTIONARY_DIRECTORY,
                CmsResourceFilter.DEFAULT_FILES);

            for (final CmsResource resource : resources) {
                final String resourceName = resource.getName();
                // Check if the name of the file matches the dictionary naming scheme
                String lang = null;
                if (resourceName.matches(DICTIONARY_NAME_REGEX)) {
                    // Extract the language code that consists of two letters (de, en, es, ...)
                    lang = resourceName.substring(5, 7);
                } else if (resourceName.matches(CUSTOM_DICTIONARY)) {
                    lang = resourceName.substring(12, 14);
                }

                if (null != lang) {
                    // Read the file
                    final CmsFile file = cms.readFile(resource);

                    // Parse file content and add it to the server
                    final List<SolrInputDocument> documents = new ArrayList<SolrInputDocument>();

                    readAndAddDocumentsFromStream(
                        client,
                        lang,
                        new ByteArrayInputStream(file.getContents()),
                        documents,
                        true);

                    // Add and commit the remaining documents to the server
                    addDocuments(client, documents, true);
                }
            }

        } catch (CmsException e) {
            LOG.warn("Could not read from resource. ");
        } catch (IOException e) {
            LOG.warn("Could not successfully parse the dictionary. ");
        } catch (SolrServerException e) {
            LOG.warn("Exception while adding documents to Solr server. ");
        }
    }

    /**
     *
     * @param client The SolrClient instance object.
     * @param cms The OpenCms instance object.
     */
    public static void parseAndAddZippedDictionaries(SolrClient client, CmsObject cms) {

        try {
            final List<CmsResource> resources = cms.getResourcesInFolder(
                DEFAULT_DICTIONARY_DIRECTORY,
                CmsResourceFilter.DEFAULT_FILES);

            // List holding all input documents, regardless of language
            final List<SolrInputDocument> documents = new LinkedList<SolrInputDocument>();

            for (CmsResource resource : resources) {
                final String zipFileName = resource.getName();
                if (zipFileName.matches(ZIP_NAME_REGEX)) {
                    final CmsFile cmsFile = cms.readFile(resource);

                    // Read zip file content
                    try (
                    ZipInputStream zipStream = new ZipInputStream(new ByteArrayInputStream(cmsFile.getContents()))) {

                        // Holds several entries (files) of the zipfile
                        ZipEntry entry = zipStream.getNextEntry();

                        // Iterate over each files in the zip file
                        while (null != entry) {
                            // Extract name to check if name matches the regex and to guess the
                            // language from the filename
                            final String name = entry.getName();

                            if (name.matches(DICTIONARY_NAME_REGEX)) {

                                // The (matching) filename reveals the language
                                final String lang = name.substring(5, 7);

                                // Parse and add documents
                                readAndAddDocumentsFromStream(client, lang, zipStream, documents, false);

                                // Get the next file in the zip
                                entry = zipStream.getNextEntry();
                            }

                        }
                    }
                }
            }

            // Add all documents
            addDocuments(client, documents, true);
        } catch (IOException e) {
            LOG.warn("Failed while reading from " + DEFAULT_DICTIONARY_DIRECTORY + ". ");
        } catch (CmsException e) {
            LOG.warn("Failed reading resource " + DEFAULT_DICTIONARY_DIRECTORY + ". ");
        } catch (SolrServerException e) {
            LOG.warn("Failed adding documents to Solr server. ");
        }
    }

    /**
     * Checks whether a built of the indices is necessary.
     * @param cms The appropriate CmsObject instance.
     * @return true, if the spellcheck indices have to be rebuilt, otherwise false
     */
    public static boolean updatingIndexNecessesary(CmsObject cms) {

        // Set request to the offline project.
        setCmsOfflineProject(cms);

        // Check whether the spellcheck index directories are empty.
        // If they are, the index has to be built obviously.
        if (isSolrSpellcheckIndexDirectoryEmpty()) {
            return true;
        }

        // Compare the most recent date of a dictionary with the oldest timestamp
        // that determines when an index has been built.
        long dateMostRecentDictionary = getMostRecentDate(cms);
        long dateOldestIndexWrite = getOldestIndexDate(cms);

        return dateMostRecentDictionary > dateOldestIndexWrite;
    }

    /**
     * Add a list of documents to the Solr client.<p>
     *
     * @param client The SolrClient instance object.
     * @param documents The documents that should be added.
     * @param commit boolean flag indicating whether a "commit" call should be made after adding the documents
     *
     * @throws IOException in case something goes wrong
     * @throws SolrServerException in case something goes wrong
     */
    static void addDocuments(SolrClient client, List<SolrInputDocument> documents, boolean commit)
    throws IOException, SolrServerException {

        if ((null == client) || (null == documents)) {
            return;
        }

        if (!documents.isEmpty()) {
            client.add(documents);
        }

        if (commit) {
            client.commit();
        }
    }

    /**
     * Deletes all documents from the Solr client.<p>
     *
     * @param client The SolrClient instance object.
     *
     * @throws IOException in case something goes wrong
     * @throws SolrServerException in case something goes wrong
     */
    static void deleteAllFiles(SolrClient client) throws IOException, SolrServerException {

        if (null == client) {
            return;
        }

        client.deleteByQuery("*:*");
        client.commit();
    }

    /**
     * Deletes a single document from the Solr client.<p>
     *
     * @param client The SolrClient instance object.
     * @param lang The affected language.
     * @param word The word that should be removed.
     *
     * @throws IOException in case something goes wrong
     * @throws SolrServerException in case something goes wrong
     */
    static void deleteDocument(SolrClient client, String lang, String word) throws IOException, SolrServerException {

        if ((null == client)
            || CmsStringUtil.isEmptyOrWhitespaceOnly(lang)
            || CmsStringUtil.isEmptyOrWhitespaceOnly(word)) {
            return;
        }

        // Make sure the parameter holding the word that should be deleted
        // contains just a single word
        if (word.trim().contains(" ")) {
            final String query = String.format("entry_%s:%s", lang, word);
            client.deleteByQuery(query);
        }
    }

    /**
     * Determines and returns the timestamp of the most recently modified spellchecker file.<p>
     *
     * @param cms the OpenCms instance.
     * @return timestamp of type long.
     */
    private static long getMostRecentDate(CmsObject cms) {

        long mostRecentDate = Long.MIN_VALUE;

        try {
            final List<CmsResource> resources = cms.getResourcesInFolder(
                DEFAULT_DICTIONARY_DIRECTORY,
                CmsResourceFilter.DEFAULT_FILES);

            for (final CmsResource resource : resources) {
                final String resourceName = resource.getName();
                // Check whether the resource matches the desired patterns
                if (resourceName.matches(DICTIONARY_NAME_REGEX)
                    || resourceName.matches(ZIP_NAME_REGEX)
                    || resourceName.matches(CUSTOM_DICTIONARY)) {
                    if (resource.getDateLastModified() > mostRecentDate) {
                        mostRecentDate = resource.getDateLastModified();
                    }
                }
            }
        } catch (CmsException e) {
            LOG.error("Could not read spellchecker dictionaries. ");
        }

        return mostRecentDate;
    }

    /**
     * Returns the timestamp of the index whose index-built operation lies the
     * furthest back in the past.<p>
     *
     * @param cms the OpenCms instance.
     * @return timestamp as type long.
     */
    private static long getOldestIndexDate(CmsObject cms) {

        final File path = new File(getSolrSpellcheckRfsPath());
        final File[] directories = path.listFiles(SPELLCHECKING_DIRECTORY_NAME_FILTER);

        // Initialize with the greatest value a long type can hold
        long oldestIndexDate = Long.MAX_VALUE;

        for (final File dir : directories) {
            long date = dir.lastModified();
            if (date < oldestIndexDate) {
                oldestIndexDate = date;
            }
        }

        // If no file(s) have been found oldestIndexDate is still holding
        // Long.MAX_VALUE. In that case return Long.MIN_VALUE to ensure
        // that no indexing operation takes place.
        if (Long.MAX_VALUE == oldestIndexDate) {
            LOG.warn("It appears that no spellcheck indices have been found in " + getSolrSpellcheckRfsPath() + ". ");
            return Long.MIN_VALUE;
        }

        return oldestIndexDate;
    }

    /**
     * Returns the path in the RFS where the Solr spellcheck files reside.
     * @return String representation of Solrs spellcheck RFS path.
     */
    private static String getSolrSpellcheckRfsPath() {

        String sPath = OpenCms.getSystemInfo().getWebInfRfsPath();

        if (!OpenCms.getSystemInfo().getWebInfRfsPath().endsWith(File.separator)) {
            sPath += File.separator;
        }

        return sPath + "solr" + File.separator + "spellcheck" + File.separator + "data";
    }

    /**
     * Returns whether the Solr spellchecking index directories are empty
     * (not initiliazed) or not.
     * @return true, if the directories contain no indexed data, otherwise false.
     */
    private static boolean isSolrSpellcheckIndexDirectoryEmpty() {

        final File path = new File(getSolrSpellcheckRfsPath());
        final File[] directories = path.listFiles(SPELLCHECKING_DIRECTORY_NAME_FILTER);

        // Each directory that has been created by Solr but hasn't been indexed yet
        // contains exactly two files. If there are more files, at least one index has
        // already been built, so return false in that case.
        if (directories != null) {
            for (final File directory : directories) {
                if (directory.list().length > 2) {
                    return false;
                }
            }
        }
        return true;
    }

    /**
     * Parses the dictionary from an InputStream.
     *
     * @param client The SolrClient instance object.
     * @param lang The language of the dictionary.
     * @param is The InputStream object.
     * @param documents List to put the assembled SolrInputObjects into.
     * @param closeStream boolean flag that determines whether to close the inputstream
     * or not.
     */
    private static void readAndAddDocumentsFromStream(
        final SolrClient client,
        final String lang,
        final InputStream is,
        final List<SolrInputDocument> documents,
        final boolean closeStream) {

        final BufferedReader br = new BufferedReader(new InputStreamReader(is));

        try {
            String line = br.readLine();
            while (null != line) {

                final SolrInputDocument document = new SolrInputDocument();
                // Each field is named after the schema "entry_xx" where xx denotes
                // the two digit language code. See the file spellcheck/conf/schema.xml.
                document.addField("entry_" + lang, line);
                documents.add(document);

                // Prevent OutOfMemoryExceptions ...
                if (documents.size() >= MAX_LIST_SIZE) {
                    addDocuments(client, documents, false);
                    documents.clear();
                }

                line = br.readLine();
            }
        } catch (IOException e) {
            LOG.error("Could not read spellcheck dictionary from input stream.");
        } catch (SolrServerException e) {
            LOG.error("Error while adding documents to Solr server. ");
        } finally {
            try {
                if (closeStream) {
                    br.close();
                }
            } catch (Exception e) {
                // Nothing to do here anymore ....
            }
        }
    }

    /**
     * Sets the appropriate OpenCms context.
     * @param cms The OpenCms instance object.
     */
    private static void setCmsOfflineProject(CmsObject cms) {

        if (null == cms) {
            return;
        }

        final CmsRequestContext cmsContext = cms.getRequestContext();
        final CmsProject cmsProject = cmsContext.getCurrentProject();

        if (cmsProject.isOnlineProject()) {
            CmsProject cmsOfflineProject;
            try {
                cmsOfflineProject = cms.readProject("Offline");
                cmsContext.setCurrentProject(cmsOfflineProject);
            } catch (CmsException e) {
                LOG.warn("Could not set the current project to \"Offline\". ");
            }
        }
    }
}