001/*
002 * This library is part of OpenCms -
003 * the Open Source Content Management System
004 *
005 * Copyright (c) Alkacon Software GmbH & Co. KG (https://www.alkacon.com)
006 *
007 * This library is free software; you can redistribute it and/or
008 * modify it under the terms of the GNU Lesser General Public
009 * License as published by the Free Software Foundation; either
010 * version 2.1 of the License, or (at your option) any later version.
011 *
012 * This library is distributed in the hope that it will be useful,
013 * but WITHOUT ANY WARRANTY; without even the implied warranty of
014 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
015 * Lesser General Public License for more details.
016 *
017 * For further information about Alkacon Software GmbH & Co. KG, please see the
018 * company website: https://www.alkacon.com
019 *
020 * For further information about OpenCms, please see the
021 * project website: https://www.opencms.org
022 *
023 * You should have received a copy of the GNU Lesser General Public
024 * License along with this library; if not, write to the Free Software
025 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
026 */
027
028package org.opencms.flex;
029
030import org.opencms.cache.CmsLruCache;
031import org.opencms.cache.I_CmsLruCacheObject;
032import org.opencms.db.CmsModificationContext;
033import org.opencms.db.CmsPublishedResource;
034import org.opencms.file.CmsObject;
035import org.opencms.flex.CmsFlexBucketConfiguration.BucketSet;
036import org.opencms.loader.CmsJspLoader;
037import org.opencms.main.CmsException;
038import org.opencms.main.CmsLog;
039import org.opencms.main.I_CmsEventListener;
040import org.opencms.main.OpenCms;
041import org.opencms.security.CmsRole;
042import org.opencms.util.CmsCollectionsGenericWrapper;
043import org.opencms.util.CmsStringUtil;
044import org.opencms.util.CmsUUID;
045
046import java.util.ArrayList;
047import java.util.Collection;
048import java.util.Collections;
049import java.util.HashMap;
050import java.util.HashSet;
051import java.util.Hashtable;
052import java.util.Iterator;
053import java.util.List;
054import java.util.Map;
055import java.util.Set;
056import java.util.concurrent.Future;
057import java.util.concurrent.LinkedBlockingQueue;
058import java.util.concurrent.TimeUnit;
059
060import org.apache.commons.collections.map.LRUMap;
061import org.apache.commons.logging.Log;
062
063import com.google.common.collect.Lists;
064
065/**
066 * This class implements the FlexCache.<p>
067 *
068 * The data structure used is a two-level hashtable.
069 * This is optimized for the structure of the keys that are used to describe the
070 * caching behaviour of the entries.
071 * The first hash-level is calculated from the resource name, i.e. the
072 * name of the resource as it is referred to in the VFS of OpenCms.
073 * The second hash-level is calculated from the cache-key of the resource,
074 * which also is a String representing the specifc variation of the cached entry.<p>
075 *
076 * A suffix [online] or [offline] is appended to te resource name
077 * to distinguish between the online and offline projects of OpenCms.
078 * Also, for support of JSP based workplace pages, a suffix [workplace]
079 * is appended. The same cached workplace pages are used both in the online and
080 * all offline projects.<p>
081 *
082 * Entries in the first level of the cache are of type CmsFlexCacheVariation,
083 * which is a sub-class of CmsFlexCache.
084 * This class is a simple data type that contains of a Map of CmsFlexCacheEntries,
085 * with variations - Strings as keys.<p>
086 *
087 * Here's a short summary of used terms:
088 * <ul>
089 * <li><b>key:</b>
090 * A combination of a resource name and a variation.
091 * The data structure used is CmsFlexCacheKey.
092 * <li><b>resource:</b>
093 * A String with the resource name and an appended [online] of [offline] suffix.
094 * <li><b>variation:</b>
095 * A String describing a variation of a cached entry in the CmsFlexCache language.
096 * <li><b>entry:</b>
097 * A CmsFlexCacheEntry data structure which is describes a cached OpenCms resource.
098 * For every entry a key is saved which contains the resource name and the variation.
099 * </ul>
100 *
101 * Cache clearing is handled using events.
102 * The cache is fully flushed if an event {@link I_CmsEventListener#EVENT_PUBLISH_PROJECT}
103 * or {@link I_CmsEventListener#EVENT_CLEAR_CACHES} is caught.<p>
104 *
105 * @since 6.0.0
106 *
107 * @see org.opencms.flex.CmsFlexCacheKey
108 * @see org.opencms.flex.CmsFlexCacheEntry
109 * @see org.opencms.cache.CmsLruCache
110 * @see org.opencms.cache.I_CmsLruCacheObject
111 */
112public class CmsFlexCache extends Object implements I_CmsEventListener {
113
114    /**
115     * A simple data container class for the FlexCache variations.<p>
116     */
117    public static class CmsFlexCacheVariation extends Object {
118
119        /** The key belonging to the resource. */
120        public CmsFlexCacheKey m_key;
121
122        /** Maps variations to CmsFlexCacheEntries. */
123        public Map<String, I_CmsLruCacheObject> m_map;
124
125        /**
126         * Generates a new instance of CmsFlexCacheVariation.<p>
127         *
128         * @param theKey The (resource) key to contruct this variation list for
129         */
130        public CmsFlexCacheVariation(CmsFlexCacheKey theKey) {
131
132            m_key = theKey;
133            m_map = new Hashtable<String, I_CmsLruCacheObject>(INITIAL_CAPACITY_VARIATIONS);
134        }
135    }
136
137    /**
138     * Extended LRUMap that handles the variations in case a key is removed.<p>
139     */
140    class CmsFlexKeyMap extends LRUMap {
141
142        /** Serial version UID required for safe serialization. */
143        private static final long serialVersionUID = 6931995916013396902L;
144
145        /**
146         * Initialize the map with the given size.<p>
147         *
148         * @param maxSize the maximum number of key to cache
149         */
150        public CmsFlexKeyMap(int maxSize) {
151
152            super(maxSize);
153        }
154
155        /**
156         * Ensures that all variations that referenced by this key are released
157         * if the key is released.<p>
158         *
159         * @param entry the entry to remove
160         *
161         * @return <code>true</code> to actually delete the entry
162         *
163         * @see LRUMap#removeLRU(LinkEntry)
164         */
165        @Override
166        protected boolean removeLRU(LinkEntry entry) {
167
168            CmsFlexCacheVariation v = (CmsFlexCacheVariation)entry.getValue();
169            if (v == null) {
170                return true;
171            }
172            Map<String, I_CmsLruCacheObject> m = v.m_map;
173            if ((m == null) || (m.size() == 0)) {
174                return true;
175            }
176
177            // make a copy to safely iterate over because the line "m_variationCache.remove(e)" modifies the variation map for the key
178            Collection<I_CmsLruCacheObject> entries = new ArrayList<I_CmsLruCacheObject>(m.values());
179            synchronized (m_variationCache) {
180                for (I_CmsLruCacheObject e : entries) {
181                    m_variationCache.remove(e);
182                }
183                v.m_map.clear();
184                v.m_map = null;
185                v.m_key = null;
186            }
187            return true;
188        }
189    }
190
191    /**Constant for distinguish cache action.*/
192    public static final String CACHE_ACTION = "action";
193
194    /** Suffix to append to online cache entries. */
195    public static final String CACHE_OFFLINESUFFIX = " [offline]";
196
197    /** Suffix to append to online cache entries. */
198    public static final String CACHE_ONLINESUFFIX = " [online]";
199
200    /** Trigger for clearcache event: Clear complete cache. */
201    public static final int CLEAR_ALL = 0;
202
203    /** Trigger for clearcache event: Clear only entries. */
204    public static final int CLEAR_ENTRIES = 1;
205
206    /** Trigger for clearcache event: Clear complete offine cache. */
207    public static final int CLEAR_OFFLINE_ALL = 4;
208
209    /** Trigger for clearcache event: Clear only offline entries. */
210    public static final int CLEAR_OFFLINE_ENTRIES = 5;
211
212    /** Trigger for clearcache event: Clear complete online cache. */
213    public static final int CLEAR_ONLINE_ALL = 2;
214
215    /** Trigger for clearcache event: Clear only online entries. */
216    public static final int CLEAR_ONLINE_ENTRIES = 3;
217
218    /** The configuration for the Flex cache buckets. */
219    public static final String CONFIG_PATH = "/system/config/flexconfig.properties";
220
221    /** Initial cache size, this should be a power of 2 because of the Java collections implementation. */
222    public static final int INITIAL_CAPACITY_CACHE = 512;
223
224    /** Initial size for variation lists, should be a power of 2. */
225    public static final int INITIAL_CAPACITY_VARIATIONS = 8;
226
227    /** Offline repository constant. */
228    public static final String REPOSITORY_OFFLINE = "offline";
229
230    /** Online repository constant. */
231    public static final String REPOSITORY_ONLINE = "online";
232
233    /** The log object for this class. */
234    private static final Log LOG = CmsLog.getLog(CmsFlexCache.class);
235
236    /** The LRU cache to organize the cached entries. */
237    protected CmsLruCache m_variationCache;
238
239    /** The Flex bucket configuration. */
240    private volatile CmsFlexBucketConfiguration m_bucketConfiguration;
241
242    /** Indicates if offline resources should be cached or not. */
243    private boolean m_cacheOffline;
244
245    /** The CMS object used for VFS operations. */
246    private CmsObject m_cmsObject;
247
248    /** Indicates if the cache is enabled or not. */
249    private boolean m_enabled;
250
251    /** Map to store the entries for fast lookup. */
252    private Map<String, CmsFlexCacheVariation> m_keyCache;
253
254    /** Counter for the size. */
255    private int m_size;
256
257    private LinkedBlockingQueue<String> m_publishPathsForDelayedClear = new LinkedBlockingQueue<>();
258
259    private volatile Future<?> m_delayedClear;
260
261    /**
262     * Constructor for class CmsFlexCache.<p>
263     *
264     * The parameter "enabled" is used to control if the cache is
265     * actually on or off. Even if you don't need the cache, you still
266     * have to create an instance of it with enabled=false.
267     * This is because you need some of the FlexCache data structures
268     * for JSP inclusion buffering.<p>
269     *
270     * @param configuration the flex cache configuration
271     */
272    public CmsFlexCache(CmsFlexCacheConfiguration configuration) {
273
274        m_enabled = configuration.isCacheEnabled();
275        m_cacheOffline = configuration.isCacheOffline();
276
277        long maxCacheBytes = configuration.getMaxCacheBytes();
278        long avgCacheBytes = configuration.getAvgCacheBytes();
279        int maxEntryBytes = configuration.getMaxEntryBytes();
280        int maxKeys = configuration.getMaxKeys();
281
282        m_variationCache = new CmsLruCache(maxCacheBytes, avgCacheBytes, maxEntryBytes);
283        OpenCms.getMemoryMonitor().register(getClass().getName() + ".m_entryLruCache", m_variationCache);
284
285        if (m_enabled) {
286            CmsFlexKeyMap flexKeyMap = new CmsFlexKeyMap(maxKeys);
287            m_keyCache = Collections.synchronizedMap(
288                CmsCollectionsGenericWrapper.<String, CmsFlexCacheVariation> map(flexKeyMap));
289            OpenCms.getMemoryMonitor().register(getClass().getName() + ".m_resourceMap", flexKeyMap);
290
291            OpenCms.addCmsEventListener(
292                this,
293                new int[] {
294                    I_CmsEventListener.EVENT_PUBLISH_PROJECT,
295                    I_CmsEventListener.EVENT_CLEAR_CACHES,
296                    I_CmsEventListener.EVENT_FLEX_PURGE_JSP_REPOSITORY,
297                    I_CmsEventListener.EVENT_FLEX_CACHE_CLEAR});
298        }
299
300        if (LOG.isInfoEnabled()) {
301            LOG.info(
302                Messages.get().getBundle().key(
303                    Messages.INIT_FLEXCACHE_CREATED_2,
304                    Boolean.valueOf(m_enabled),
305                    Boolean.valueOf(m_cacheOffline)));
306        }
307    }
308
309    /**
310     * Copies the key set of a map while synchronizing on the map.<p>
311     *
312     * @param map the map whose key set should be copied
313     * @return the copied key set
314     */
315    private static <K, V> Set<K> synchronizedCopyKeys(Map<K, V> map) {
316
317        if (map == null) {
318            return new HashSet<K>();
319        }
320        synchronized (map) {
321            return new HashSet<K>(map.keySet());
322        }
323    }
324
325    /**
326     * Copies a map while synchronizing on it.<p>
327     *
328     * @param map the map to copy
329     * @return the copied map
330     */
331    private static <K, V> Map<K, V> synchronizedCopyMap(Map<K, V> map) {
332
333        if (map == null) {
334            return new HashMap<K, V>();
335        }
336
337        synchronized (map) {
338
339            return new HashMap<K, V>(map);
340        }
341    }
342
343    /**
344     * Indicates if offline project resources are cached.<p>
345     *
346     * @return true if offline projects are cached, false if not
347     */
348    public boolean cacheOffline() {
349
350        return m_cacheOffline;
351    }
352
353    /**
354     * Implements the CmsEvent interface,
355     * the FlexCache uses the events to clear itself in case a project is published.<p>
356     *
357     * @param event CmsEvent that has occurred
358     */
359    public void cmsEvent(org.opencms.main.CmsEvent event) {
360
361        if (!isEnabled()) {
362            return;
363        }
364
365        switch (event.getType()) {
366            case I_CmsEventListener.EVENT_PUBLISH_PROJECT:
367                if (LOG.isDebugEnabled()) {
368                    LOG.debug("FlexCache: Received event PUBLISH_PROJECT");
369                }
370                String publishIdStr = (String)(event.getData().get(I_CmsEventListener.KEY_PUBLISHID));
371                boolean isInstantPublish = Boolean.TRUE.equals(
372                    event.getData().get(I_CmsEventListener.KEY_INSTANT_PUBLISH));
373                if (!CmsUUID.isValidUUID(publishIdStr)) {
374                    clear();
375                } else {
376                    try {
377                        CmsUUID publishId = new CmsUUID(publishIdStr);
378                        List<CmsPublishedResource> publishedResources = m_cmsObject.readPublishedResources(publishId);
379                        Set<String> paths = new HashSet<>();
380                        for (CmsPublishedResource pubRes : publishedResources) {
381                            paths.add(pubRes.getRootPath());
382                        }
383                        boolean updateConfiguration = false;
384                        for (String path : paths) {
385                            if (CONFIG_PATH.equals(path)) {
386                                updateConfiguration = true;
387                                break;
388                            }
389                        }
390                        if (updateConfiguration) {
391                            LOG.info("Flex bucket configuration was updated, re-initializing configuration...");
392                            try {
393                                m_bucketConfiguration = CmsFlexBucketConfiguration.loadFromVfsFile(
394                                    m_cmsObject,
395                                    CONFIG_PATH);
396                            } catch (CmsException e) {
397                                LOG.error(e.getLocalizedMessage(), e);
398                            }
399                            // Make sure no entries built for the old configuration remain in the cache
400                            clear();
401                        } else {
402                            if (isInstantPublish) {
403                                // Instant publishing happens when writing to online-only folders, which we expect to happen mostly in import jobs of some sort.
404                                // In this scenario, a lot of publish events will be triggered by resource modifications, potentially over a significant time span,
405                                // and we don't want to clear the FlexCache after each one. So we schedule a task to do it 5 seconds in the future, and cancel that
406                                // task if another publish event happens, which means the Flex Cache will eventually be cleared after the 'last' resource is imported.
407                                // (Of course we can't actually know if it's the last resource, that's just a heuristic - if nothing happens for 5 seconds, we assume that it is.)
408                                synchronized (m_publishPathsForDelayedClear) {
409                                    m_publishPathsForDelayedClear.addAll(paths);
410                                    if (m_delayedClear != null) {
411                                        m_delayedClear.cancel(false);
412                                        m_delayedClear = null;
413                                    }
414                                    m_delayedClear = OpenCms.getExecutor().schedule(() -> {
415                                        synchronized (m_publishPathsForDelayedClear) {
416                                            Set<String> fullPaths = new HashSet<>();
417                                            m_publishPathsForDelayedClear.drainTo(fullPaths);
418                                            clearBucketsForPublishList(null, fullPaths);
419                                        }
420                                    },
421                                        CmsModificationContext.getOnlineFolderOptions().getFlexCacheDelay(),
422                                        TimeUnit.MILLISECONDS);
423                                }
424                            } else {
425                                clearBucketsForPublishList(publishId, paths);
426                            }
427                        }
428                    } catch (CmsException e1) {
429                        LOG.error(e1.getLocalizedMessage(), e1);
430                        clear();
431                    }
432                }
433                break;
434            case I_CmsEventListener.EVENT_CLEAR_CACHES:
435                if (LOG.isDebugEnabled()) {
436                    LOG.debug(Messages.get().getBundle().key(Messages.LOG_FLEXCACHE_RECEIVED_EVENT_CLEAR_CACHE_0));
437                }
438                clear();
439                break;
440            case I_CmsEventListener.EVENT_FLEX_PURGE_JSP_REPOSITORY:
441                if (LOG.isDebugEnabled()) {
442                    LOG.debug(Messages.get().getBundle().key(Messages.LOG_FLEXCACHE_RECEIVED_EVENT_PURGE_REPOSITORY_0));
443                }
444                purgeJspRepository();
445                break;
446            case I_CmsEventListener.EVENT_FLEX_CACHE_CLEAR:
447                if (LOG.isDebugEnabled()) {
448                    LOG.debug(
449                        Messages.get().getBundle().key(Messages.LOG_FLEXCACHE_RECEIVED_EVENT_CLEAR_CACHE_PARTIALLY_0));
450                }
451                Map<String, ?> m = event.getData();
452                if (m == null) {
453                    break;
454                }
455                Integer it = null;
456                try {
457                    it = (Integer)m.get(CACHE_ACTION);
458                } catch (Exception e) {
459                    // it will be null
460                }
461                if (it == null) {
462                    LOG.error("Flex cache clear event with no action parameter received");
463                    break;
464                }
465                int i = it.intValue();
466                switch (i) {
467                    case CLEAR_ALL:
468                        clear();
469                        break;
470                    case CLEAR_ENTRIES:
471                        clearEntries();
472                        break;
473                    case CLEAR_ONLINE_ALL:
474                        clearOnline();
475                        break;
476                    case CLEAR_ONLINE_ENTRIES:
477                        clearOnlineEntries();
478                        break;
479                    case CLEAR_OFFLINE_ALL:
480                        clearOffline();
481                        break;
482                    case CLEAR_OFFLINE_ENTRIES:
483                        clearOfflineEntries();
484                        break;
485                    default:
486                        // no operation
487                }
488                break;
489            default:
490                // no operation
491        }
492    }
493
494    /**
495     * Dumps keys and variations to a string buffer, for debug purposes.<p>
496     *
497     * @param buffer the buffer to which the key information should be written
498     */
499    public void dumpKeys(StringBuffer buffer) {
500
501        synchronized (this) {
502            for (Map.Entry<String, CmsFlexCacheVariation> entry : synchronizedCopyMap(m_keyCache).entrySet()) {
503                String key = entry.getKey();
504                CmsFlexCacheVariation variations = entry.getValue();
505                Map<String, I_CmsLruCacheObject> variationMap = variations.m_map;
506                for (Map.Entry<String, I_CmsLruCacheObject> varEntry : variationMap.entrySet()) {
507                    String varKey = varEntry.getKey();
508                    I_CmsLruCacheObject value = varEntry.getValue();
509                    buffer.append(key + " VAR " + varKey + "\n");
510                    if (value instanceof CmsFlexCacheEntry) {
511                        CmsFlexCacheEntry singleCacheEntry = (CmsFlexCacheEntry)value;
512                        BucketSet buckets = singleCacheEntry.getBucketSet();
513                        if (buckets != null) {
514                            buffer.append("buckets = " + buckets.toString() + "\n");
515                        }
516                    }
517                }
518            }
519        }
520    }
521
522    /**
523     * Returns the CmsFlexCacheKey data structure for a given
524     * key (i.e. resource name).<p>
525     *
526     * Useful if you want to show the cache key for a resources,
527     * like on the FlexCache administration page.<p>
528     *
529     * Only users with administrator permissions are allowed
530     * to perform this operation.<p>
531     *
532     * @param key the resource name for which to look up the variation for
533     * @param cms the CmsObject used for user authorization
534     * @return the CmsFlexCacheKey data structure found for the resource
535     */
536    public CmsFlexCacheKey getCachedKey(String key, CmsObject cms) {
537
538        if (!isEnabled() || !OpenCms.getRoleManager().hasRole(cms, CmsRole.WORKPLACE_MANAGER)) {
539            return null;
540        }
541        Object o = m_keyCache.get(key);
542        if (o != null) {
543            return ((CmsFlexCacheVariation)o).m_key;
544        }
545        return null;
546    }
547
548    /**
549     * Returns a set of all cached resource names.<p>
550     *
551     * Useful if you want to show a list of all cached resources,
552     * like on the FlexCache administration page.<p>
553     *
554     * Only users with administrator permissions are allowed
555     * to perform this operation.<p>
556     *
557     * @param cms the CmsObject used for user authorization
558     * @return a Set of cached resource names (which are of type String)
559     */
560    public Set<String> getCachedResources(CmsObject cms) {
561
562        if (!isEnabled() || !OpenCms.getRoleManager().hasRole(cms, CmsRole.WORKPLACE_MANAGER)) {
563            return null;
564        }
565        return synchronizedCopyKeys(m_keyCache);
566    }
567
568    /**
569     * Returns all variations in the cache for a given resource name.
570     * The variations are of type String.<p>
571     *
572     * Useful if you want to show a list of all cached entry - variations,
573     * like on the FlexCache administration page.<p>
574     *
575     * Only users with administrator permissions are allowed
576     * to perform this operation.<p>
577     *
578     * @param key the resource name for which to look up the variations for
579     * @param cms the CmsObject used for user authorization
580     * @return a Set of cached variations (which are of type String)
581     */
582    public Set<String> getCachedVariations(String key, CmsObject cms) {
583
584        if (!isEnabled() || !OpenCms.getRoleManager().hasRole(cms, CmsRole.WORKPLACE_MANAGER)) {
585            return null;
586        }
587        Object o = m_keyCache.get(key);
588        if (o != null) {
589            return synchronizedCopyKeys(((CmsFlexCacheVariation)o).m_map);
590        }
591        return null;
592    }
593
594    /**
595     * Returns the LRU cache where the CacheEntries are cached.<p>
596     *
597     * @return the LRU cache where the CacheEntries are cached
598     */
599    public CmsLruCache getEntryLruCache() {
600
601        return m_variationCache;
602    }
603
604    /**
605     * Initializes the flex cache.<p>
606     *
607     * @param adminCms a CMS context with admin privileges
608     */
609    public void initializeCms(CmsObject adminCms) {
610
611        try {
612            m_cmsObject = adminCms;
613            try {
614                String path = CONFIG_PATH;
615                if (m_cmsObject.existsResource(path)) {
616                    LOG.info("Flex configuration found at " + CONFIG_PATH + ", initializing...");
617                    m_bucketConfiguration = CmsFlexBucketConfiguration.loadFromVfsFile(m_cmsObject, path);
618                }
619            } catch (Exception e) {
620                LOG.error(e.getLocalizedMessage(), e);
621            }
622        } catch (Exception e) {
623            LOG.error(e.getLocalizedMessage(), e);
624        }
625    }
626
627    /**
628     * Indicates if the cache is enabled (i.e. actually
629     * caching entries) or not.<p>
630     *
631     * @return true if the cache is enabled, false if not
632     */
633    public boolean isEnabled() {
634
635        return m_enabled;
636    }
637
638    /**
639     * Returns the total number of cached resource keys.
640     *
641     * @return the number of resource keys in the cache
642     */
643    public int keySize() {
644
645        if (!isEnabled()) {
646            return 0;
647        }
648        return m_keyCache.size();
649    }
650
651    /**
652     * Returns the total number of entries in the cache.<p>
653     *
654     * @return the number of entries in the cache
655     */
656    public int size() {
657
658        return m_variationCache.size();
659    }
660
661    /**
662     * Looks up a specific entry in the cache.<p>
663     *
664     * In case a found entry has a timeout set, it will be checked upon lookup.
665     * In case the timeout of the entry has been reached, it will be removed from
666     * the cache (and null will be returned in this case).<p>
667     *
668     * @param key The key to look for in the cache
669     * @return the entry found for the key, or null if key is not in the cache
670     */
671    CmsFlexCacheEntry get(CmsFlexRequestKey key) {
672
673        if (!isEnabled()) {
674            // cache is disabled
675            return null;
676        }
677        Object o = m_keyCache.get(key.getResource());
678        if (o != null) {
679            // found a matching key in the cache
680            CmsFlexCacheVariation v = (CmsFlexCacheVariation)o;
681            String variation = v.m_key.matchRequestKey(key);
682
683            if (CmsStringUtil.isEmpty(variation)) {
684                // requested resource is not cacheable
685                return null;
686            }
687            CmsFlexCacheEntry entry = (CmsFlexCacheEntry)v.m_map.get(variation);
688            if (entry == null) {
689                // no cache entry available for variation
690                return null;
691            }
692            if (entry.getDateExpires() < System.currentTimeMillis()) {
693                // cache entry avaiable but expired, remove entry
694                m_variationCache.remove(entry);
695                return null;
696            }
697            // return the found cache entry
698            return entry;
699        } else {
700            return null;
701        }
702    }
703
704    /**
705     * Returns the CmsFlexCacheKey data structure for a given resource name.<p>
706     *
707     * @param resource the resource name for which to look up the key for
708     * @return the CmsFlexCacheKey data structure found for the resource
709     */
710    CmsFlexCacheKey getKey(String resource) {
711
712        if (!isEnabled()) {
713            return null;
714        }
715        Object o = m_keyCache.get(resource);
716        if (o != null) {
717            if (LOG.isDebugEnabled()) {
718                LOG.debug(Messages.get().getBundle().key(Messages.LOG_FLEXCACHEKEY_FOUND_1, resource));
719            }
720            return ((CmsFlexCacheVariation)o).m_key;
721        } else {
722            if (LOG.isDebugEnabled()) {
723                LOG.debug(Messages.get().getBundle().key(Messages.LOG_FLEXCACHEKEY_NOT_FOUND_1, resource));
724            }
725            return null;
726        }
727    }
728
729    /**
730     * Checks if the cache is empty or if at last one element is contained.<p>
731     *
732     * @return true if the cache is empty, false otherwise
733     */
734    boolean isEmpty() {
735
736        if (!isEnabled()) {
737            return true;
738        }
739        return m_keyCache.isEmpty();
740    }
741
742    /**
743     * This method adds new entries to the cache.<p>
744     *
745     * The key describes the conditions under which the value can be cached.
746     * Usually the key belongs to the response.
747     * The variation describes the conditions under which the
748     * entry was created. This is usually calculated from the request.
749     * If the variation is != null, the entry is cachable.<p>
750     *
751     * @param key the key for the new value entry
752     * @param entry the CmsFlexCacheEntry to store in the cache
753     * @param variation the pre-calculated variation for the entry
754     * @param requestKey the request key from which the variation was determined
755     * @return true if the value was added to the cache, false otherwise
756     */
757    boolean put(CmsFlexCacheKey key, CmsFlexCacheEntry entry, String variation, CmsFlexRequestKey requestKey) {
758
759        if (!isEnabled()) {
760            return false;
761        }
762        if (LOG.isDebugEnabled()) {
763            LOG.debug(Messages.get().getBundle().key(Messages.LOG_FLEXCACHE_ADD_ENTRY_1, key.getResource()));
764        }
765        if (variation != null) {
766            // This is a cachable result
767            if (LOG.isDebugEnabled()) {
768                LOG.debug(
769                    Messages.get().getBundle().key(
770                        Messages.LOG_FLEXCACHE_ADD_ENTRY_WITH_VARIATION_2,
771                        key.getResource(),
772                        variation));
773            }
774            put(key, entry, variation);
775            if (m_bucketConfiguration != null) {
776                try {
777                    List<String> paths = key.getPathsForBuckets(requestKey);
778                    if (paths.size() > 0) {
779                        BucketSet buckets = m_bucketConfiguration.getBucketSet(paths);
780                        entry.setBucketSet(buckets);
781                    } else {
782                        entry.setBucketSet(null); // bucket set of null means entries will be deleted for every publish job
783                    }
784                } catch (Exception e) {
785                    LOG.error(e.getLocalizedMessage(), e);
786                }
787            }
788            // Note that duplicates are NOT checked, it it assumed that this is done beforehand,
789            // while checking if the entry is already in the cache or not.
790            return true;
791        } else {
792            // Result is not cachable
793            if (LOG.isDebugEnabled()) {
794                LOG.debug(Messages.get().getBundle().key(Messages.LOG_FLEXCACHE_RESOURCE_NOT_CACHEABLE_0));
795            }
796            return false;
797        }
798    }
799
800    /**
801     * Adds a key with a new, empty variation map to the cache.<p>
802     *
803     * @param key the key to add to the cache.
804     */
805    void putKey(CmsFlexCacheKey key) {
806
807        if (!isEnabled()) {
808            return;
809        }
810        Object o = m_keyCache.get(key.getResource());
811        if (o == null) {
812            // No variation map for this resource yet, so create one
813            CmsFlexCacheVariation variationMap = new CmsFlexCacheVariation(key);
814            m_keyCache.put(key.getResource(), variationMap);
815            if (LOG.isDebugEnabled()) {
816                LOG.debug(Messages.get().getBundle().key(Messages.LOG_FLEXCACHE_ADD_KEY_1, key.getResource()));
817            }
818        }
819        // If != null the key is already in the cache, so we just do nothing
820    }
821
822    /**
823     * Empties the cache completely.<p>
824     */
825    private synchronized void clear() {
826
827        if (!isEnabled()) {
828            return;
829        }
830        m_keyCache.clear();
831        m_size = 0;
832
833        m_variationCache.clear();
834
835        if (LOG.isInfoEnabled()) {
836            LOG.info(Messages.get().getBundle().key(Messages.LOG_FLEXCACHE_CLEAR_0));
837        }
838    }
839
840    /**
841     * Internal method to perform cache clearance.<p>
842     *
843     * It clears "one half" of the cache, i.e. either
844     * the online or the offline parts.
845     * A parameter is used to indicate if only
846     * the entries or keys and entries are to be cleared.<p>
847     *
848     * @param suffix used to distinguish between "[Online]" and "[Offline]" entries
849     * @param entriesOnly if <code>true</code>, only entries will be cleared, otherwise
850     *         the entries and the keys will be cleared
851     */
852    private synchronized void clearAccordingToSuffix(String suffix, boolean entriesOnly) {
853
854        Set<String> keys = synchronizedCopyKeys(m_keyCache);
855        Iterator<String> i = keys.iterator();
856        while (i.hasNext()) {
857            String s = i.next();
858            if (s.endsWith(suffix)) {
859                CmsFlexCacheVariation v = m_keyCache.get(s);
860                if (entriesOnly) {
861                    // Clear only entry
862                    m_size -= v.m_map.size();
863                    Iterator<I_CmsLruCacheObject> allEntries = v.m_map.values().iterator();
864                    while (allEntries.hasNext()) {
865                        I_CmsLruCacheObject nextObject = allEntries.next();
866                        allEntries.remove();
867                        m_variationCache.remove(nextObject);
868                    }
869                    v.m_map = new Hashtable<String, I_CmsLruCacheObject>(INITIAL_CAPACITY_VARIATIONS);
870                } else {
871                    // Clear key and entry
872                    m_size -= v.m_map.size();
873                    Iterator<I_CmsLruCacheObject> allEntries = v.m_map.values().iterator();
874                    while (allEntries.hasNext()) {
875                        I_CmsLruCacheObject nextObject = allEntries.next();
876                        allEntries.remove();
877                        m_variationCache.remove(nextObject);
878                    }
879
880                    v.m_map = null;
881                    v.m_key = null;
882                    m_keyCache.remove(s);
883                }
884            }
885        }
886        if (LOG.isInfoEnabled()) {
887            LOG.info(
888                Messages.get().getBundle().key(
889                    Messages.LOG_FLEXCACHE_CLEAR_HALF_2,
890                    suffix,
891                    Boolean.valueOf(entriesOnly)));
892        }
893    }
894
895    /**
896     * Clears the Flex cache buckets matching the given publish list.<p>
897     *
898     * @param bucketConfig the bucket configuration to be used for checking which flex cache entry should be purged
899     * @param publishId the publish id
900     * @param publishedResources the published resources
901     *
902     * @return true if the flex buckets could be cleared successfully (if this returns false, the flex cache should fall back to the old behavior, i.e. clearing everything)
903     */
904    private void clearBucketsForPublishList(CmsUUID publishId, Collection<String> paths) {
905
906        CmsFlexBucketConfiguration bucketConfig = m_bucketConfiguration;
907        long startTime = System.currentTimeMillis();
908        String p = "[" + publishId + "] "; // Prefix for log messages
909        if (bucketConfig == null) {
910            clear();
911            return;
912        }
913        try {
914
915            LOG.debug(p + "Trying bucket-based flex entry cleanup");
916            if (bucketConfig.shouldClearAll(paths)) {
917                LOG.info(p + "Clearing Flex cache completely based on Flex bucket configuration.");
918                clear();
919            } else {
920                long totalEntries = 0;
921                long removedEntries = 0;
922                BucketSet publishListBucketSet = bucketConfig.getBucketSet(paths);
923                if (LOG.isInfoEnabled()) {
924                    LOG.info(p + "Flex cache buckets for publish list: " + publishListBucketSet.toString());
925                }
926                synchronized (this) {
927                    List<CmsFlexCacheEntry> entriesToDelete = Lists.newArrayList();
928                    for (Map.Entry<String, CmsFlexCacheVariation> entry : synchronizedCopyMap(m_keyCache).entrySet()) {
929                        CmsFlexCacheVariation variation = entry.getValue();
930                        if (LOG.isDebugEnabled()) {
931                            LOG.debug(p + "Processing entries for " + entry.getKey());
932                        }
933                        entriesToDelete.clear();
934
935                        for (Map.Entry<String, I_CmsLruCacheObject> variationEntry : synchronizedCopyMap(
936                            variation.m_map).entrySet()) {
937                            CmsFlexCacheEntry flexEntry = (CmsFlexCacheEntry)(variationEntry.getValue());
938                            totalEntries += 1;
939                            BucketSet entryBucketSet = flexEntry.getBucketSet();
940                            if (publishListBucketSet.matchForDeletion(entryBucketSet)) {
941                                entriesToDelete.add(flexEntry);
942                                if (LOG.isInfoEnabled()) {
943                                    LOG.info(p + "Match: " + variationEntry.getKey());
944                                }
945                            } else {
946                                if (LOG.isDebugEnabled()) {
947                                    LOG.debug(p + "No match: " + variationEntry.getKey());
948                                }
949                            }
950                        }
951                        for (CmsFlexCacheEntry entryToDelete : entriesToDelete) {
952                            m_variationCache.remove(entryToDelete);
953                            removedEntries += 1;
954                        }
955                    }
956                    long endTime = System.currentTimeMillis();
957                    LOG.info(
958                        p
959                            + "Removed "
960                            + removedEntries
961                            + " of "
962                            + totalEntries
963                            + " Flex cache entries, took "
964                            + (endTime - startTime)
965                            + " milliseconds");
966                }
967            }
968        } catch (Exception e) {
969            LOG.error(p + "Exception while trying to selectively purge flex cache: " + e.getLocalizedMessage(), e);
970            clear();
971        }
972    }
973
974    /**
975     * Clears all entries in the cache, online or offline.<p>
976     *
977     * The keys are not cleared.<p>
978     *
979     * Only users with administrator permissions are allowed
980     * to perform this operation.<p>
981     */
982    private synchronized void clearEntries() {
983
984        if (!isEnabled()) {
985            return;
986        }
987        if (LOG.isInfoEnabled()) {
988            LOG.info(Messages.get().getBundle().key(Messages.LOG_FLEXCACHE_CLEAR_ALL_0));
989        }
990        // create new set to avoid ConcurrentModificationExceptions
991        Set<String> cacheKeys = synchronizedCopyKeys(m_keyCache);
992        Iterator<String> i = cacheKeys.iterator();
993        while (i.hasNext()) {
994            CmsFlexCacheVariation v = m_keyCache.get(i.next());
995            Iterator<I_CmsLruCacheObject> allEntries = v.m_map.values().iterator();
996            while (allEntries.hasNext()) {
997                I_CmsLruCacheObject nextObject = allEntries.next();
998                allEntries.remove();
999                m_variationCache.remove(nextObject);
1000            }
1001            v.m_map = new Hashtable<String, I_CmsLruCacheObject>(INITIAL_CAPACITY_VARIATIONS);
1002        }
1003        m_size = 0;
1004    }
1005
1006    /**
1007     * Clears all entries and all keys from offline projects in the cache.<p>
1008     *
1009     * Cached resources from the online project are not touched.<p>
1010     *
1011     * Only users with administrator permissions are allowed
1012     * to perform this operation.<p>
1013     */
1014    private void clearOffline() {
1015
1016        if (!isEnabled()) {
1017            return;
1018        }
1019        if (LOG.isInfoEnabled()) {
1020            LOG.info(Messages.get().getBundle().key(Messages.LOG_FLEXCACHE_CLEAR_KEYS_AND_ENTRIES_0));
1021        }
1022        clearAccordingToSuffix(CACHE_OFFLINESUFFIX, false);
1023    }
1024
1025    /**
1026     * Clears all entries from offline projects in the cache.<p>
1027     *
1028     * The keys from the offline projects are not cleared.
1029     * Cached resources from the online project are not touched.<p>
1030     *
1031     * Only users with administrator permissions are allowed
1032     * to perform this operation.<p>
1033     */
1034    private void clearOfflineEntries() {
1035
1036        if (!isEnabled()) {
1037            return;
1038        }
1039        if (LOG.isInfoEnabled()) {
1040            LOG.info(Messages.get().getBundle().key(Messages.LOG_FLEXCACHE_CLEAR_OFFLINE_ENTRIES_0));
1041        }
1042        clearAccordingToSuffix(CACHE_OFFLINESUFFIX, true);
1043    }
1044
1045    /**
1046     * Clears all entries and all keys from the online project in the cache.<p>
1047     *
1048     * Cached resources from the offline projects are not touched.<p>
1049     *
1050     * Only users with administrator permissions are allowed
1051     * to perform this operation.<p>
1052     */
1053    private void clearOnline() {
1054
1055        if (!isEnabled()) {
1056            return;
1057        }
1058        if (LOG.isInfoEnabled()) {
1059            LOG.info(Messages.get().getBundle().key(Messages.LOG_FLEXCACHE_CLEAR_ONLINE_KEYS_AND_ENTRIES_0));
1060        }
1061        clearAccordingToSuffix(CACHE_ONLINESUFFIX, false);
1062    }
1063
1064    /**
1065     * Clears all entries from the online project in the cache.<p>
1066     *
1067     * The keys from the online project are not cleared.
1068     * Cached resources from the offline projects are not touched.<p>
1069     *
1070     * Only users with administrator permissions are allowed
1071     * to perform this operation.<p>
1072     */
1073    private void clearOnlineEntries() {
1074
1075        if (!isEnabled()) {
1076            return;
1077        }
1078        if (LOG.isInfoEnabled()) {
1079            LOG.info(Messages.get().getBundle().key(Messages.LOG_FLEXCACHE_CLEAR_ONLINE_ENTRIES_0));
1080        }
1081        clearAccordingToSuffix(CACHE_ONLINESUFFIX, true);
1082    }
1083
1084    /**
1085     * This method purges the JSP repository dirs,
1086     * i.e. it deletes all JSP files that OpenCms has written to the
1087     * real FS.<p>
1088     *
1089     * Obviously this method must be used with caution.
1090     * Purpose of this method is to allow
1091     * a complete purge of all JSP pages on a machine after
1092     * a major update of JSP templates was made.<p>
1093     */
1094    private synchronized void purgeJspRepository() {
1095
1096        CmsJspLoader cmsJspLoader = (CmsJspLoader)OpenCms.getResourceManager().getLoader(
1097            CmsJspLoader.RESOURCE_LOADER_ID);
1098
1099        cmsJspLoader.triggerPurge(new Runnable() {
1100
1101            @SuppressWarnings("synthetic-access")
1102            public void run() {
1103
1104                clear();
1105            }
1106        });
1107    }
1108
1109    /**
1110     * Save a value to the cache.<p>
1111     *
1112     * @param key the key under which the value is saved
1113     * @param theCacheEntry the entry to cache
1114     * @param variation the variation string
1115     */
1116    private void put(CmsFlexCacheKey key, CmsFlexCacheEntry theCacheEntry, String variation) {
1117
1118        CmsFlexCacheVariation o = m_keyCache.get(key.getResource());
1119        if (o != null) {
1120            // We already have a variation map for this resource
1121            Map<String, I_CmsLruCacheObject> m = o.m_map;
1122            boolean wasAdded = true;
1123            if (!m.containsKey(variation)) {
1124                wasAdded = m_variationCache.add(theCacheEntry);
1125            } else {
1126                wasAdded = m_variationCache.touch(theCacheEntry);
1127            }
1128
1129            if (wasAdded) {
1130                theCacheEntry.setVariationData(variation, m);
1131                m.put(variation, theCacheEntry);
1132            }
1133        } else {
1134            // No variation map for this resource yet, so create one
1135            CmsFlexCacheVariation list = new CmsFlexCacheVariation(key);
1136
1137            boolean wasAdded = m_variationCache.add(theCacheEntry);
1138
1139            if (wasAdded) {
1140                theCacheEntry.setVariationData(variation, list.m_map);
1141                list.m_map.put(variation, theCacheEntry);
1142                m_keyCache.put(key.getResource(), list);
1143            }
1144        }
1145
1146        if (LOG.isDebugEnabled()) {
1147            LOG.debug(
1148                Messages.get().getBundle().key(
1149                    Messages.LOG_FLEXCACHE_ADDED_ENTRY_FOR_RESOURCE_WITH_VARIATION_3,
1150                    Integer.valueOf(m_size),
1151                    key.getResource(),
1152                    variation));
1153            LOG.debug(Messages.get().getBundle().key(Messages.LOG_FLEXCACHE_ADDED_ENTRY_1, theCacheEntry.toString()));
1154        }
1155    }
1156}