001/*
002 * This library is part of OpenCms -
003 * the Open Source Content Management System
004 *
005 * Copyright (c) Alkacon Software GmbH & Co. KG (https://www.alkacon.com)
006 *
007 * This library is free software; you can redistribute it and/or
008 * modify it under the terms of the GNU Lesser General Public
009 * License as published by the Free Software Foundation; either
010 * version 2.1 of the License, or (at your option) any later version.
011 *
012 * This library is distributed in the hope that it will be useful,
013 * but WITHOUT ANY WARRANTY; without even the implied warranty of
014 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
015 * Lesser General Public License for more details.
016 *
017 * For further information about Alkacon Software, please see the
018 * company website: https://www.alkacon.com
019 *
020 * For further information about OpenCms, please see the
021 * project website: https://www.opencms.org
022 *
023 * You should have received a copy of the GNU Lesser General Public
024 * License along with this library; if not, write to the Free Software
025 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
026 */
027
028package org.opencms.file.quota;
029
030import org.opencms.db.CmsDriverManager;
031import org.opencms.db.CmsPublishedResource;
032import org.opencms.file.CmsObject;
033import org.opencms.file.CmsResource;
034import org.opencms.main.CmsEvent;
035import org.opencms.main.CmsException;
036import org.opencms.main.CmsLog;
037import org.opencms.main.I_CmsEventListener;
038import org.opencms.main.OpenCms;
039import org.opencms.util.CmsCollectionsGenericWrapper;
040import org.opencms.util.CmsUUID;
041
042import java.util.ArrayList;
043import java.util.Collection;
044import java.util.Collections;
045import java.util.HashSet;
046import java.util.List;
047import java.util.Map;
048import java.util.Set;
049import java.util.concurrent.ExecutionException;
050import java.util.concurrent.LinkedBlockingQueue;
051import java.util.concurrent.TimeUnit;
052
053import org.apache.commons.logging.Log;
054
055import com.google.common.base.Functions;
056import com.google.common.cache.CacheBuilder;
057import com.google.common.cache.CacheLoader;
058import com.google.common.cache.LoadingCache;
059
060/**
061 * Maintains folder size information for the system and updates it regularly.
062 *
063 * <p>The folder size information is updated asynchronously and with a delay, so it is not necessarily 100% exact at any particular time.
064 */
065public class CmsFolderSizeTracker {
066
067    /** Default interval for the update timer (in ms). */
068    public static final long DEFAULT_TIMER_INTERVAL = 30000;
069
070    /** The logger instance for the class. */
071    private static final Log LOG = CmsLog.getLog(CmsFolderSizeTracker.class);
072
073    /** The CMS context. */
074    private CmsObject m_cms;
075
076    /** True if object has been initialized. */
077    private boolean m_initialized;
078
079    /** Used to synchronize access to the internal state. */
080    private Object m_lock = new Object();
081
082    /** A read-only copy of the folder size information that is used for normal read operations. */
083    private volatile CmsFolderSizeTable m_table;
084
085    /** Set of paths that still need to be processed. */
086    private LinkedBlockingQueue<String> m_todo = new LinkedBlockingQueue<>();
087
088    /** Timer interval. */
089    private long m_interval;
090
091    /** Just a simple timed cache to save space for commonly used paths in the work queue. */
092    private LoadingCache<String, String> m_pathCache;
093
094    private boolean m_online;
095
096    /**
097     * Creates a new instance.
098     *
099     * @param cms the CMS context
100     * @param true if we want to track folder sizes in the Online project instead of the Offline project
101     */
102    public CmsFolderSizeTracker(CmsObject cms, boolean online) {
103
104        try {
105            m_cms = OpenCms.initCmsObject(cms);
106        } catch (CmsException e) {
107            // shouldn't happen
108            LOG.error(e.getLocalizedMessage(), e);
109        }
110        m_table = new CmsFolderSizeTable(m_cms, online);
111        m_online = online;
112        CacheLoader<String, String> loader = CacheLoader.from(Functions.identity());
113        m_pathCache = CacheBuilder.newBuilder().concurrencyLevel(4).expireAfterAccess(30, TimeUnit.SECONDS).build(
114            loader);
115    }
116
117    /**
118     * Prepares a folder report consisting of subtree sizes for a bunch of folders.
119     *
120     * <p>This is more efficient than querying for folder sizes individually.
121     *
122     * @param folders the folders (list of root paths)
123     * @return the folder report
124     */
125    public Map<String, CmsFolderReportEntry> getFolderReport(Collection<String> folders) {
126
127        if (!m_initialized) {
128            return Collections.emptyMap();
129        }
130
131        return m_table.getFolderReport(folders);
132    }
133
134    /**
135     * Gets the timer interval.
136     *
137     * @return the timer interval
138     */
139    public long getTimerInterval() {
140
141        return m_interval;
142    }
143
144    /**
145     * Gets the total folder size for the complete subtree at the given root path.
146     *
147     * @param rootPath the root path for which to compute the size
148     * @return the total size
149     */
150    public long getTotalFolderSize(String rootPath) {
151
152        if (!m_initialized) {
153            return -1;
154        }
155        return m_table.getTotalFolderSize(rootPath);
156    }
157
158    /**
159     * Gets the folder size for the subtree at the given root path, but without including any folder sizes
160     * of subtrees at any paths from 'otherPaths' of which rootPath is a proper prefix.
161     *
162     * @param rootPath the root path for which to calculate the size
163     * @param otherPaths the other paths to exclude from the size
164     *
165     * @return the total size
166     */
167    public long getTotalFolderSizeExclusive(String rootPath, Collection<String> otherPaths) {
168
169        if (!m_initialized) {
170            return -1;
171        }
172        return m_table.getTotalFolderSizeExclusive(rootPath, otherPaths);
173    }
174
175    /**
176     * Initializes this object (and then returns it).
177     *
178     * @return this instance
179     */
180    public CmsFolderSizeTracker initialize() {
181
182        if (!m_initialized) {
183            Object prop = OpenCms.getRuntimeProperty("folderSizeTrackerInterval");
184            m_interval = DEFAULT_TIMER_INTERVAL;
185            if (prop != null) {
186                try {
187                    m_interval = Long.parseLong("" + prop);
188                } catch (Exception e) {
189                    LOG.error(e.getLocalizedMessage(), e);
190                }
191            }
192            if (m_interval > 0) {
193                reload();
194                OpenCms.getEventManager().addCmsEventListener(this::handleEvent);
195                OpenCms.getExecutor().scheduleWithFixedDelay(
196                    this::processUpdates,
197                    m_interval,
198                    m_interval,
199                    TimeUnit.MILLISECONDS);
200
201                // Just in case something gets corrupted - reload every day
202                OpenCms.getExecutor().scheduleWithFixedDelay(this::reload, 24, 24, TimeUnit.HOURS);
203
204                m_initialized = true;
205            }
206        }
207        return this;
208
209    }
210
211    /**
212     * The scheduled task.
213     */
214    public void processUpdates() {
215
216        long start = System.currentTimeMillis();
217        try {
218            synchronized (m_lock) {
219                Set<String> paths = new HashSet<>();
220                m_todo.drainTo(paths);
221                LOG.debug("Processing path update set of size " + paths.size());
222                if (LOG.isTraceEnabled()) {
223                    LOG.trace("Update set: " + paths);
224                }
225                if (paths.size() > 0) {
226                    CmsFolderSizeTable newTable = new CmsFolderSizeTable(m_table);
227                    for (String path : paths) {
228                        newTable.updateSingle(path);
229                    }
230                    newTable.updateSubtreeCache();
231                    m_table = newTable;
232                }
233            }
234        } catch (Exception e) {
235            LOG.error(e.getLocalizedMessage(), e);
236        }
237        long end = System.currentTimeMillis();
238        long duration = end - start;
239        if (LOG.isDebugEnabled() && (duration > 250)) {
240            LOG.debug("folder size tracker update took " + duration + "ms");
241        }
242    }
243
244    /**
245     * Refreshes the data for a particular subtree.
246     *
247     * @param rootPath the root path to refresh the data for
248     */
249    public void refresh(String rootPath) {
250
251        synchronized (m_lock) {
252            try {
253                CmsFolderSizeTable newTable = new CmsFolderSizeTable(m_table);
254                newTable.updateTree(rootPath);
255                newTable.updateSubtreeCache();
256                m_table = newTable;
257            } catch (CmsException e) {
258                LOG.error(e.getLocalizedMessage(), e);
259            }
260        }
261    }
262
263    /**
264     * Reloads the complete folder size information (this is expensive!).
265     */
266    public void reload() {
267
268        synchronized (m_lock) {
269            try {
270                CmsFolderSizeTable newTable = new CmsFolderSizeTable(m_table);
271                newTable.loadAll();
272                newTable.updateSubtreeCache();
273                m_table = newTable;
274            } catch (CmsException e) {
275                LOG.error(e.getLocalizedMessage(), e);
276            }
277        }
278    }
279
280    /**
281     * Adds a modified folder path to be processed.
282     * @param parentFolder the folder path
283     */
284    private void addPath(String parentFolder) {
285
286        try {
287            m_todo.add(m_pathCache.get(parentFolder));
288        } catch (ExecutionException e) {
289            // can't happen
290            LOG.error(e.getLocalizedMessage(), e);
291        }
292
293    }
294
295    /**
296     * Adds a resource that needs to be processed.
297     *
298     * @param resource the resource to add
299     */
300    private void addUpdate(CmsPublishedResource resource) {
301
302        if (resource.isFile()) {
303            addPath(CmsResource.getParentFolder(resource.getRootPath()));
304        } else {
305            addPath(resource.getRootPath());
306        }
307    }
308
309    /**
310     * Adds a resource that needs to be processed
311     *
312     * @param resource the resource to add
313     */
314    private void addUpdate(CmsResource resource) {
315
316        if (resource.isFile()) {
317            addPath(CmsResource.getParentFolder(resource.getRootPath()));
318        } else {
319            addPath(resource.getRootPath());
320        }
321    }
322
323    /**
324     * Handles CMS events.
325     *
326     * @param event the event to process
327     */
328    private void handleEvent(CmsEvent event) {
329
330        CmsResource resource = null;
331        List<CmsResource> resources = null;
332        if (m_online) {
333            switch (event.getType()) {
334                case I_CmsEventListener.EVENT_PUBLISH_PROJECT:
335                    String publishIdStr = (String)event.getData().get(I_CmsEventListener.KEY_PUBLISHID);
336                    if (publishIdStr != null) {
337                        CmsUUID publishId = new CmsUUID(publishIdStr);
338                        try {
339                            List<CmsPublishedResource> publishedResources = m_cms.readPublishedResources(publishId);
340                            for (CmsPublishedResource res : publishedResources) {
341                                addUpdate(res);
342                            }
343                        } catch (CmsException e) {
344                            LOG.error(e.getLocalizedMessage(), e);
345                        }
346                    }
347                    break;
348                default:
349                    // do nothing
350                    break;
351            }
352        } else {
353            List<Object> irrelevantChangeTypes = new ArrayList<Object>();
354            irrelevantChangeTypes.add(Integer.valueOf(CmsDriverManager.NOTHING_CHANGED));
355            irrelevantChangeTypes.add(Integer.valueOf(CmsDriverManager.CHANGED_PROJECT));
356            switch (event.getType()) {
357                case I_CmsEventListener.EVENT_RESOURCE_AND_PROPERTIES_MODIFIED:
358                case I_CmsEventListener.EVENT_RESOURCE_MODIFIED:
359                case I_CmsEventListener.EVENT_RESOURCE_CREATED:
360                    Object change = event.getData().get(I_CmsEventListener.KEY_CHANGE);
361                    if ((change != null) && irrelevantChangeTypes.contains(change)) {
362                        return;
363                    }
364                    resource = (CmsResource)event.getData().get(I_CmsEventListener.KEY_RESOURCE);
365                    addUpdate(resource);
366                    break;
367                case I_CmsEventListener.EVENT_RESOURCES_AND_PROPERTIES_MODIFIED:
368                    resources = CmsCollectionsGenericWrapper.list(
369                        event.getData().get(I_CmsEventListener.KEY_RESOURCES));
370                    for (CmsResource res : resources) {
371                        addUpdate(res);
372                    }
373                    break;
374
375                case I_CmsEventListener.EVENT_RESOURCE_MOVED:
376                    resources = CmsCollectionsGenericWrapper.list(
377                        event.getData().get(I_CmsEventListener.KEY_RESOURCES));
378                    // source, source folder, dest, dest folder
379                    // - OR -
380                    // source, dest, dest folder
381                    addUpdate(resources.get(0));
382                    addUpdate(resources.get(resources.size() - 2));
383                    break;
384
385                case I_CmsEventListener.EVENT_RESOURCE_DELETED:
386                    resources = CmsCollectionsGenericWrapper.list(
387                        event.getData().get(I_CmsEventListener.KEY_RESOURCES));
388                    for (CmsResource res : resources) {
389                        addUpdate(res);
390                    }
391                    break;
392                case I_CmsEventListener.EVENT_RESOURCES_MODIFIED:
393                    resources = CmsCollectionsGenericWrapper.list(
394                        event.getData().get(I_CmsEventListener.KEY_RESOURCES));
395                    for (CmsResource res : resources) {
396                        addUpdate(res);
397                    }
398                    break;
399                case I_CmsEventListener.EVENT_PUBLISH_PROJECT:
400                    String publishIdStr = (String)event.getData().get(I_CmsEventListener.KEY_PUBLISHID);
401                    if (publishIdStr != null) {
402                        CmsUUID publishId = new CmsUUID(publishIdStr);
403                        try {
404                            List<CmsPublishedResource> publishedResources = m_cms.readPublishedResources(publishId);
405                            for (CmsPublishedResource res : publishedResources) {
406                                addUpdate(res);
407                            }
408                        } catch (CmsException e) {
409                            LOG.error(e.getLocalizedMessage(), e);
410                        }
411                    }
412                    break;
413                default:
414                    // do nothing
415                    break;
416            }
417        }
418    }
419
420}