001/*
002 * This library is part of OpenCms -
003 * the Open Source Content Management System
004 *
005 * Copyright (c) Alkacon Software GmbH & Co. KG (http://www.alkacon.com)
006 *
007 * This library is free software; you can redistribute it and/or
008 * modify it under the terms of the GNU Lesser General Public
009 * License as published by the Free Software Foundation; either
010 * version 2.1 of the License, or (at your option) any later version.
011 *
012 * This library is distributed in the hope that it will be useful,
013 * but WITHOUT ANY WARRANTY; without even the implied warranty of
014 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
015 * Lesser General Public License for more details.
016 *
017 * For further information about Alkacon Software GmbH & Co. KG, please see the
018 * company website: http://www.alkacon.com
019 *
020 * For further information about OpenCms, please see the
021 * project website: http://www.opencms.org
022 *
023 * You should have received a copy of the GNU Lesser General Public
024 * License along with this library; if not, write to the Free Software
025 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
026 */
027
028package org.opencms.workplace.tools.database;
029
030import org.opencms.configuration.CmsParameterConfiguration;
031import org.opencms.db.CmsDbIoException;
032import org.opencms.file.CmsFolder;
033import org.opencms.file.CmsObject;
034import org.opencms.file.CmsProperty;
035import org.opencms.file.CmsPropertyDefinition;
036import org.opencms.file.CmsResource;
037import org.opencms.file.CmsResourceFilter;
038import org.opencms.file.types.CmsResourceTypeFolder;
039import org.opencms.file.types.CmsResourceTypeImage;
040import org.opencms.file.types.CmsResourceTypePlain;
041import org.opencms.file.types.CmsResourceTypePointer;
042import org.opencms.file.types.CmsResourceTypeXmlPage;
043import org.opencms.i18n.CmsEncoder;
044import org.opencms.i18n.CmsLocaleManager;
045import org.opencms.i18n.CmsMessageContainer;
046import org.opencms.importexport.CmsImportExportException;
047import org.opencms.loader.CmsLoaderException;
048import org.opencms.loader.CmsResourceManager;
049import org.opencms.lock.CmsLock;
050import org.opencms.lock.CmsLockType;
051import org.opencms.main.CmsException;
052import org.opencms.main.CmsIllegalArgumentException;
053import org.opencms.main.CmsLog;
054import org.opencms.main.OpenCms;
055import org.opencms.relations.CmsLink;
056import org.opencms.report.I_CmsReport;
057import org.opencms.staticexport.CmsLinkManager;
058import org.opencms.staticexport.CmsLinkTable;
059import org.opencms.util.CmsFileUtil;
060import org.opencms.util.CmsStringUtil;
061import org.opencms.xml.page.CmsXmlPage;
062
063import java.io.File;
064import java.io.FileInputStream;
065import java.io.FileOutputStream;
066import java.io.IOException;
067import java.io.InputStream;
068import java.net.MalformedURLException;
069import java.net.URL;
070import java.util.ArrayList;
071import java.util.Enumeration;
072import java.util.HashMap;
073import java.util.HashSet;
074import java.util.Hashtable;
075import java.util.Iterator;
076import java.util.List;
077import java.util.Locale;
078import java.util.Map;
079import java.util.StringTokenizer;
080import java.util.zip.ZipEntry;
081import java.util.zip.ZipInputStream;
082
083import org.apache.commons.fileupload.FileItem;
084import org.apache.commons.logging.Log;
085
086/**
087 * This class implements the HTML->OpenCms Template converter for OpenCms 6.x.<p>
088 *
089 * The HTML files can lay in a directory or in a zip file. The entries in the zip file
090 * are saved temporary in the tmp-directory of the system. Every file is stored into the
091 * correct location in the OpenCms VFS.<p>
092 *
093 *
094 * @since 6.0.0
095 */
096public class CmsHtmlImport {
097
098    /** filename of the meta.properties file. */
099    public static final String META_PROPERTIES = "meta.properties";
100
101    /** The log object for this class. */
102    private static final Log LOG = CmsLog.getLog(CmsHtmlImport.class);
103
104    /** the CmsObject to use. */
105    private CmsObject m_cmsObject;
106
107    /** the destination directory in the OpenCms VFS. */
108    private String m_destinationDir;
109
110    /** the download gallery name. */
111    private String m_downloadGallery;
112
113    /** the element name of the template. */
114    private String m_element;
115
116    /** the end pattern for extracting content. */
117    private String m_endPattern;
118
119    /** HashMap of all known extensions in OpenCms. */
120    private Map m_extensions;
121
122    /** Storage for external links. */
123    private HashSet m_externalLinks;
124
125    /** The file index contains all resource names in the real file system and their renamed ones in the OpenCms VFS. */
126    private HashMap m_fileIndex;
127
128    /** the HTML converter to parse and modify the content. */
129    private CmsHtmlImportConverter m_htmlConverter;
130
131    /** the path of the import temporary file in the "real" file system.*/
132    private String m_httpDir;
133
134    /** the image gallery name. */
135    private String m_imageGallery;
136
137    /** Storage for image alt tags, it is filled by the HtmlConverter each time a new image is found. */
138    private HashMap m_imageInfo;
139
140    /** the input directory in the "real" file system. */
141    private String m_inputDir;
142
143    /** the encoding used for all imported input files. */
144    private String m_inputEncoding;
145
146    /** should broken links be kept. */
147    private boolean m_keepBrokenLinks;
148
149    /** the external link gallery name. */
150    private String m_linkGallery;
151
152    /** the local use for content definition. */
153    private String m_locale;
154
155    /** the overwrite value new resources. */
156    private boolean m_overwrite;
157
158    /** Map with all parents in file system to OpenCms. */
159    private HashMap m_parents;
160
161    /** the report for the output. */
162    private I_CmsReport m_report;
163
164    /** the start pattern for extracting content. */
165    private String m_startPattern;
166
167    /** the template use for all pages. */
168    private String m_template;
169
170    /**
171     * Default Constructor.<p>
172     */
173    public CmsHtmlImport() {
174
175        m_overwrite = true;
176        m_extensions = OpenCms.getResourceManager().getExtensionMapping();
177        m_fileIndex = new HashMap();
178        m_parents = new HashMap();
179        m_imageInfo = new HashMap();
180        m_externalLinks = new HashSet();
181        m_htmlConverter = new CmsHtmlImportConverter(this, false);
182    }
183
184    /**
185     * Creates a new import object for the given cms object.<p>
186     *
187     * @param cms the current cms context
188     */
189    public CmsHtmlImport(CmsObject cms) {
190
191        this();
192        m_cmsObject = cms;
193    }
194
195    /**
196     * This function creates a folder in the temporary-directory.<p>
197     *
198     * @param name the name of the folder
199     *
200     * @return the folder file
201     *
202     * @throws Exception if the folder can not create
203     */
204    public static File createTempFolder(String name) throws Exception {
205
206        File folder = null;
207        folder = File.createTempFile(name, "", null);
208        folder.delete();
209        folder.mkdirs();
210        folder.deleteOnExit();
211        return folder;
212    }
213
214    /**
215     * Calculates an absolute uri from a relative "uri" and the given absolute "baseUri".<p>
216     *
217     * If "uri" is already absolute, it is returned unchanged.
218     * This method also returns "uri" unchanged if it is not well-formed.<p>
219     *
220     * @param relativeUri the relative uri to calculate an absolute uri for
221     * @param baseUri the base uri, this must be an absolute uri
222     * @return an absolute uri calculated from "uri" and "baseUri"
223     */
224    public String getAbsoluteUri(String relativeUri, String baseUri) {
225
226        if ((relativeUri == null) || (relativeUri.charAt(0) == '/') || (relativeUri.startsWith("#"))) {
227
228            return relativeUri;
229        }
230
231        // if we are on a windows system, we must add a ":" in the uri later
232        String windowsAddition = "";
233        if (File.separator.equals("\\")) {
234            windowsAddition = ":";
235        }
236
237        try {
238            URL baseUrl = new URL("file://");
239            URL url = new URL(new URL(baseUrl, "file://" + baseUri), relativeUri);
240            if (url.getQuery() == null) {
241                if (url.getRef() == null) {
242                    return url.getHost() + windowsAddition + url.getPath();
243                } else {
244                    return url.getHost() + windowsAddition + url.getPath() + "#" + url.getRef();
245                }
246            } else {
247                return url.getHost() + windowsAddition + url.getPath() + "?" + url.getQuery();
248            }
249        } catch (MalformedURLException e) {
250            return relativeUri;
251        }
252    }
253
254    /**
255     * Returns the destinationDir.<p>
256     *
257     * @return the destinationDir
258     */
259    public String getDestinationDir() {
260
261        return m_destinationDir;
262    }
263
264    /**
265     * Returns the downloadGallery.<p>
266     *
267     * @return the downloadGallery
268     */
269    public String getDownloadGallery() {
270
271        return m_downloadGallery;
272    }
273
274    /**
275     * Returns the element.<p>
276     *
277     * @return the element
278     */
279    public String getElement() {
280
281        return m_element;
282    }
283
284    /**
285     * Returns the endPattern.<p>
286     *
287     * @return the endPattern
288     */
289    public String getEndPattern() {
290
291        return m_endPattern;
292    }
293
294    /**
295     * Returns the httpDir.<p>
296     *
297     * @return the httpDir
298     */
299    public String getHttpDir() {
300
301        return m_httpDir;
302    }
303
304    /**
305     * Returns the imageGallery.<p>
306     *
307     * @return the imageGallery
308     */
309    public String getImageGallery() {
310
311        return m_imageGallery;
312    }
313
314    /**
315     * Returns the inputDir.<p>
316     *
317     * @return the inputDir
318     */
319    public String getInputDir() {
320
321        return m_inputDir;
322    }
323
324    /**
325     * Returns the inputEncoding.<p>
326     *
327     * @return the inputEncoding
328     */
329    public String getInputEncoding() {
330
331        return m_inputEncoding;
332    }
333
334    /**
335     * Returns the linkGallery.<p>
336     *
337     * @return the linkGallery
338     */
339    public String getLinkGallery() {
340
341        return m_linkGallery;
342    }
343
344    /**
345     * Returns the local.<p>
346     *
347     * @return the local
348     */
349    public String getLocale() {
350
351        return m_locale;
352    }
353
354    /**
355     * Returns the startPattern.<p>
356     *
357     * @return the startPattern
358     */
359    public String getStartPattern() {
360
361        return m_startPattern;
362    }
363
364    /**
365     * Returns the template.<p>
366     *
367     * @return the template
368     */
369    public String getTemplate() {
370
371        return m_template;
372    }
373
374    /**
375     * Returns the keepBrokenLinks.<p>
376     *
377     * @return the keepBrokenLinks
378     */
379    public boolean isKeepBrokenLinks() {
380
381        return m_keepBrokenLinks;
382    }
383
384    /**
385     * Returns the overwrite.<p>
386     *
387     * @return the overwrite
388     */
389    public boolean isOverwrite() {
390
391        return m_overwrite;
392    }
393
394    /**
395     * Sets the cmsObject.<p>
396     *
397     * @param cmsObject the cmsObject to set
398     */
399    public void setCmsObject(CmsObject cmsObject) {
400
401        m_cmsObject = cmsObject;
402    }
403
404    /**
405     * Sets the destinationDir.<p>
406     *
407     * @param destinationDir the destinationDir to set
408     */
409    public void setDestinationDir(String destinationDir) {
410
411        m_destinationDir = destinationDir;
412    }
413
414    /**
415     * Sets the downloadGallery.<p>
416     *
417     * @param downloadGallery the downloadGallery to set
418     */
419    public void setDownloadGallery(String downloadGallery) {
420
421        m_downloadGallery = downloadGallery;
422    }
423
424    /**
425     * Sets the element.<p>
426     *
427     * @param element the element to set
428     */
429    public void setElement(String element) {
430
431        m_element = element;
432    }
433
434    /**
435     * Sets the endPattern.<p>
436     *
437     * @param endPattern the endPattern to set
438     */
439    public void setEndPattern(String endPattern) {
440
441        m_endPattern = endPattern;
442    }
443
444    /**
445     * Sets the httpDir.<p>
446     *
447     * @param httpDir the httpDir to set
448     */
449    public void setHttpDir(String httpDir) {
450
451        m_httpDir = httpDir;
452    }
453
454    /**
455     * Sets the imageGallery.<p>
456     *
457     * @param imageGallery the imageGallery to set
458     */
459    public void setImageGallery(String imageGallery) {
460
461        m_imageGallery = imageGallery;
462    }
463
464    /**
465     * Sets the inputDir.<p>
466     *
467     * @param inputDir the inputDir to set
468     */
469    public void setInputDir(String inputDir) {
470
471        m_inputDir = inputDir;
472    }
473
474    /**
475     * Sets the inputEncoding.<p>
476     *
477     * @param inputEncoding the inputEncoding to set
478     */
479    public void setInputEncoding(String inputEncoding) {
480
481        m_inputEncoding = inputEncoding;
482    }
483
484    /**
485     * Sets the keepBrokenLinks.<p>
486     *
487     * @param keepBrokenLinks the keepBrokenLinks to set
488     */
489    public void setKeepBrokenLinks(boolean keepBrokenLinks) {
490
491        m_keepBrokenLinks = keepBrokenLinks;
492    }
493
494    /**
495     * Sets the linkGallery.<p>
496     *
497     * @param linkGallery the linkGallery to set
498     */
499    public void setLinkGallery(String linkGallery) {
500
501        m_linkGallery = linkGallery;
502    }
503
504    /**
505     * Sets the local.<p>
506     *
507     * @param locale the local to set
508     */
509    public void setLocale(String locale) {
510
511        m_locale = locale;
512    }
513
514    /**
515     * Sets the overwrite.<p>
516     *
517     * @param overwrite the overwrite to set
518     */
519    public void setOverwrite(boolean overwrite) {
520
521        m_overwrite = overwrite;
522    }
523
524    /**
525     * Sets the startPattern.<p>
526     *
527     * @param startPattern the startPattern to set
528     */
529    public void setStartPattern(String startPattern) {
530
531        m_startPattern = startPattern;
532    }
533
534    /**
535     * Sets the template.<p>
536     *
537     * @param template the template to set
538     */
539    public void setTemplate(String template) {
540
541        m_template = template;
542    }
543
544    /**
545     * Imports all resources from the real file system, stores them into the correct locations
546     * in the OpenCms VFS and modifies all links. This method is called form the JSP to start the
547     * import process.<p>
548     *
549     * @param report StringBuffer for reporting
550     *
551     * @throws Exception if something goes wrong
552     */
553    public void startImport(I_CmsReport report) throws Exception {
554
555        try {
556            m_report = report;
557            m_report.println(Messages.get().container(Messages.RPT_HTML_IMPORT_BEGIN_0), I_CmsReport.FORMAT_HEADLINE);
558
559            boolean isStream = !CmsStringUtil.isEmptyOrWhitespaceOnly(m_httpDir);
560            File streamFolder = null;
561            if (isStream) {
562                // the input is starting through the HTTP upload
563                streamFolder = unzipStream();
564                m_inputDir = streamFolder.getAbsolutePath();
565            }
566
567            // first build the index of all resources
568            buildIndex(m_inputDir);
569            // build list with all parent resources of input directory for links to outside import folder
570            buildParentPath();
571            // copy and parse all HTML files first. during the copy process we will collect all
572            // required data for downloads and images
573            copyHtmlFiles(m_inputDir);
574            // now copy the other files
575            copyOtherFiles(m_inputDir);
576            // finally create all the external links
577            createExternalLinks();
578
579            if (isStream && (streamFolder != null)) {
580                m_report.println(Messages.get().container(Messages.RPT_HTML_DELETE_0), I_CmsReport.FORMAT_NOTE);
581                // delete the files of the zip file
582                CmsFileUtil.purgeDirectory(streamFolder);
583                // deletes the zip file
584                File file = new File(m_httpDir);
585                if (file.exists() && file.canWrite()) {
586                    file.delete();
587                }
588            }
589            m_report.println(Messages.get().container(Messages.RPT_HTML_IMPORT_END_0), I_CmsReport.FORMAT_HEADLINE);
590        } catch (Exception e) {
591            e.printStackTrace();
592        }
593    }
594
595    /**
596     * Add a new external link to the storage of external links.<p>
597     *
598     * All links in this storage are later used to create entries in the external link gallery.<p>
599     *
600     * @param externalLink link to an external resource
601     *
602     * @return the complete path to the external link file, if one is created.
603     */
604    public String storeExternalLink(String externalLink) {
605
606        if (!CmsStringUtil.isEmptyOrWhitespaceOnly(m_linkGallery)) {
607            m_externalLinks.add(externalLink);
608            return getExternalLinkFile(externalLink);
609        }
610
611        return null;
612    }
613
614    /**
615     * Add a new image info to the storage of image info's.<p>
616     *
617     * The image info's are later used to set the description properties of the images.<p>
618     *
619     * @param image the name of the image
620     * @param altText the alt-text of the image
621     */
622    public void storeImageInfo(String image, String altText) {
623
624        m_imageInfo.put(image, altText);
625    }
626
627    /**
628     * Translated a link into the real file system to its new location in the OpenCms VFS.<p>
629     *
630     * This is needed by the HtmlConverter to get the correct links for link translation.<p>
631     *
632     * @param link link to the real file system
633     *
634     * @return string containing absolute link into the OpenCms VFS
635     */
636    public String translateLink(String link) {
637
638        String translatedLink = null;
639        translatedLink = (String)m_fileIndex.get(link.replace('\\', '/'));
640
641        if (translatedLink == null) {
642            // its an anchor link, so copy use it
643            if (link.startsWith("#")) {
644                translatedLink = link;
645            }
646
647            // relative link to OpenCms root
648            else if (link.startsWith("/")) {
649
650                // strip cms context path
651                link = CmsLinkManager.removeOpenCmsContext(link);
652
653                // check if resource exists
654                if ((m_keepBrokenLinks) || (m_cmsObject.existsResource(link))) {
655                    translatedLink = link;
656                }
657            }
658
659            else {
660
661                String fileBase = getBasePath(m_inputDir, link);
662                String cmsBase = (String)m_parents.get(fileBase);
663                if (cmsBase != null) {
664                    String outLink = cmsBase + link.substring(fileBase.length()).replace('\\', '/');
665                    if ((m_keepBrokenLinks) || (m_cmsObject.existsResource(outLink))) {
666                        translatedLink = outLink;
667                    }
668                }
669            }
670        }
671
672        // if the link goes to a directory, lets link to the index page within
673        if ((translatedLink != null) && translatedLink.endsWith("/")) {
674            translatedLink += "index.html";
675        }
676
677        // final check: if the translated link is still null the original link found
678        // was broken
679        // lets link it to the same page, the link is found on
680        if (translatedLink == null) {
681            translatedLink = "#";
682        }
683
684        return translatedLink;
685    }
686
687    /**
688     * Tests if all given input parameters for the HTML Import are valid, that is that all the
689     * given folders do exist. <p>
690     *
691     * @param fi a file item if a file is uploaded per HTTP otherwise <code>null</code>
692     * @param isdefault if this sets, then the destination and input directory can be empty
693     *
694     * @throws CmsIllegalArgumentException if some parameters are not valid
695     */
696    public void validate(FileItem fi, boolean isdefault) throws CmsIllegalArgumentException {
697
698        // check the input directory and the HTTP upload file
699        if (fi == null) {
700
701            if (CmsStringUtil.isEmptyOrWhitespaceOnly(m_inputDir) && !isdefault) {
702                throw new CmsIllegalArgumentException(
703                    Messages.get().container(Messages.GUI_HTMLIMPORT_INPUTDIR_1, m_inputDir));
704            } else if (!CmsStringUtil.isEmptyOrWhitespaceOnly(m_inputDir)) {
705
706                File inputDir = new File(m_inputDir);
707                if (!inputDir.exists() || inputDir.isFile()) {
708                    throw new CmsIllegalArgumentException(
709                        Messages.get().container(Messages.GUI_HTMLIMPORT_INPUTDIR_1, m_inputDir));
710                }
711            }
712        }
713
714        // check the destination directory
715        try {
716            if (CmsStringUtil.isEmptyOrWhitespaceOnly(m_destinationDir) && !isdefault) {
717                throw new CmsIllegalArgumentException(
718                    Messages.get().container(Messages.GUI_HTMLIMPORT_DESTDIR_1, m_destinationDir));
719            } else if (!CmsStringUtil.isEmptyOrWhitespaceOnly(m_destinationDir)) {
720                m_cmsObject.readFolder(m_destinationDir);
721            }
722
723        } catch (CmsException e) {
724            // an exception is thrown if the folder does not exist
725            throw new CmsIllegalArgumentException(
726                Messages.get().container(Messages.GUI_HTMLIMPORT_DESTDIR_1, m_destinationDir),
727                e);
728        }
729
730        // check the image gallery
731        // only if flag for leaving images at original location is off
732        if (!CmsStringUtil.isEmptyOrWhitespaceOnly(m_imageGallery)) {
733            try {
734                CmsFolder folder = m_cmsObject.readFolder(m_imageGallery);
735                // check if folder is a image gallery
736                String name = OpenCms.getResourceManager().getResourceType(folder.getTypeId()).getTypeName();
737                if (!name.equals("imagegallery")) {
738                    throw new CmsIllegalArgumentException(
739                        Messages.get().container(Messages.GUI_HTMLIMPORT_IMGGALLERY_INVALID_1, m_imageGallery));
740                }
741            } catch (CmsException e) {
742                // an exception is thrown if the folder does not exist
743                throw new CmsIllegalArgumentException(
744                    Messages.get().container(Messages.GUI_HTMLIMPORT_IMGGALLERY_1, m_imageGallery),
745                    e);
746            }
747        }
748
749        // check the link gallery
750        // only if flag for leaving external links at original location is off
751        if (!CmsStringUtil.isEmptyOrWhitespaceOnly(m_linkGallery)) {
752            try {
753                CmsFolder folder = m_cmsObject.readFolder(m_linkGallery);
754                // check if folder is a link gallery
755                String name = OpenCms.getResourceManager().getResourceType(folder.getTypeId()).getTypeName();
756                if (!name.equals("linkgallery")) {
757                    throw new CmsIllegalArgumentException(
758                        Messages.get().container(Messages.GUI_HTMLIMPORT_LINKGALLERY_INVALID_1, m_linkGallery));
759                }
760            } catch (CmsException e) {
761                // an exception is thrown if the folder does not exist
762                throw new CmsIllegalArgumentException(
763                    Messages.get().container(Messages.GUI_HTMLIMPORT_LINKGALLERY_1, m_linkGallery),
764                    e);
765            }
766        }
767
768        // check the download gallery
769        if ((!isExternal(m_downloadGallery)) && (!CmsStringUtil.isEmptyOrWhitespaceOnly(m_downloadGallery))) {
770            try {
771                CmsFolder folder = m_cmsObject.readFolder(m_downloadGallery);
772                // check if folder is a download gallery
773                String name = OpenCms.getResourceManager().getResourceType(folder.getTypeId()).getTypeName();
774                if (!name.equals("downloadgallery")) {
775                    throw new CmsIllegalArgumentException(
776                        Messages.get().container(Messages.GUI_HTMLIMPORT_DOWNGALLERY_INVALID_1, m_downloadGallery));
777                }
778            } catch (CmsException e) {
779                // an exception is thrown if the folder does not exist
780                throw new CmsIllegalArgumentException(
781                    Messages.get().container(Messages.GUI_HTMLIMPORT_DOWNGALLERY_1, m_downloadGallery),
782                    e);
783            }
784        }
785
786        // check the template
787        try {
788            m_cmsObject.readResource(m_template, CmsResourceFilter.ALL);
789        } catch (CmsException e) {
790            // an exception is thrown if the template does not exist
791            if (!isValidElement()) {
792                throw new CmsIllegalArgumentException(
793                    Messages.get().container(Messages.GUI_HTMLIMPORT_TEMPLATE_1, m_template),
794                    e);
795            }
796        }
797
798        // check the element
799        if (!isValidElement()) {
800            throw new CmsIllegalArgumentException(
801                Messages.get().container(Messages.GUI_HTMLIMPORT_INVALID_ELEM_2, m_element, m_template));
802        }
803
804        // check if we are in an offline project
805        if (m_cmsObject.getRequestContext().getCurrentProject().isOnlineProject()) {
806            throw new CmsIllegalArgumentException(
807                Messages.get().container(Messages.GUI_HTMLIMPORT_CONSTRAINT_OFFLINE_0));
808        }
809    }
810
811    /**
812     * Builds an index of all files to be imported and determines their new names in the OpenCms.<p>
813     *
814     * @param startfolder the folder to start with
815     *
816     * @throws Exception if something goes wrong
817     */
818    private void buildIndex(String startfolder) throws Exception {
819
820        File folder = new File(startfolder);
821        // get all subresources
822
823        File[] subresources = folder.listFiles();
824        // now loop through all subresources and add them to the index list
825        for (int i = 0; i < subresources.length; i++) {
826            try {
827
828                String relativeFSName = subresources[i].getAbsolutePath().substring(m_inputDir.length() + 1);
829                String absoluteVFSName = getVfsName(
830                    relativeFSName,
831                    subresources[i].getName(),
832                    subresources[i].isFile());
833                m_report.print(Messages.get().container(Messages.RPT_CREATE_INDEX_0), I_CmsReport.FORMAT_NOTE);
834                m_report.print(
835                    org.opencms.report.Messages.get().container(
836                        org.opencms.report.Messages.RPT_ARGUMENT_1,
837                        relativeFSName.replace('\\', '/')));
838                m_report.print(org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_DOTS_0));
839                m_report.print(Messages.get().container(Messages.RPT_ARROW_RIGHT_0), I_CmsReport.FORMAT_NOTE);
840                m_report.print(
841                    org.opencms.report.Messages.get().container(
842                        org.opencms.report.Messages.RPT_ARGUMENT_1,
843                        absoluteVFSName));
844                m_report.print(org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_DOTS_0));
845                m_fileIndex.put(subresources[i].getAbsolutePath().replace('\\', '/'), absoluteVFSName);
846                // if the subresource is a folder, get all subresources of it as well
847                if (subresources[i].isDirectory()) {
848                    buildIndex(subresources[i].getAbsolutePath());
849                }
850                m_report.println(
851                    org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_OK_0),
852                    I_CmsReport.FORMAT_OK);
853            } catch (Exception e) {
854                LOG.error(e.getLocalizedMessage(), e);
855                m_report.println(e);
856            }
857        }
858    }
859
860    /**
861     * Builds a map with all parents of the destination directory to the real file system.<p>
862     * So links to resources of outside the import folder can be found.<p>
863     */
864    private void buildParentPath() {
865
866        String destFolder = m_destinationDir;
867        String inputDir = m_inputDir.replace('\\', '/');
868        if (!inputDir.endsWith("/")) {
869            inputDir += "/";
870        }
871        int pos = inputDir.lastIndexOf("/");
872        while ((pos > 0) && (destFolder != null)) {
873            inputDir = inputDir.substring(0, pos);
874            m_parents.put(inputDir + "/", destFolder);
875
876            pos = inputDir.lastIndexOf("/", pos - 1);
877            destFolder = CmsResource.getParentFolder(destFolder);
878        }
879    }
880
881    /**
882     * This function close a InputStream.<p>
883     *
884     * @param stream the <code> {@link InputStream} </code> Object
885     */
886    private void closeStream(InputStream stream) {
887
888        if (stream != null) {
889            try {
890                stream.close();
891            } catch (Exception ex) {
892                LOG.error(ex.getLocalizedMessage(), ex);
893            }
894        }
895    }
896
897    /**
898     * Copies all  HTML files to the VFS.<p>
899     *
900     * @param startfolder the folder to start with
901     *
902     * @throws Exception if something goes wrong
903     */
904    private void copyHtmlFiles(String startfolder) throws Exception {
905
906        try {
907            File folder = new File(startfolder);
908            // get all subresources
909            File[] subresources = folder.listFiles();
910            int plainId = OpenCms.getResourceManager().getResourceType(
911                CmsResourceTypePlain.getStaticTypeName()).getTypeId();
912            // now loop through all subresources
913            for (int i = 0; i < subresources.length; i++) {
914                // if the subresource is a folder, get all subresources of it as well
915                if (subresources[i].isDirectory()) {
916                    // first, create the folder in the VFS
917                    Hashtable properties = new Hashtable();
918                    createFolder(subresources[i].getAbsolutePath(), i, properties);
919                    // now process all rescources inside of the folder
920                    copyHtmlFiles(subresources[i].getAbsolutePath());
921                } else {
922                    // create a new file in the VFS
923                    String vfsFileName = (String)m_fileIndex.get(subresources[i].getAbsolutePath().replace('\\', '/'));
924                    // check if this is an HTML file, do only import and parse those
925                    int type = getFileType(vfsFileName);
926                    if (plainId == type) {
927                        Hashtable properties = new Hashtable();
928                        // the subresource is a file, so start the parsing process
929                        String content = "";
930                        try {
931                            content = parseHtmlFile(subresources[i], properties);
932                        } catch (CmsException e) {
933                            m_report.println(e);
934                        }
935                        properties.put("template", m_template);
936
937                        // create the file in the VFS
938                        createFile(subresources[i].getAbsolutePath(), i, content, properties);
939                    }
940                }
941            }
942        } catch (Exception e) {
943            LOG.error(e.getLocalizedMessage(), e);
944        }
945    }
946
947    /**
948     * Copies all files except HTML files to the VFS.<p>
949     *
950     * @param startfolder the folder to start with
951     */
952    private void copyOtherFiles(String startfolder) {
953
954        try {
955            File folder = new File(startfolder);
956            // get all subresources
957            File[] subresources = folder.listFiles();
958            int plainId = OpenCms.getResourceManager().getResourceType(
959                CmsResourceTypePlain.getStaticTypeName()).getTypeId();
960            // now loop through all subresources
961            for (int i = 0; i < subresources.length; i++) {
962                // if the subresource is a folder, get all subresources of it as well
963                if (subresources[i].isDirectory()) {
964                    copyOtherFiles(subresources[i].getAbsolutePath());
965                } else {
966                    // do not import the "meta.properties" file
967                    if (!subresources[i].getName().equals(META_PROPERTIES)) {
968                        // create a new file in the VFS
969                        String vfsFileName = (String)m_fileIndex.get(
970                            subresources[i].getAbsolutePath().replace('\\', '/'));
971                        // get the file type of the FS file
972                        int type = getFileType(vfsFileName);
973                        if (plainId != type) {
974                            if (isExternal(vfsFileName)) {
975                                m_report.print(
976                                    Messages.get().container(Messages.RPT_SKIP_EXTERNAL_0),
977                                    I_CmsReport.FORMAT_NOTE);
978                                m_report.print(
979                                    org.opencms.report.Messages.get().container(
980                                        org.opencms.report.Messages.RPT_ARGUMENT_1,
981                                        subresources[i]));
982                                m_report.print(
983                                    org.opencms.report.Messages.get().container(
984                                        org.opencms.report.Messages.RPT_DOTS_0));
985                                m_report.print(
986                                    Messages.get().container(Messages.RPT_ARROW_RIGHT_0),
987                                    I_CmsReport.FORMAT_NOTE);
988                                m_report.println(
989                                    org.opencms.report.Messages.get().container(
990                                        org.opencms.report.Messages.RPT_ARGUMENT_1,
991                                        vfsFileName));
992                            } else {
993
994                                m_report.print(
995                                    Messages.get().container(Messages.RPT_IMPORT_0),
996                                    I_CmsReport.FORMAT_NOTE);
997                                m_report.print(
998                                    org.opencms.report.Messages.get().container(
999                                        org.opencms.report.Messages.RPT_ARGUMENT_1,
1000                                        vfsFileName));
1001                                m_report.print(
1002                                    org.opencms.report.Messages.get().container(
1003                                        org.opencms.report.Messages.RPT_DOTS_0));
1004
1005                                // get the content of the FS file
1006                                byte[] content = getFileBytes(subresources[i]);
1007                                // get the filename from the fileIndex list
1008
1009                                // check if there are some image info's stored for this resource
1010                                List properties = new ArrayList();
1011                                String altText = (String)m_imageInfo.get(
1012                                    subresources[i].getAbsolutePath().replace('\\', '/'));
1013                                CmsProperty property1 = new CmsProperty(
1014                                    CmsPropertyDefinition.PROPERTY_DESCRIPTION,
1015                                    altText,
1016                                    altText);
1017                                CmsProperty property2 = new CmsProperty(
1018                                    CmsPropertyDefinition.PROPERTY_TITLE,
1019                                    altText,
1020                                    altText);
1021                                // add them to the title and description property
1022                                if (altText != null) {
1023                                    properties.add(property1);
1024                                    properties.add(property2);
1025                                }
1026                                // create the file
1027                                if (!m_overwrite) {
1028                                    m_cmsObject.createResource(vfsFileName, type, content, properties);
1029                                } else {
1030                                    try {
1031                                        CmsLock lock = m_cmsObject.getLock(vfsFileName);
1032                                        if (lock.getType() != CmsLockType.EXCLUSIVE) {
1033                                            m_cmsObject.lockResource(vfsFileName);
1034                                        }
1035                                        m_cmsObject.deleteResource(vfsFileName, CmsResource.DELETE_PRESERVE_SIBLINGS);
1036                                    } catch (CmsException e) {
1037                                        // the file did not exist, so create it
1038                                    } finally {
1039                                        m_cmsObject.createResource(vfsFileName, type, content, properties);
1040                                    }
1041
1042                                    m_report.print(
1043                                        Messages.get().container(Messages.RPT_OVERWRITE_0),
1044                                        I_CmsReport.FORMAT_NOTE);
1045                                    m_report.print(
1046                                        org.opencms.report.Messages.get().container(
1047                                            org.opencms.report.Messages.RPT_DOTS_0));
1048                                }
1049                                m_report.println(
1050                                    org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_OK_0),
1051                                    I_CmsReport.FORMAT_OK);
1052                            }
1053                        }
1054                    }
1055                }
1056            }
1057        } catch (Exception e) {
1058            LOG.error(e.getLocalizedMessage(), e);
1059            m_report.println(e);
1060        }
1061    }
1062
1063    /**
1064     * Creates all external links, which were found during the HTML-page processing.<p>
1065     *
1066     */
1067    private void createExternalLinks() {
1068
1069        int pointerId;
1070        try {
1071            pointerId = OpenCms.getResourceManager().getResourceType(
1072                CmsResourceTypePointer.getStaticTypeName()).getTypeId();
1073        } catch (CmsLoaderException e) {
1074            // should not never ever happen
1075            pointerId = CmsResourceTypePointer.getStaticTypeId();
1076        }
1077        // loop through all links
1078        Iterator i = m_externalLinks.iterator();
1079        while (i.hasNext()) {
1080            String linkUrl = (String)i.next();
1081            String filename = getExternalLinkFile(linkUrl);
1082
1083            m_report.print(Messages.get().container(Messages.RPT_CREATE_EXTERNAL_LINK_0), I_CmsReport.FORMAT_NOTE);
1084            m_report.print(
1085                org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_ARGUMENT_1, filename));
1086            m_report.print(org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_DOTS_0));
1087
1088            List properties = new ArrayList();
1089            CmsProperty property1 = new CmsProperty(
1090                CmsPropertyDefinition.PROPERTY_TITLE,
1091                "Link to " + linkUrl,
1092                "Link to " + linkUrl);
1093            properties.add(property1);
1094            try {
1095                m_cmsObject.createResource(m_linkGallery + filename, pointerId, linkUrl.getBytes(), properties);
1096            } catch (CmsException e) {
1097                // do nothing here, an exception will be thrown if this link already exists
1098            }
1099            m_report.println(
1100                org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_OK_0),
1101                I_CmsReport.FORMAT_OK);
1102        }
1103    }
1104
1105    /**
1106     * Creates a file in the VFS.<p>
1107     *
1108     * @param filename the complete filename in the real file system
1109     * @param position the default navigation position of this folder
1110     * @param content the HTML content of the file
1111     * @param properties the file properties
1112     */
1113    private void createFile(String filename, int position, String content, Hashtable properties) {
1114
1115        String vfsFileName = (String)m_fileIndex.get(filename.replace('\\', '/'));
1116
1117        if (vfsFileName != null) {
1118            try {
1119
1120                m_report.print(Messages.get().container(Messages.RPT_CREATE_FILE_0), I_CmsReport.FORMAT_NOTE);
1121                m_report.print(
1122                    org.opencms.report.Messages.get().container(
1123                        org.opencms.report.Messages.RPT_ARGUMENT_1,
1124                        vfsFileName));
1125                m_report.print(org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_DOTS_0));
1126
1127                // check if we have to set the navpos property.
1128                if ((properties.get(CmsPropertyDefinition.PROPERTY_NAVPOS) == null)
1129                    && (properties.get(CmsPropertyDefinition.PROPERTY_NAVTEXT) != null)) {
1130                    // set the position in the folder as navpos
1131                    // we have to add one to the position, since it is counted from 0
1132                    properties.put(CmsPropertyDefinition.PROPERTY_NAVPOS, (position + 1) + "");
1133                }
1134
1135                // create new XML page
1136                Locale locale = CmsLocaleManager.getLocale(m_locale);
1137                CmsXmlPage page = new CmsXmlPage(locale, OpenCms.getSystemInfo().getDefaultEncoding());
1138                page.addValue(m_element, locale);
1139                page.setStringValue(m_cmsObject, m_element, locale, content);
1140
1141                // check links
1142                CmsLinkTable linkTable = page.getLinkTable(m_element, locale);
1143                Iterator i = linkTable.iterator();
1144                while (i.hasNext()) {
1145                    CmsLink link = (CmsLink)i.next();
1146                    String target = link.getTarget();
1147                    // do only update internal links
1148                    if (link.isInternal()) {
1149                        target = m_cmsObject.getRequestContext().getFileTranslator().translateResource(target);
1150                        // update link
1151                        link.updateLink(target, link.getAnchor(), link.getQuery());
1152                        link.checkConsistency(m_cmsObject);
1153                    }
1154                }
1155                // marshal XML page and get the content
1156                byte[] contentByteArray = page.marshal();
1157                List oldProperties = new ArrayList();
1158
1159                int xmlPageId = OpenCms.getResourceManager().getResourceType(
1160                    CmsResourceTypeXmlPage.getStaticTypeName()).getTypeId();
1161                if (!m_overwrite) {
1162                    m_cmsObject.createResource(vfsFileName, xmlPageId, contentByteArray, new ArrayList());
1163                } else {
1164                    try {
1165                        // try if the file is there
1166                        oldProperties = m_cmsObject.readPropertyObjects(vfsFileName, false);
1167                        CmsLock lock = m_cmsObject.getLock(vfsFileName);
1168                        if (lock.getType() != CmsLockType.EXCLUSIVE) {
1169                            m_cmsObject.lockResource(vfsFileName);
1170                        }
1171                        m_cmsObject.deleteResource(vfsFileName, CmsResource.DELETE_PRESERVE_SIBLINGS);
1172                    } catch (CmsException e) {
1173                        // the file did not exist, so we do not have to delete it
1174                    } finally {
1175                        // create the new resource
1176                        m_report.print(Messages.get().container(Messages.RPT_OVERWRITE_0), I_CmsReport.FORMAT_NOTE);
1177                        m_report.print(
1178                            org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_DOTS_0));
1179                        m_cmsObject.createResource(vfsFileName, xmlPageId, contentByteArray, new ArrayList());
1180                    }
1181                }
1182                // create all properties and put them in an ArrayList
1183                Iterator it = properties.entrySet().iterator();
1184                List propertyList = new ArrayList();
1185                while (it.hasNext()) {
1186                    // get property and value
1187                    Map.Entry entry = (Map.Entry)it.next();
1188                    String propertyKey = (String)entry.getKey();
1189                    String propertyVal = (String)entry.getValue();
1190                    // create new Property Object
1191                    CmsProperty property = new CmsProperty(propertyKey, propertyVal, propertyVal);
1192                    // create implicitly if Property doesn't exist already
1193                    property.setAutoCreatePropertyDefinition(true);
1194                    // add new property to the list
1195                    propertyList.add(property);
1196                }
1197                // try to write the properties
1198                try {
1199                    m_cmsObject.writePropertyObjects(vfsFileName, propertyList);
1200                    // write the old properties if available
1201                    m_cmsObject.writePropertyObjects(vfsFileName, oldProperties);
1202                } catch (CmsException e1) {
1203                    e1.printStackTrace();
1204                }
1205                m_report.println(
1206                    org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_OK_0),
1207                    I_CmsReport.FORMAT_OK);
1208            } catch (CmsException e) {
1209                m_report.println(e);
1210                LOG.error(e.getLocalizedMessage(), e);
1211            }
1212        }
1213    }
1214
1215    /**
1216     * Creates a folder in the VFS.<p>
1217     *
1218     * @param foldername the complete folder name in the real file system
1219     * @param position the default navigation position of this folder
1220     * @param properties the file properties
1221     */
1222    private void createFolder(String foldername, int position, Hashtable properties) {
1223
1224        String vfsFolderName = (String)m_fileIndex.get(foldername.replace('\\', '/'));
1225
1226        m_report.print(Messages.get().container(Messages.RPT_CREATE_FOLDER_0), I_CmsReport.FORMAT_NOTE);
1227        m_report.print(
1228            org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_ARGUMENT_1, vfsFolderName));
1229        m_report.print(org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_DOTS_0));
1230
1231        if (vfsFolderName != null) {
1232            String path = vfsFolderName.substring(
1233                0,
1234                vfsFolderName.substring(0, vfsFolderName.length() - 1).lastIndexOf("/"));
1235            String folder = vfsFolderName.substring(path.length(), vfsFolderName.length());
1236            try {
1237                // try to find a meta.properties file in the folder
1238                String propertyFileName = foldername + File.separator + META_PROPERTIES;
1239
1240                boolean metaPropertiesFound = false;
1241                CmsParameterConfiguration propertyFile = new CmsParameterConfiguration();
1242                try {
1243                    propertyFile.load(new FileInputStream(new File(propertyFileName)));
1244                    metaPropertiesFound = true;
1245                } catch (Exception e1) {
1246                    // do nothing if the property file could not be loaded since it is not required
1247                    // that such s file does exist
1248                }
1249                // now copy all values from the property file to the already found properties of the
1250                // new folder in OpenCms
1251                // only do this if we have found a meta.properties file
1252                if (metaPropertiesFound) {
1253                    properties.putAll(propertyFile);
1254
1255                    // check if we have to set the navpos property.
1256                    if (properties.get(CmsPropertyDefinition.PROPERTY_NAVPOS) == null) {
1257                        // set the position in the folder as navpos
1258                        // we have to add one to the position, since it is counted from 0
1259                        properties.put(CmsPropertyDefinition.PROPERTY_NAVPOS, (position + 1) + "");
1260                    }
1261                    // check if we have to set the navpos property.
1262                    if (properties.get(CmsPropertyDefinition.PROPERTY_NAVTEXT) == null) {
1263                        // set the foldername in the folder as navtext
1264                        String navtext = folder.substring(1, 2).toUpperCase()
1265                            + folder.substring(2, folder.length() - 1);
1266                        properties.put(CmsPropertyDefinition.PROPERTY_NAVTEXT, navtext);
1267                    }
1268                } else {
1269                    // if there was no meta.properties file, no properties should be added to the
1270                    // folder
1271                    properties = new Hashtable();
1272                }
1273                // try to read the folder, it its there we must not create it again
1274                try {
1275                    m_cmsObject.readFolder(path + folder);
1276                    m_cmsObject.lockResource(path + folder);
1277                } catch (CmsException e1) {
1278                    // the folder was not there, so create it
1279                    m_cmsObject.createResource(
1280                        path + folder,
1281                        OpenCms.getResourceManager().getResourceType(
1282                            CmsResourceTypeFolder.getStaticTypeName()).getTypeId());
1283                }
1284                // create all properties and put them in an ArrayList
1285                Enumeration enu = properties.keys();
1286                List propertyList = new ArrayList();
1287                while (enu.hasMoreElements()) {
1288                    // get property and value
1289                    String propertyKey = (String)enu.nextElement();
1290                    String propertyVal = (String)properties.get(propertyKey);
1291                    CmsProperty property = new CmsProperty(propertyKey, propertyVal, propertyVal);
1292                    // create implicitly if Property doesn't exist already
1293                    property.setAutoCreatePropertyDefinition(true);
1294                    // add new property to the list
1295                    propertyList.add(property);
1296                }
1297                // try to write the property Objects
1298                try {
1299                    m_cmsObject.writePropertyObjects(path + folder, propertyList);
1300                } catch (CmsException e1) {
1301                    e1.printStackTrace();
1302                }
1303                m_report.println(
1304                    org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_OK_0),
1305                    I_CmsReport.FORMAT_OK);
1306            } catch (CmsException e) {
1307                m_report.println(e);
1308                LOG.error(e.getLocalizedMessage(), e);
1309            }
1310        }
1311    }
1312
1313    /**
1314     * Compares two path's for the base part which have both equal.<p>
1315     *
1316     * @param path1 the first path to compare
1317     * @param path2 the second path to compare
1318     *
1319     * @return the base path of both which are equal
1320     */
1321    private String getBasePath(String path1, String path2) {
1322
1323        StringBuffer base = new StringBuffer();
1324        path1 = path1.replace('\\', '/');
1325        path2 = path2.replace('\\', '/');
1326
1327        String[] parts1 = path1.split("/");
1328        String[] parts2 = path2.split("/");
1329
1330        for (int i = 0; i < parts1.length; i++) {
1331            if (i >= parts2.length) {
1332                break;
1333            }
1334            if (parts1[i].equals(parts2[i])) {
1335                base.append(parts1[i] + "/");
1336            }
1337        }
1338
1339        return base.toString();
1340    }
1341
1342    /**
1343     * Creates the filename of the file of the external link.<p>
1344     *
1345     * @param link the link to get the file path for.
1346     *
1347     * @return the filename of the file for the external link.
1348     */
1349    private String getExternalLinkFile(String link) {
1350
1351        String filename = link.substring(link.indexOf("://") + 3, link.length());
1352        filename = m_cmsObject.getRequestContext().getFileTranslator().translateResource(filename.replace('/', '-'));
1353        return filename;
1354    }
1355
1356    /**
1357     * Returns a byte array containing the content of server FS file.<p>
1358     *
1359     * @param file the name of the file to read
1360     *
1361     * @return bytes[] the content of the file
1362     *
1363     * @throws CmsException if something goes wrong
1364     */
1365    private byte[] getFileBytes(File file) throws CmsException {
1366
1367        byte[] buffer = null;
1368
1369        FileInputStream fileStream = null;
1370        int charsRead;
1371        int size;
1372        try {
1373            fileStream = new FileInputStream(file);
1374            charsRead = 0;
1375            size = new Long(file.length()).intValue();
1376            buffer = new byte[size];
1377            while (charsRead < size) {
1378                charsRead += fileStream.read(buffer, charsRead, size - charsRead);
1379            }
1380            return buffer;
1381        } catch (IOException e) {
1382            throw new CmsDbIoException(
1383                Messages.get().container(Messages.ERR_GET_FILE_BYTES_1, file.getAbsolutePath()),
1384                e);
1385        } finally {
1386            closeStream(fileStream);
1387        }
1388    }
1389
1390    /**
1391     * Returns the OpenCms file type of a real file system file. <p>
1392     * This is made by checking the extension.<p>
1393     *
1394     * @param filename the name of the file in the real file system
1395     *
1396     * @return the id of the OpenCms file type
1397     *
1398     * @throws Exception if something goes wrong
1399     */
1400    private int getFileType(String filename) throws Exception {
1401
1402        String extension = "";
1403        if (filename.indexOf(".") > -1) {
1404            extension = filename.substring((filename.lastIndexOf(".")));
1405        }
1406
1407        String typename = (String)m_extensions.get(extension.toLowerCase());
1408        if (typename == null) {
1409            typename = "binary";
1410        }
1411        CmsResourceManager resourceManager = OpenCms.getResourceManager();
1412
1413        return resourceManager.getResourceType(typename).getTypeId();
1414    }
1415
1416    /**
1417     * Gets a valid VfsName form a given name in the real file system.<p>
1418     *
1419     * This name will later be used for all link translations during the HTML-parsing process.<p>
1420     *
1421     * @param relativeName the name in the real file system, relative to the start folder
1422     * @param name the name of the file
1423     * @param isFile flag to indicate that the resource is a file
1424     *
1425     * @return a valid name in the VFS
1426     *
1427     * @throws Exception if something goes wrong
1428     */
1429    private String getVfsName(String relativeName, String name, boolean isFile) throws Exception {
1430
1431        // first translate all file-separators to the valid "/" in OpenCms
1432        String vfsName = relativeName.replace('\\', '/');
1433        // the resource is a file
1434        if (isFile) {
1435            // we must check if it might be copied into a gallery. this can be done by checking the
1436            // file extension
1437            int filetype = getFileType(name);
1438
1439            // there is no name before the ".extension"
1440            if (name.indexOf(".") == 0) {
1441                name = "unknown" + name;
1442                int dot = relativeName.lastIndexOf(".");
1443
1444                relativeName = relativeName.substring(0, dot) + name;
1445            }
1446
1447            // depending on the file-type, the resource must be moved into a special folder in
1448            // OpenCms:
1449            // images -> move into image gallery, if flag to leave at original location is off
1450            // binary -> move into download gallery, if flag to leave at original location is off
1451            // plain -> move into destination folder
1452            // other -> move into download gallery, if flag to leave at original location is off
1453            boolean leaveImages = CmsStringUtil.isEmptyOrWhitespaceOnly(m_imageGallery);
1454            boolean leaveDownload = CmsStringUtil.isEmptyOrWhitespaceOnly(m_downloadGallery);
1455            int imageId = OpenCms.getResourceManager().getResourceType(
1456                CmsResourceTypeImage.getStaticTypeName()).getTypeId();
1457            int plainId = OpenCms.getResourceManager().getResourceType(
1458                CmsResourceTypePlain.getStaticTypeName()).getTypeId();
1459            if ((imageId == filetype) && (!leaveImages)) {
1460                // move to image gallery
1461                // as the image gallery is "flat", we must use the file name and not the complete
1462                // relative name
1463                vfsName = m_imageGallery + name;
1464            } else if ((plainId == filetype) || (leaveImages) || (leaveDownload)) {
1465                // move to destination folder
1466                //vfsName=m_destinationDir+relativeName;
1467
1468                // we have to check if there is a folder with the same name but without extension
1469                // if so, we will move the file into the folder and name it "index.html"
1470                String folderName = relativeName;
1471                if (folderName.indexOf(".") > 0) {
1472                    folderName = folderName.substring(0, folderName.indexOf("."));
1473                }
1474                folderName = m_inputDir + "\\" + folderName;
1475                File folder = new File(folderName);
1476
1477                if (folder.isDirectory()) {
1478                    vfsName = m_destinationDir + relativeName.substring(0, relativeName.indexOf(".")) + "/index.html";
1479                    // System.err.println("MOVING "+ relativeName + " -> " + name.substring(0,name.indexOf("."))+"/index.html");
1480                } else {
1481                    // move to destination folder
1482                    vfsName = m_destinationDir + relativeName;
1483                }
1484
1485            } else {
1486                // everything else will be moved to the download gallery.
1487                // as the download gallery is "flat", we must use the file name and not the complete
1488                // relative name
1489                vfsName = m_downloadGallery + name;
1490            }
1491            // now we have the filename in the VFS. its possible that a file with the same name
1492            // is already existing, in this case, we have to adjust the filename.
1493            return validateFilename(vfsName);
1494        } else {
1495            // folders are always moved to the destination folder
1496            vfsName = m_destinationDir + vfsName + "/";
1497            return vfsName;
1498        }
1499    }
1500
1501    /**
1502     * Tests if a filename is an external name, that is this name does not point into the OpenCms VFS.<p>
1503     * A filename is an external name if it contains the string "://", e.g. "http://" or "ftp://".<p>
1504     *
1505     * @param filename the filename to test
1506     *
1507     * @return true or false
1508     */
1509    private boolean isExternal(String filename) {
1510
1511        boolean external = false;
1512        if (filename.indexOf("://") > 0) {
1513            external = true;
1514        }
1515        return external;
1516    }
1517
1518    /**
1519     * Checks if m_element is valid element.<p>
1520     *
1521     * @return true if element is valid, otherwise false
1522     */
1523    private boolean isValidElement() {
1524
1525        boolean validElement = false;
1526        List elementList = new ArrayList();
1527        try {
1528            // get Elements of template stored in Property "template-elements"
1529            String elements = m_cmsObject.readPropertyObject(
1530                m_template,
1531                CmsPropertyDefinition.PROPERTY_TEMPLATE_ELEMENTS,
1532                false).getValue();
1533            // template may contain more than one Element
1534            // Elements are separated by the delimiter ","
1535            if (elements != null) {
1536                StringTokenizer T = new StringTokenizer(elements, ",");
1537                while (T.hasMoreTokens()) {
1538                    // current element probably looks like "body*|Body" <name><mandatory>|<nicename>
1539                    String currentElement = T.nextToken();
1540                    int sepIndex = currentElement.indexOf("|");
1541                    if (sepIndex != -1) {
1542                        // current element == "body*"
1543                        currentElement = currentElement.substring(0, sepIndex);
1544                    }
1545                    if (currentElement.endsWith("*")) {
1546                        // current element == "body"
1547                        currentElement = currentElement.substring(0, currentElement.length() - 1);
1548                    }
1549                    elementList.add(currentElement);
1550                }
1551            }
1552            if (elementList.contains(m_element)) {
1553                validElement = true;
1554            }
1555        } catch (Exception e) {
1556            e.printStackTrace();
1557        }
1558
1559        return validElement;
1560    }
1561
1562    /**
1563     * Reads the content of an HTML file from the real file system and parses it for link
1564     * transformation.<p>
1565     *
1566     * @param  file the file in the real file system
1567     * @param properties the file properties
1568     *
1569     * @return the modified HTML code of the file
1570     *
1571     * @throws CmsException if something goes wrong
1572     */
1573    private String parseHtmlFile(File file, Hashtable properties) throws CmsException {
1574
1575        String parsedHtml = "";
1576        try {
1577
1578            byte[] content = getFileBytes(file);
1579
1580            // use the correct encoding to get the string from the file bytes
1581            String contentString = new String(content, m_inputEncoding);
1582            // escape the string to remove all special chars
1583            contentString = CmsEncoder.escapeNonAscii(contentString);
1584            // we must substitute all occurrences of "&#", otherwise tidy would remove them
1585            contentString = CmsStringUtil.substitute(contentString, "&#", "{_subst1_}");
1586            // we must substitute all occurrences of &lt; and &gt;  otherwise tidy would replace them with < and >
1587            contentString = CmsStringUtil.substitute(contentString, "&lt;", "{_subst2_}");
1588            contentString = CmsStringUtil.substitute(contentString, "&gt;", "{_subst3_}");
1589
1590            // parse the content
1591            parsedHtml = m_htmlConverter.convertHTML(
1592                file.getAbsolutePath(),
1593                contentString,
1594                m_startPattern,
1595                m_endPattern,
1596                properties);
1597            // resubstitute the converted HTML code
1598            parsedHtml = CmsStringUtil.substitute(parsedHtml, "{_subst1_}", "&#");
1599            parsedHtml = CmsStringUtil.substitute(parsedHtml, "{_subst2_}", "&lt;");
1600            parsedHtml = CmsStringUtil.substitute(parsedHtml, "{_subst3_}", "&gt;");
1601        } catch (Exception e) {
1602            CmsMessageContainer message = Messages.get().container(
1603                Messages.ERR_HTMLIMPORT_PARSE_1,
1604                file.getAbsolutePath());
1605            LOG.error(e.getLocalizedMessage(), e);
1606            throw new CmsImportExportException(message, e);
1607        }
1608        return parsedHtml;
1609    }
1610
1611    /**
1612     * This function reads the zip-file and saved the files and directories in a new
1613     * temporary-folder.<p>
1614     *
1615     * @return the temporary-folder where the files from the zip-file are saved
1616     */
1617    private File unzipStream() {
1618
1619        ZipInputStream importZip = null;
1620        File folder = null;
1621        try {
1622            // read the zip file
1623            importZip = new ZipInputStream(new FileInputStream(m_httpDir));
1624            // create a temporary-folder, where to unzip the zip file
1625            folder = createTempFolder("import_html");
1626            ZipEntry entry = null;
1627            byte[] buffer = null;
1628            while (true) {
1629                try {
1630                    // get the next entry
1631                    entry = importZip.getNextEntry();
1632                    if (entry == null) {
1633                        break;
1634                    }
1635                    String name = entry.getName();
1636                    // make a report for the user
1637                    m_report.print(Messages.get().container(Messages.RPT_HTML_UNZIP_0), I_CmsReport.FORMAT_NOTE);
1638                    m_report.print(
1639                        org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_ARGUMENT_1, name));
1640                    m_report.print(org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_DOTS_0));
1641                    // replace the VFS separator with the separator of the file system
1642                    name = name.replace('/', File.separatorChar);
1643                    String path = folder + File.separator + name;
1644                    if (entry.isDirectory()) {
1645                        // create a directory
1646                        File importFile = new File(path);
1647                        importFile.mkdirs();
1648                    } else {
1649                        // create a file and read the content
1650                        int size = new Long(entry.getSize()).intValue();
1651                        if (size == -1) {
1652                            buffer = CmsFileUtil.readFully(importZip, false);
1653                        } else {
1654                            buffer = CmsFileUtil.readFully(importZip, size, false);
1655                        }
1656                        // create a new temporary file
1657                        File importFile = new File(path);
1658                        File parent = importFile.getParentFile();
1659                        if (parent != null) {
1660                            parent.mkdirs();
1661                        }
1662                        importFile.createNewFile();
1663                        // write the content in the file
1664                        FileOutputStream fileOutput = new FileOutputStream(importFile.getAbsoluteFile());
1665                        fileOutput.write(buffer);
1666                        fileOutput.close();
1667                    }
1668                    importZip.closeEntry();
1669                    m_report.println(
1670                        org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_OK_0),
1671                        I_CmsReport.FORMAT_OK);
1672                } catch (Exception ex) {
1673                    String name = (entry != null ? entry.getName() : "");
1674                    if (LOG.isErrorEnabled()) {
1675                        LOG.error(Messages.get().getBundle().key(Messages.ERR_ZIPFILE_UNZIP_1, name), ex);
1676                    }
1677                    m_report.println(
1678                        Messages.get().container(Messages.ERR_ZIPFILE_UNZIP_1, name),
1679                        I_CmsReport.FORMAT_ERROR);
1680                }
1681                entry = null;
1682            }
1683        } catch (Exception ex) {
1684            if (LOG.isErrorEnabled()) {
1685                LOG.error(Messages.get().getBundle().key(Messages.ERR_ZIPFILE_READ_1, m_httpDir), ex);
1686            }
1687            m_report.println(
1688                Messages.get().container(Messages.ERR_ZIPFILE_READ_1, m_httpDir),
1689                I_CmsReport.FORMAT_ERROR);
1690        } finally {
1691            closeStream(importZip);
1692        }
1693        return folder;
1694
1695    }
1696
1697    /**
1698     * Validates a filename for OpenCms.<p>
1699     *
1700     * This method checks if there are any illegal characters in the filename and modifies them
1701     * if necessary. In addition it ensures that no duplicate filenames are created.<p>
1702     *
1703     * @param filename the filename to validate
1704     *
1705     * @return a validated and unique filename in OpenCms
1706     */
1707    private String validateFilename(String filename) {
1708
1709        // if its an external filename, use it directly
1710        if (isExternal(filename)) {
1711            return filename;
1712        }
1713
1714        // check if this resource name does already exist
1715        // if so add a postfix to the name
1716
1717        int postfix = 1;
1718        boolean found = true;
1719        String validFilename = filename;
1720
1721        // if we are not in overwrite mode, we must find a valid, non-existing filename
1722        // otherwise we will use the current translated name
1723        if (!m_overwrite) {
1724
1725            while (found) {
1726                try {
1727                    // get the translated name, this one only contains valid chars in OpenCms
1728                    validFilename = m_cmsObject.getRequestContext().getFileTranslator().translateResource(
1729                        validFilename);
1730
1731                    // try to read the file.....
1732                    found = true;
1733                    // first try to read it form the fileIndex of already processed files
1734                    if (!m_fileIndex.containsValue(validFilename.replace('\\', '/'))) {
1735                        found = false;
1736                    }
1737                    if (!found) {
1738                        found = true;
1739                        // there was no entry in the fileIndex, so try to read from the VFS
1740                        m_cmsObject.readResource(validFilename, CmsResourceFilter.ALL);
1741                    }
1742                    // ....it's there, so add a postfix and try again
1743                    String path = filename.substring(0, filename.lastIndexOf("/") + 1);
1744                    String name = filename.substring(filename.lastIndexOf("/") + 1, filename.length());
1745                    validFilename = path;
1746                    if (name.lastIndexOf(".") > 0) {
1747                        validFilename += name.substring(0, name.lastIndexOf("."));
1748                    } else {
1749                        validFilename += name;
1750                    }
1751                    validFilename += "_" + postfix;
1752                    if (name.lastIndexOf(".") > 0) {
1753                        validFilename += name.substring(name.lastIndexOf("."), name.length());
1754                    }
1755                    postfix++;
1756                } catch (CmsException e) {
1757                    // the file does not exist, so we can use this filename
1758                    found = false;
1759                }
1760            }
1761
1762        } else {
1763            validFilename = validFilename.replace('\\', '/');
1764        }
1765
1766        return OpenCms.getResourceManager().getFileTranslator().translateResource(validFilename);
1767    }
1768
1769}