import java.net.URL;

import java.net.URLConnection;

                                if (pathTo.endsWith(".jar.external")) {

                                    File downloaded = new File(destFile.getParentFile(), destFile.getName().substring(0, destFile.getName().lastIndexOf(".external")));

                                    final InputStream spec = jarFile.getInputStream(entry);

                                    InputStream is = externalDownload(spec);

                                    spec.close();

                                    FileOutputStream os = new FileOutputStream(downloaded);

                                    copyStreams(is, os, -1);

                                    is.close();

                                    os.close();

                                    crc = UpdateTracking.getFileCRC(downloaded);

                                }

    private InputStream externalDownload(InputStream is) throws IOException {

        BufferedReader br = new BufferedReader(new InputStreamReader(is));

        URLConnection conn;

        for (;;) {

            String line = br.readLine();

            if (line == null) {

                break;

            }

            if (line.startsWith("URL:")) {

                String url = line.substring(4).trim();

                for (;;) {

                    int index = url.indexOf("${");

                    if (index == -1) {

                        break;

                    }

                    int end = url.indexOf("}", index);

                    String propName = url.substring(index + 2, end);

                    final String propVal = System.getProperty(propName);

                    if (propVal == null) {

                        throw new IOException("Can't find property " + propName);

                    }

                    url = url.substring(0, index) + propVal + url.substring(end + 1);

                }

                XMLUtil.LOG.log(Level.INFO, "Trying external URL: {0}", url);

                try {

                    conn = new URL(url).openConnection();

                    conn.connect();

                    return conn.getInputStream();

                } catch (IOException ex) {

                    XMLUtil.LOG.log(Level.WARNING, "Cannot connect to {0}", url);

                    XMLUtil.LOG.log(Level.INFO, "Details", ex);

                }

            }

        }

        throw new IOException("Cannot resolve external references");

    }

/*

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.

 *

 * Copyright 2010 Oracle and/or its affiliates. All rights reserved.

 *

 * Oracle and Java are registered trademarks of Oracle and/or its affiliates.

 * Other names may be trademarks of their respective owners.

 *

 * The contents of this file are subject to the terms of either the GNU

 * General Public License Version 2 only ("GPL") or the Common

 * Development and Distribution License("CDDL") (collectively, the

 * "License"). You may not use this file except in compliance with the

 * License. You can obtain a copy of the License at

 * http://www.netbeans.org/cddl-gplv2.html

 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the

 * specific language governing permissions and limitations under the

 * License.  When distributing the software, include this License Header

 * Notice in each file and include the License file at

 * nbbuild/licenses/CDDL-GPL-2-CP.  Oracle designates this

 * particular file as subject to the "Classpath" exception as provided

 * by Oracle in the GPL Version 2 section of the License file that

 * accompanied this code. If applicable, add the following below the

 * License Header, with the fields enclosed by brackets [] replaced by

 * your own identifying information:

 * "Portions Copyrighted [year] [name of copyright owner]"

 *

 * If you wish your version of this file to be governed by only the CDDL

 * or only the GPL Version 2, indicate your decision by adding

 * "[Contributor] elects to include this software in this distribution

 * under the [CDDL or GPL Version 2] license." If you do not indicate a

 * single choice of license, a recipient has the option to distribute

 * your version of this file under either the CDDL, the GPL Version 2 or

 * to extend the choice of license to its licensees as provided above.

 * However, if you add GPL Version 2 code and therefore, elected the GPL

 * Version 2 license, then the option applies only if the new code is

 * made subject to such option by the copyright holder.

 *

 * Contributor(s):

 *

 * Portions Copyrighted 2010 Sun Microsystems, Inc.

 */

package org.netbeans.modules.autoupdate.services;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.OutputStream;

import java.io.OutputStreamWriter;

import java.io.PrintWriter;

import java.net.URL;

import java.util.ArrayList;

import java.util.List;

import java.util.jar.Attributes;

import java.util.jar.JarFile;

import java.util.jar.JarOutputStream;

import java.util.jar.Manifest;

import java.util.jar.Pack200;

import java.util.logging.Level;

import java.util.zip.GZIPOutputStream;

import java.util.zip.ZipEntry;

import org.netbeans.api.autoupdate.InstallSupport;

import org.netbeans.api.autoupdate.OperationContainer;

import org.netbeans.api.autoupdate.OperationContainer.OperationInfo;

import org.netbeans.api.autoupdate.OperationException;

import org.netbeans.api.autoupdate.OperationSupport.Restarter;

import org.netbeans.api.autoupdate.TestUtils;

import org.netbeans.api.autoupdate.UpdateElement;

import org.netbeans.api.autoupdate.UpdateUnit;

import org.netbeans.api.autoupdate.UpdateUnitProvider;

import org.netbeans.api.autoupdate.UpdateUnitProviderFactory;

import org.netbeans.core.startup.MainLookup;

import org.netbeans.junit.NbTestCase;

import org.netbeans.modules.autoupdate.updateprovider.AutoupdateCatalogProvider;

import org.openide.filesystems.FileUtil;

import org.openide.util.Lookup;

public class NbmExternalTest extends NbTestCase {

    protected List<UpdateUnit> keepItNotToGC;

    private static File catalogFile;

    private static URL catalogURL;

    private File tmpDirectory;

    private List<File> nbms = new ArrayList<File>();

    private List<String> moduleElements = new ArrayList<String>();

    public NbmExternalTest(String testName) {

        super(testName);

    }

    @Override

    protected Level logLevel() {

        return Level.FINE;

    }

    public static class MyProvider extends AutoupdateCatalogProvider {

        public MyProvider() {

            super("test-updates-provider", "test-updates-provider", catalogURL, UpdateUnitProvider.CATEGORY.STANDARD);

        }

    }

    private String getModuleElement(boolean visible, String codeName, String releaseVersion, String implVersion, String moduleName, String distr, String specVersion, String dependency) {

        String releaseVersionAppendix = ((releaseVersion != null /*&& Integer.parseInt(releaseVersion)!=0*/) ? ("/" + releaseVersion) : "");

        return "\n<module "

                + "\n     codenamebase='" + codeName + "' "

                + "\n     distribution='" + distr + "' "

                + "\n     downloadsize='0' "

                + "\n     homepage='' "

                + "\n     license='AD9FBBC9' "

                + "\n     moduleauthor='' "

                + "\n     needsrestart='false' "

                + "\n     releasedate='2007/01/30'>"

                + "\n    <manifest "

                + "\n       AutoUpdate-Show-In-Client='" + visible + "' "

                + "\n       OpenIDE-Module='" + codeName + releaseVersionAppendix + "'"

                + "\n       OpenIDE-Module-Implementation-Version='" + (implVersion == null ? "070130" : implVersion) + "' "

                + "\n       OpenIDE-Module-Java-Dependencies='Java > 1.4' "

                + "\n       OpenIDE-Module-Name='" + moduleName + "' "

                + (dependency != null ? " OpenIDE-Module-Module-Dependencies=\"" + dependency.replace(">", ">") + "\" " : "")

                + "\n       OpenIDE-Module-Requires='org.openide.modules.ModuleFormat1' "

                + "\n       OpenIDE-Module-Specification-Version='" + specVersion + "'/>"

                + "\n</module>";

    }

    private String createInfoXML(boolean visible, String codeName, String releaseVersion, String implVersion, String moduleName, String distr, String specVersion, String dependency) {

        String moduleElement = getModuleElement(visible, codeName, releaseVersion, implVersion, moduleName, distr, specVersion, dependency);

        moduleElements.add(moduleElement);

        moduleElements.add("\n    <license name='AD9FBBC9'>[NO LICENSE SPECIFIED]"

                         + "\n</license>\n");

        return "<?xml version='1.0' encoding='UTF-8'?>"

                + "<!DOCTYPE module PUBLIC '-//NetBeans//DTD Autoupdate Module Info 2.5//EN' 'http://www.netbeans.org/dtds/autoupdate-info-2_5.dtd'>"

                + moduleElement;

    }

    private void writeCatalog() throws IOException {

        String res = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"

                + "<!DOCTYPE module_updates PUBLIC \"-//NetBeans//DTD Autoupdate Catalog 2.5//EN\" \"http://www.netbeans.org/dtds/autoupdate-catalog-2_5.dtd\">"

                + "<module_updates timestamp=\"00/00/19/08/03/2006\">\n";

        for (String element : moduleElements) {

            res += element;

        }

        res += "</module_updates>\n";

        if (catalogFile == null) {

            catalogFile = File.createTempFile("catalog-", ".xml", tmpDirectory);

            catalogURL = catalogFile.toURI().toURL();

        }

        PrintWriter pw = new PrintWriter(new OutputStreamWriter(new FileOutputStream(catalogFile), "UTF-8"));

        pw.write(res);

        pw.close();

    }

    private String getConfigXML(String codeName, String moduleFile, String specVersion) {

        return "<?xml version='1.0' encoding='UTF-8'?>"

                + " <!DOCTYPE module PUBLIC '-//NetBeans//DTD Module Status 1.0//EN'"

                + " 'http://www.netbeans.org/dtds/module-status-1_0.dtd'>"

                + " <module name='" + codeName + "'>"

                + " <param name='autoload'>false</param>"

                + " <param name='eager'>false</param>"

                + " <param name='enabled'>true</param>"

                + " <param name='jar'>modules/" + moduleFile + ".jar</param>"

                + " <param name='reloadable'>false</param>"

                + " <param name='specversion'>" + specVersion + "</param>"

                + " </module>";

    }

    private String getManifest(String codeName, String releaseVersion, String implVersion, String moduleDir, String specVersion, boolean visible, String dependency) {

        String releaseVersionAppendix = ((releaseVersion != null /*&& Integer.parseInt(releaseVersion)!=0*/) ? ("/" + releaseVersion) : "");

        return "Manifest-Version: 1.0\n"

                + "Ant-Version: Apache Ant 1.7.0\n"

                + "Created-By: 1.6.0-b105 (Sun Microsystems Inc.)\n"

                + "OpenIDE-Module-Public-Packages: -\n"

                + "OpenIDE-Module-Java-Dependencies: Java > 1.4\n"

                + "OpenIDE-Module-Implementation-Version: " + (implVersion == null ? "070130" : implVersion)

                + "\n"

                + (dependency != null ? ("OpenIDE-Module-Module-Dependencies: " + dependency + "\n") : "")

                + "OpenIDE-Module: " + codeName + releaseVersionAppendix

                + "\n"

                + "OpenIDE-Module-Localizing-Bundle: " + moduleDir + "Bundle.properties\n"

                + "OpenIDE-Module-Specification-Version: " + specVersion + "\n"

                + "OpenIDE-Module-Requires: org.openide.modules.ModuleFormat1\n"

                + "AutoUpdate-Show-In-Client: " + visible + "\n"

                + "\n";

    }

    private File prepareNBM(String codeName, String releaseVersion, String implVersion, String specVersion, boolean visible, String dependency) throws Exception {

        String moduleName = codeName.substring(codeName.lastIndexOf(".") + 1);

        String moduleFile = codeName.replace(".", "-");

        String moduleDir = codeName.replace(".", "/") + "/";

        File nbm = File.createTempFile(moduleFile + "-", ".nbm", tmpDirectory);

        final String MODULE_NAME_PROP = "OpenIDE-Module-Name";

        File jar = new File(tmpDirectory, "x.jar");

        jar.getParentFile().mkdirs();

        JarOutputStream jos = new JarOutputStream(new FileOutputStream(jar));

        int idx = moduleDir.indexOf("/");

        while (idx != -1) {

            jos.putNextEntry(new ZipEntry(moduleDir.substring(0, idx + 1)));

            idx = moduleDir.indexOf("/", idx + 1);

        }

        jos.putNextEntry(new ZipEntry(moduleDir + "Bundle.properties"));

        jos.write(new String(MODULE_NAME_PROP + "=" + moduleName).getBytes("UTF-8"));

        jos.putNextEntry(new ZipEntry("META-INF/"));

        jos.putNextEntry(new ZipEntry("META-INF/manifest.mf"));

        jos.write(getManifest(codeName, releaseVersion, implVersion, moduleDir, specVersion, visible, dependency).getBytes("UTF-8"));

        jos.close();

        File ext = new File(jar.getParentFile(), jar.getName() + ".external");

        FileOutputStream os = new FileOutputStream(ext);

        os.write(("URL: " + jar.toURI().toString() + "\n").getBytes());

        os.close();

        Manifest mf = new Manifest();

        mf.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");

        jos = new JarOutputStream(new FileOutputStream(nbm), mf);

        jos.putNextEntry(new ZipEntry("Info/"));

        jos.putNextEntry(new ZipEntry("Info/info.xml"));

        jos.write(createInfoXML(visible, codeName, releaseVersion, implVersion, moduleName, nbm.toURI().toURL().toString(), specVersion, dependency).getBytes("UTF-8"));

        jos.putNextEntry(new ZipEntry("netbeans/"));

        jos.putNextEntry(new ZipEntry("netbeans/modules/"));

        jos.putNextEntry(new ZipEntry("netbeans/config/"));

        jos.putNextEntry(new ZipEntry("netbeans/config/Modules/"));

        jos.putNextEntry(new ZipEntry("netbeans/config/Modules/" + moduleFile + ".xml"));

        jos.write(getConfigXML(codeName, moduleFile, specVersion).getBytes("UTF-8"));

        jos.putNextEntry(new ZipEntry("netbeans/modules/" + moduleFile + ".jar.external"));

        FileInputStream fis = new FileInputStream(ext);

        FileUtil.copy(fis, jos);

        fis.close();

        ext.delete();

        jos.close();

        nbms.add(nbm);

        return nbm;

    }

    protected void setUp() throws Exception {

        super.setUp();

        this.clearWorkDir();

        tmpDirectory = new File(getWorkDirPath(), "tmp");

        tmpDirectory.mkdirs();

        writeCatalog();

        TestUtils.setUserDir(getWorkDirPath());

        TestUtils.testInit();

        MainLookup.register(new MyProvider());

        assert Lookup.getDefault().lookup(MyProvider.class) != null;

        UpdateUnitProviderFactory.getDefault().refreshProviders(null, true);

    }

    private void doInstall(OperationContainer<InstallSupport> installContainer) throws OperationException {

        InstallSupport support = installContainer.getSupport();

        assertNotNull(support);

        InstallSupport.Validator v = support.doDownload(null, false);

        assertNotNull(v);

        InstallSupport.Installer i = support.doValidate(v, null);

        assertNotNull(i);

        Restarter r = null;

        try {

            r = support.doInstall(i, null);

        } catch (OperationException ex) {

            if (OperationException.ERROR_TYPE.INSTALL == ex.getErrorType()) {

                // can ingore

                // module system cannot load the module either

            } else {

                fail(ex.toString());

            }

        }

        assertNull("Installing new element require restarting though it should not", r);

    }

    public void testNbmWithExternal() throws Exception {

        String moduleCNB = "org.netbeans.modules.mymodule";

        String moduleReleaseVersion = "1";

        String moduleImplVersion = "2";

        String moduleSpecVersion = "1.0";

        prepareNBM(moduleCNB, moduleReleaseVersion, moduleImplVersion, moduleSpecVersion, false, null);

        writeCatalog();

        UpdateUnitProviderFactory.getDefault().refreshProviders(null, true);

        OperationContainer<InstallSupport> installContainer = OperationContainer.createForInstall();

        UpdateUnit moduleUnit = getUpdateUnit(moduleCNB);

        assertNull("cannot be installed", moduleUnit.getInstalled());

        UpdateElement moduleElement = getAvailableUpdate(moduleUnit, 0);

        assertEquals(moduleElement.getSpecificationVersion(), moduleSpecVersion);

        OperationInfo<InstallSupport> independentInfo = installContainer.add(moduleElement);

        assertNotNull(independentInfo);

        doInstall(installContainer);

        File module = new File(new File(getWorkDir(), "modules"), "org-netbeans-modules-mymodule.jar");

        assertTrue("module file exists", module.exists());

        assertTrue("module was not installed from NBM external", moduleUnit.getInstalled() != null);

    }

    public UpdateUnit getUpdateUnit(String codeNameBase) {

        UpdateUnit uu = UpdateManagerImpl.getInstance().getUpdateUnit(codeNameBase);

        assertNotNull(uu);

        return uu;

    }

    public UpdateElement getAvailableUpdate(UpdateUnit updateUnit, int idx) {

        List<UpdateElement> available = updateUnit.getAvailableUpdates();

        assertTrue(available.size() > idx);

        return available.get(idx);

    }

}

import java.io.BufferedReader;

import java.io.InputStreamReader;

import java.net.URLConnection;

import java.util.concurrent.atomic.AtomicLong;

                    OutputStream os;

                    if (relName.endsWith(".external")) {

                        is = externalDownload(is);

                        File dest = new File(trgt.getParentFile(), trgt.getName().substring(0, trgt.getName().length() - 9));

                        os = new FileOutputStream(dest);

                    } else {

                        os = new FileOutputStream(trgt);

                    }

    private InputStream externalDownload(InputStream is) throws IOException {

        BufferedReader br = new BufferedReader(new InputStreamReader(is));

        URLConnection conn;

        for (;;) {

            String line = br.readLine();

            if (line == null) {

                break;

            }

            if (line.startsWith("URL:")) {

                String url = line.substring(4).trim();

                for (;;) {

                    int index = url.indexOf("${");

                    if (index == -1) {

                        break;

                    }

                    int end = url.indexOf("}", index);

                    String propName = url.substring(index + 2, end);

                    final String propVal = System.getProperty(propName);

                    if (propVal == null) {

                        throw new IOException("Can't find property " + propName);

                    }

                    url = url.substring(0, index) + propVal + url.substring(end + 1);

                }

                log("Trying external URL: " + url, Project.MSG_INFO);

                try {

                    conn = new URL(url).openConnection();

                    conn.connect();

                    return conn.getInputStream();

                } catch (IOException ex) {

                    log("Cannot connect to " + url, Project.MSG_WARN);

                    log("Details", ex, Project.MSG_VERBOSE);

                }

            }

        }

        throw new IOException("Cannot resolve external references");

    }

import java.util.ArrayList;

import java.util.List;

    public void testDownloadAndExtractExternal() throws Exception {

        File f = new File(getWorkDir(), "org-netbeans-api-annotations-common.xml");

        extractResource(f, "org-netbeans-api-annotations-common.xml");

        File ext = extractString("external content");

        String extRef =

            "URL: " + ext.toURI().toString() + "\n";

        File nbm = generateNBMContent("org-netbeans-api-annotations-common.nbm",

            "netbeans/config/Modules/org-netbeans-api-annotations-common.xml", "empty",

            "netbeans/modules/org-netbeans-api-annotations-common.jar.external", extRef,

            "netbeans/docs/My Manager's So-Called \"README\"", "empty");

        File target = new File(getWorkDir(), "target");

        target.mkdirs();

        execute(

            "autoupdate.xml", "-verbose", "-Durl=" + f.toURI().toURL(),

            "-Dincludes=org.netbeans.api.annotations.common",

            "-Dtarget=" + target

        );

        File xml = new File(

            new File(new File(target, "platform"), "update_tracking"),

            "org-netbeans-api-annotations-common.xml"

        );

        assertTrue("xml file created", xml.exists());

        Document doc = XMLUtil.parse(new InputSource(xml.toURI().toString()), false, false, null, null);

        NodeList nl = doc.getElementsByTagName("file");

        assertEquals(3, nl.getLength());

        assertEquals("docs/My Manager's So-Called \"README\"", ((Element) nl.item(2)).getAttribute("name"));

        File jar = new File(

            new File(new File(target, "platform"), "modules"),

            "org-netbeans-api-annotations-common.jar"

        );

        File jarExt = new File(

            new File(new File(target, "platform"), "modules"),

            "org-netbeans-api-annotations-common.jar.external"

        );

        assertFalse("no external file created", jarExt.exists());

        assertTrue("jar file created", jar.exists());

        assertEquals("Contains expected", "external content", readFile(jar));

        File lastM = new File(new File(target, "platform"), ".lastModified");

        assertTrue("Last modified file created", lastM.exists());

    }

    public void testDownloadAndExtractExternalWithFirstURLBroken() throws Exception {

        File f = new File(getWorkDir(), "org-netbeans-api-annotations-common.xml");

        extractResource(f, "org-netbeans-api-annotations-common.xml");

        File ext = extractString("external content");

        String extRef =

            "URL: file:/I/dont/exist/At/All\n" +

            "URL: " + ext.toURI().toString() + "\n";

        File nbm = generateNBMContent("org-netbeans-api-annotations-common.nbm",

            "netbeans/config/Modules/org-netbeans-api-annotations-common.xml", "empty",

            "netbeans/modules/org-netbeans-api-annotations-common.jar.external", extRef,

            "netbeans/docs/My Manager's So-Called \"README\"", "empty");

        File target = new File(getWorkDir(), "target");

        target.mkdirs();

        execute(

            "autoupdate.xml", "-verbose", "-Durl=" + f.toURI().toURL(),

            "-Dincludes=org.netbeans.api.annotations.common",

            "-Dtarget=" + target

        );

        File xml = new File(

            new File(new File(target, "platform"), "update_tracking"),

            "org-netbeans-api-annotations-common.xml"

        );

        assertTrue("xml file created", xml.exists());

        Document doc = XMLUtil.parse(new InputSource(xml.toURI().toString()), false, false, null, null);

        NodeList nl = doc.getElementsByTagName("file");

        assertEquals(3, nl.getLength());

        assertEquals("docs/My Manager's So-Called \"README\"", ((Element) nl.item(2)).getAttribute("name"));

        File jar = new File(

            new File(new File(target, "platform"), "modules"),

            "org-netbeans-api-annotations-common.jar"

        );

        File jarExt = new File(

            new File(new File(target, "platform"), "modules"),

            "org-netbeans-api-annotations-common.jar.external"

        );

        assertFalse("no external file created", jarExt.exists());

        assertTrue("jar file created", jar.exists());

        assertEquals("Contains expected", "external content", readFile(jar));

        File lastM = new File(new File(target, "platform"), ".lastModified");

        assertTrue("Last modified file created", lastM.exists());

    }

    public void testDownloadAndExtractExternalWithProperty() throws Exception {

        File f = new File(getWorkDir(), "org-netbeans-api-annotations-common.xml");

        extractResource(f, "org-netbeans-api-annotations-common.xml");

        File ext = extractString("external content");

        System.setProperty("my.ref", ext.getParent());

        String extRef =

            "URL: file:${my.ref}/" + ext.getName() + "\n";

        File nbm = generateNBMContent("org-netbeans-api-annotations-common.nbm",

            "netbeans/config/Modules/org-netbeans-api-annotations-common.xml", "empty",

            "netbeans/modules/org-netbeans-api-annotations-common.jar.external", extRef,

            "netbeans/docs/My Manager's So-Called \"README\"", "empty");

        File target = new File(getWorkDir(), "target");

        target.mkdirs();

        execute(

            "autoupdate.xml", "-verbose", "-Durl=" + f.toURI().toURL(),

            "-Dincludes=org.netbeans.api.annotations.common",

            "-Dtarget=" + target

        );

        File xml = new File(

            new File(new File(target, "platform"), "update_tracking"),

            "org-netbeans-api-annotations-common.xml"

        );

        assertTrue("xml file created", xml.exists());

        Document doc = XMLUtil.parse(new InputSource(xml.toURI().toString()), false, false, null, null);

        NodeList nl = doc.getElementsByTagName("file");

        assertEquals(3, nl.getLength());

        assertEquals("docs/My Manager's So-Called \"README\"", ((Element) nl.item(2)).getAttribute("name"));

        File jar = new File(

            new File(new File(target, "platform"), "modules"),

            "org-netbeans-api-annotations-common.jar"

        );

        File jarExt = new File(

            new File(new File(target, "platform"), "modules"),

            "org-netbeans-api-annotations-common.jar.external"

        );

        assertFalse("no external file created", jarExt.exists());

        assertTrue("jar file created", jar.exists());

        assertEquals("Contains expected", "external content", readFile(jar));

        File lastM = new File(new File(target, "platform"), ".lastModified");

        assertTrue("Last modified file created", lastM.exists());

    }

        List<String> filesAndContent = new ArrayList<String>();

        for (String s : files) {

            filesAndContent.add(s);

            filesAndContent.add("empty");

        }

        return generateNBMContent(name, filesAndContent.toArray(new String[0]));

    }

    public File generateNBMContent (String name, String... filesAndContent) throws IOException {