# HG changeset patch
# Parent 09b9cc69d382d65cec3d844a02d0f0251a6fb781
# User Jesse Glick <jglick@netbeans.org>
#207137: cache downloaded NBMs.

diff --git a/apisupport.ant/src/org/netbeans/modules/apisupport/project/suite/resources/platform.xsl b/apisupport.ant/src/org/netbeans/modules/apisupport/project/suite/resources/platform.xsl
--- a/apisupport.ant/src/org/netbeans/modules/apisupport/project/suite/resources/platform.xsl
+++ b/apisupport.ant/src/org/netbeans/modules/apisupport/project/suite/resources/platform.xsl
@@ -31,7 +31,7 @@
                 <property name="tasks.jar" location="${{java.io.tmpdir}}/tasks.jar"/>
                 <get src="${{bootstrap.url}}" dest="${{tasks.jar}}" usetimestamp="true" verbose="true"/>
                 <taskdef name="autoupdate" classname="org.netbeans.nbbuild.AutoUpdate" classpath="${{tasks.jar}}"/>
-                <autoupdate installdir="${{nbplatform.active.dir}}" updatecenter="${{autoupdate.catalog.url}}">
+                <autoupdate installdir="${{nbplatform.active.dir}}" updatecenter="${{autoupdate.catalog.url}}" cache="${{autoupdate.cache}}">
                     <modules includes="${{module.includes}}.*" clusters="${{download.clusters}}"/>
                     <modules includes="org[.]netbeans[.]modules[.]apisupport[.]harness" clusters="harness"/>
                 </autoupdate>
diff --git a/apisupport.harness/release/README b/apisupport.harness/release/README
--- a/apisupport.harness/release/README
+++ b/apisupport.harness/release/README
@@ -777,11 +777,13 @@
 nbm.target.cluster [since 5.0u2] - allow to declare a target cluster where
 install if NBM install globally.
 
-autoupdate.catalog.url - if you want to use module's or suite's update target
-during build, you need to specify the URL of your NBMs catalog. Since 7.1, this
-may be used (in conjunction with bootstrap.url) to download the harness and
+autoupdate.catalog.url [since 7.1] - if you want to use module's or suite's update target
+during build, you need to specify the URL of your NBMs catalog.
+May be used (in conjunction with bootstrap.url) to download the harness and
 platform from an update site.
 
+autoupdate.cache [since 7.2] - optional location of per-user/machine update cache.
+
 netbeans.dest.dir - [up to 6.5.1] absolute pathname of NB platform (or IDE or 
 whatever) that you are building your module against, and into which your module 
 will be built. Available in project.properties (and in all properties files for 
@@ -1042,6 +1044,7 @@
 +harness.dir=${nbplatform.active.dir}/harness
 +bootstrap.url=http://deadlock.netbeans.org/hudson/job/nbms-and-javadoc/lastStableBuild/artifact/nbbuild/netbeans/harness/tasks.jar
 +autoupdate.catalog.url=http://updates.netbeans.org/netbeans/updates/7.0.1/uc/final/distribution/catalog.xml.gz
++autoupdate.cache=${user.home}/.cache/nbupdate
 
 cluster.path [since 6.7] - list of clusters to build your modules against and
 include in your application.
diff --git a/apisupport.harness/release/suite.xml b/apisupport.harness/release/suite.xml
--- a/apisupport.harness/release/suite.xml
+++ b/apisupport.harness/release/suite.xml
@@ -599,6 +599,7 @@
         <subant target="update" buildpath="${modules.test.sorted}" inheritrefs="false" inheritall="false">
             <property name="cluster.path.evaluated" value="${cluster.path.evaluated}"/> <!-- Just for speed of pre-7.0 projects -->
             <property name="autoupdate.catalog.url" value="${autoupdate.catalog.url}"/>
+            <property name="autoupdate.cache" value="${autoupdate.cache}"/>
         </subant>
     </target>
 
diff --git a/nbbuild/antsrc/org/netbeans/nbbuild/AutoUpdate.java b/nbbuild/antsrc/org/netbeans/nbbuild/AutoUpdate.java
--- a/nbbuild/antsrc/org/netbeans/nbbuild/AutoUpdate.java
+++ b/nbbuild/antsrc/org/netbeans/nbbuild/AutoUpdate.java
@@ -74,6 +74,7 @@
 import org.apache.tools.ant.Task;
 import org.apache.tools.ant.taskdefs.Get;
 import org.apache.tools.ant.types.FileSet;
+import org.apache.tools.ant.util.FileUtils;
 import org.netbeans.nbbuild.AutoUpdateCatalogParser.ModuleItem;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
@@ -94,6 +95,8 @@
     private File cluster;
     private URL catalog;
     private boolean force;
+    private File cache;
+    boolean optimizeForLocal; // for test
 
     public void setUpdateCenter(URL u) {
         catalog = u;
@@ -130,6 +133,19 @@
         return m;
     }
 
+    /**
+     * Location of per-user cache of already downloaded binaries.
+     * Optional; no cache will be used if unset.
+     * The directory will be created if it does not yet exist.
+     */
+    public void setCache(File cache) {
+        if (cache.getAbsolutePath().contains("${")) {
+            log("Ignoring unevaluated cache dir: " + cache, Project.MSG_INFO);
+        } else {
+            this.cache = cache;
+        }
+    }
+
     @Override
     public void execute() throws BuildException {
         boolean downloadOnly = false;
@@ -183,7 +199,9 @@
         }
 
 
-        for (ModuleItem uu : units.values()) {
+        for (Map.Entry<String,ModuleItem> entry : units.entrySet()) {
+            String id = entry.getKey();
+            ModuleItem uu = entry.getValue();
             if (!matches(uu.getCodeName(), uu.targetcluster)) {
                 continue;
             }
@@ -201,7 +219,7 @@
             boolean delete = false;
             File lastM = null;
             try {
-                if (download == null && uu.getURL().getProtocol().equals("file")) {
+                if (download == null && uu.getURL().getProtocol().equals("file") && !optimizeForLocal) {
                     try {
                         tmp = new File(uu.getURL().toURI());
                     } catch (URISyntaxException ex) {
@@ -232,13 +250,23 @@
                     } else {
                         log("Version " + info.get(0) + " of " + uu.getCodeName() + " needs update to " + uu.getSpecVersion(), Project.MSG_INFO);
                     }
-                    Get get = new Get();
-                    get.setProject(getProject());
-                    get.setTaskName("get:" + uu.getCodeName());
-                    get.setSrc(uu.getURL());
-                    get.setDest(tmp);
-                    get.setVerbose(true);
-                    get.execute();
+                    if (cache == null) {
+                        download(uu, tmp);
+                    } else {
+                        File cacheLoc = new File(cache, id + ".nbm");
+                        if (cacheLoc.isFile()) {
+                            log("Cache hit: " + cacheLoc, Project.MSG_INFO);
+                        } else {
+                            cache.mkdirs();
+                            File momentary = new File(cacheLoc.getParentFile(), cacheLoc.getName() + ".tmp");
+                            momentary.delete();
+                            download(uu, momentary);
+                            if (!momentary.renameTo(cacheLoc)) {
+                                throw new IOException("rename " + momentary + " to " + cacheLoc);
+                            }
+                        }
+                        FileUtils.getFileUtils().copyFile(cacheLoc, tmp, null, true);
+                    }
                 }
                 if (downloadOnly) {
                     continue;
@@ -346,7 +374,7 @@
                     config.close();
                 }
             } catch (IOException ex) {
-                throw new BuildException(ex);
+                throw new BuildException("Failed to unpack " + tmp, ex);
             } finally {
                 if (delete && tmp != null) {
                     tmp.delete();
@@ -357,6 +385,16 @@
             }
         }
     }
+
+    private void download(ModuleItem uu, File f) throws BuildException {
+        Get get = new Get();
+        get.setProject(getProject());
+        get.setTaskName("get:" + uu.getCodeName());
+        get.setSrc(uu.getURL());
+        get.setDest(f);
+        get.setVerbose(true);
+        get.execute();
+    }
     
     private InputStream externalDownload(InputStream is, AtomicLong crc) throws IOException {
         BufferedReader br = new BufferedReader(new InputStreamReader(is));
diff --git a/nbbuild/antsrc/org/netbeans/nbbuild/AutoUpdateCatalogParser.java b/nbbuild/antsrc/org/netbeans/nbbuild/AutoUpdateCatalogParser.java
--- a/nbbuild/antsrc/org/netbeans/nbbuild/AutoUpdateCatalogParser.java
+++ b/nbbuild/antsrc/org/netbeans/nbbuild/AutoUpdateCatalogParser.java
@@ -61,6 +61,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Stack;
+import java.util.TreeMap;
 import java.util.jar.Manifest;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -132,6 +133,7 @@
     private static final String LICENSE_ATTR_URL = "url"; // NOI18N
     
     private static final String MANIFEST_ATTR_SPECIFICATION_VERSION = "OpenIDE-Module-Specification-Version"; // NOI18N
+    private static final String MANIFEST_ATTR_IMPLEMENTATION_VERSION = "OpenIDE-Module-Implementation-Version"; // NOI18N
     
     private static final String TIME_STAMP_FORMAT = "ss/mm/hh/dd/MM/yyyy"; // NOI18N
     
@@ -148,7 +150,7 @@
     private static URI cacheURI;
     synchronized static Map<String, ModuleItem> getUpdateItems (URL url, URL provider, Task task) throws IOException {
 
-        Map<String, ModuleItem> items = new HashMap<String, ModuleItem> ();
+        Map<String, ModuleItem> items = new TreeMap<String, ModuleItem> ();
         URI base;
         try {
             if (provider != null) {
@@ -471,8 +473,12 @@
         
         public void appendManifest (Attributes manifest) {
             specVersion = manifest.getValue (MANIFEST_ATTR_SPECIFICATION_VERSION);
+            String implVersion = manifest.getValue(MANIFEST_ATTR_IMPLEMENTATION_VERSION);
+            /* Useful fallback for implVersion, but 2.6 DTD lacks it (#207137 comment #2):
+            String buildVersion = manifest.getValue(MANIFEST_ATTR_BUILD_VERSION);
+            */
+            id = moduleCodeName + '_' + specVersion + '_' + (implVersion != null && !implVersion.matches("\\d{1,3}") ? implVersion : catalogDate != null ? catalogDate : "").replaceAll("[^a-zA-Z0-9._-]+", "_");
             mf = getManifest (manifest);
-            id = moduleCodeName + '_' + specVersion; // NOI18N
         }
         
         public void appendNotification (String notification) {
diff --git a/nbbuild/default-properties.xml b/nbbuild/default-properties.xml
--- a/nbbuild/default-properties.xml
+++ b/nbbuild/default-properties.xml
@@ -73,6 +73,7 @@
   <property name="moduleCluster.file" location="${netbeans.dest.dir}/moduleCluster.properties"/>
 
   <property name="autoupdate.catalog.url" value="http://deadlock.netbeans.org/hudson/job/nbms-and-javadoc/lastStableBuild/artifact/nbbuild/nbms/updates.xml.gz"/>
+  <property name="autoupdate.cache" location="${user.home}/.cache/nbupdate"/>
 
   <macrodef name="resolve">
     <attribute name="name"/>
diff --git a/nbbuild/templates/common.xml b/nbbuild/templates/common.xml
--- a/nbbuild/templates/common.xml
+++ b/nbbuild/templates/common.xml
@@ -192,6 +192,7 @@
         <autoupdate
             updatecenter="${autoupdate.catalog.url}"
             todir="${cluster}"
+            cache="${autoupdate.cache}"
         >
             <modules includes="${code.name.base}"/>
         </autoupdate>
diff --git a/nbbuild/test/unit/src/org/netbeans/nbbuild/AutoUpdateTest.java b/nbbuild/test/unit/src/org/netbeans/nbbuild/AutoUpdateTest.java
--- a/nbbuild/test/unit/src/org/netbeans/nbbuild/AutoUpdateTest.java
+++ b/nbbuild/test/unit/src/org/netbeans/nbbuild/AutoUpdateTest.java
@@ -52,6 +52,8 @@
 import java.util.zip.CRC32;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
+import org.apache.tools.ant.Project;
+import org.apache.tools.ant.types.FileSet;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 import org.w3c.dom.NodeList;
@@ -677,6 +679,39 @@
         }
     }
 
+    public void testDownloadOnlyWithCache() throws Exception {
+        AutoUpdate au = new AutoUpdate();
+        au.setProject(new Project());
+        generateNBM("org-netbeans-api-annotations-common.nbm", "netbeans/modules/org-netbeans-api-annotations-common.jar");
+        au.setUpdateCenter(extractResource("org-netbeans-api-annotations-common.xml").toURI().toURL());
+        File cache = new File(getWorkDir(), "cache");
+        au.setCache(cache);
+        File target = new File(getWorkDir(), "target");
+        target.mkdirs();
+        au.setDownloadDir(target);
+        au.createModules().setIncludes(".+");
+        au.execute();
+        assertTrue(new File(target, "org-netbeans-api-annotations-common.nbm").isFile());
+        assertTrue(new File(cache, "org.netbeans.api.annotations.common_1.3_nbms-and-javadoc-4296-on-091113.nbm").isFile());
+    }
+
+    public void testUnpackWithCache() throws Exception {
+        AutoUpdate au = new AutoUpdate();
+        au.setProject(new Project());
+        au.optimizeForLocal = true;
+        generateNBM("org-netbeans-api-annotations-common.nbm", "netbeans/modules/org-netbeans-api-annotations-common.jar");
+        au.setUpdateCenter(extractResource("org-netbeans-api-annotations-common.xml").toURI().toURL());
+        File cache = new File(getWorkDir(), "cache");
+        au.setCache(cache);
+        File cluster = new File(getWorkDir(), "cluster");
+        cluster.mkdirs();
+        au.setToDir(cluster);
+        au.createModules().setIncludes(".+");
+        au.execute();
+        assertTrue(new File(cluster, "modules/org-netbeans-api-annotations-common.jar").isFile());
+        assertTrue(new File(cache, "org.netbeans.api.annotations.common_1.3_nbms-and-javadoc-4296-on-091113.nbm").isFile());
+    }
+
     public File generateNBM (String name, String... files) throws IOException {
         List<String> filesAndContent = new ArrayList<String>();
         for (String s : files) {