View | Details | Raw Unified | Return to bug 141817 | Differences between
and this patch

Collapse All | Expand All

(-)a/.hgignore (-1 / +1 lines)
 Lines 19-25    Link Here 
19
^\.project$
19
^\.project$
20
^\.externalToolBuilders/.*$
20
^\.externalToolBuilders/.*$
21
^hs_err_pid[0-9]*\.log
21
^hs_err_pid[0-9]*\.log
22
^nbbuild/antsrc/org/netbeans/nbbuild/extlibs/external\.pyc$
22
^[^/]+/external/[^/]+\.(zip|jar|gz|bz2|gem|dll)$
23
^o.apache.tools.ant.module/external/lib$
23
^o.apache.tools.ant.module/external/lib$
24
^apisupport.project/src/org/netbeans/modules/apisupport/project/resources/sample-build-impl\.xml$
24
^apisupport.project/src/org/netbeans/modules/apisupport/project/resources/sample-build-impl\.xml$
25
^apisupport.project/src/org/netbeans/modules/apisupport/project/resources/sample-build\.xml$
25
^apisupport.project/src/org/netbeans/modules/apisupport/project/resources/sample-build\.xml$
(-)a/.hgundead (+1 lines)
 Lines 4-9    Link Here 
4
cnd.editor/build/classes/org/netbeans/modules/cnd/editor/Bundle.properties
4
cnd.editor/build/classes/org/netbeans/modules/cnd/editor/Bundle.properties
5
form/external/beansbinding-1.2.1-doc.zip
5
form/external/beansbinding-1.2.1-doc.zip
6
form/external/beansbinding-1.2.1.jar
6
form/external/beansbinding-1.2.1.jar
7
nbbuild/antsrc/org/netbeans/nbbuild/extlibs/external.pyc
7
nbbuild/velocity.log
8
nbbuild/velocity.log
8
websvc.saas.api/build/classes/org/netbeans/modules/websvc/saas/Bundle.properties
9
websvc.saas.api/build/classes/org/netbeans/modules/websvc/saas/Bundle.properties
9
websvc.saas.api/build/classes/org/netbeans/modules/websvc/saas/resources/layer.xml
10
websvc.saas.api/build/classes/org/netbeans/modules/websvc/saas/resources/layer.xml
(-)a/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/DeregisterExternalHook.java (+112 lines)
Line 0    Link Here 
1
/*
2
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3
 *
4
 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
5
 *
6
 * The contents of this file are subject to the terms of either the GNU
7
 * General Public License Version 2 only ("GPL") or the Common
8
 * Development and Distribution License("CDDL") (collectively, the
9
 * "License"). You may not use this file except in compliance with the
10
 * License. You can obtain a copy of the License at
11
 * http://www.netbeans.org/cddl-gplv2.html
12
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13
 * specific language governing permissions and limitations under the
14
 * License.  When distributing the software, include this License Header
15
 * Notice in each file and include the License file at
16
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
17
 * particular file as subject to the "Classpath" exception as provided
18
 * by Sun in the GPL Version 2 section of the License file that
19
 * accompanied this code. If applicable, add the following below the
20
 * License Header, with the fields enclosed by brackets [] replaced by
21
 * your own identifying information:
22
 * "Portions Copyrighted [year] [name of copyright owner]"
23
 *
24
 * If you wish your version of this file to be governed by only the CDDL
25
 * or only the GPL Version 2, indicate your decision by adding
26
 * "[Contributor] elects to include this software in this distribution
27
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
28
 * single choice of license, a recipient has the option to distribute
29
 * your version of this file under either the CDDL, the GPL Version 2 or
30
 * to extend the choice of license to its licensees as provided above.
31
 * However, if you add GPL Version 2 code and therefore, elected the GPL
32
 * Version 2 license, then the option applies only if the new code is
33
 * made subject to such option by the copyright holder.
34
 *
35
 * Contributor(s):
36
 *
37
 * Portions Copyrighted 2008 Sun Microsystems, Inc.
38
 */
39
40
package org.netbeans.nbbuild.extlibs;
41
42
import java.io.ByteArrayOutputStream;
43
import java.io.File;
44
import java.io.FileInputStream;
45
import java.io.FileOutputStream;
46
import java.io.IOException;
47
import java.io.InputStream;
48
import java.io.OutputStream;
49
import org.apache.tools.ant.BuildException;
50
import org.apache.tools.ant.Project;
51
import org.apache.tools.ant.Task;
52
53
/**
54
 * Issue #141817: remove external.py registration.
55
 */
56
public class DeregisterExternalHook extends Task {
57
58
    private File root;
59
    /** Location of NB source root. */
60
    public void setRoot(File root) {
61
        this.root = root;
62
    }
63
64
    @Override
65
    public void execute() throws BuildException {
66
        new File(root, "nbbuild/antsrc/org/netbeans/nbbuild/extlibs/external.pyc").delete();
67
        File[] repos = {
68
            root,
69
            new File(root, "contrib"),
70
        };
71
        for (File repo : repos) {
72
            File dotHg = new File(repo, ".hg");
73
            if (!dotHg.isDirectory()) {
74
                log(repo + " is not a Mercurial repository", Project.MSG_VERBOSE);
75
                continue;
76
            }
77
            try {
78
                File hgrc = new File(dotHg, "hgrc");
79
                ByteArrayOutputStream baos = new ByteArrayOutputStream();
80
                if (hgrc.isFile()) {
81
                    InputStream is = new FileInputStream(hgrc);
82
                    try {
83
                        byte[] buf = new byte[4096];
84
                        int read;
85
                        while ((read = is.read(buf)) != -1) {
86
                            baos.write(buf, 0, read);
87
                        }
88
                    } finally {
89
                        is.close();
90
                    }
91
                }
92
                String config = baos.toString();
93
                String newConfig = config.
94
                        replaceAll("(?m)^external *=.+\n", "").
95
                        replaceAll("(?m)^\\*/external/\\*\\.\\{zip,jar,gz,bz2,gem,dll\\} = (up|down)load:.+\n", "").
96
                        replaceAll("(^|\n)\n*(\\[(extensions|encode|decode)\\]\n+)+(?=\\[|$)", "$1");
97
                if (!newConfig.equals(config)) {
98
                    log("Unregister external hook from " + hgrc);
99
                    OutputStream os = new FileOutputStream(hgrc);
100
                    try {
101
                        os.write(config.getBytes());
102
                    } finally {
103
                        os.close();
104
                    }
105
                }
106
            } catch (IOException x) {
107
                throw new BuildException(x, getLocation());
108
            }
109
        }
110
    }
111
112
}
(-)a/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/DownloadBinaries.java (+332 lines)
Line 0    Link Here 
1
/*
2
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3
 *
4
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5
 *
6
 * The contents of this file are subject to the terms of either the GNU
7
 * General Public License Version 2 only ("GPL") or the Common
8
 * Development and Distribution License("CDDL") (collectively, the
9
 * "License"). You may not use this file except in compliance with the
10
 * License. You can obtain a copy of the License at
11
 * http://www.netbeans.org/cddl-gplv2.html
12
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13
 * specific language governing permissions and limitations under the
14
 * License.  When distributing the software, include this License Header
15
 * Notice in each file and include the License file at
16
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
17
 * particular file as subject to the "Classpath" exception as provided
18
 * by Sun in the GPL Version 2 section of the License file that
19
 * accompanied this code. If applicable, add the following below the
20
 * License Header, with the fields enclosed by brackets [] replaced by
21
 * your own identifying information:
22
 * "Portions Copyrighted [year] [name of copyright owner]"
23
 *
24
 * If you wish your version of this file to be governed by only the CDDL
25
 * or only the GPL Version 2, indicate your decision by adding
26
 * "[Contributor] elects to include this software in this distribution
27
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
28
 * single choice of license, a recipient has the option to distribute
29
 * your version of this file under either the CDDL, the GPL Version 2 or
30
 * to extend the choice of license to its licensees as provided above.
31
 * However, if you add GPL Version 2 code and therefore, elected the GPL
32
 * Version 2 license, then the option applies only if the new code is
33
 * made subject to such option by the copyright holder.
34
 *
35
 * Contributor(s):
36
 *
37
 * Portions Copyrighted 2007 Sun Microsystems, Inc.
38
 */
39
40
package org.netbeans.nbbuild.extlibs;
41
42
import java.io.BufferedReader;
43
import java.io.File;
44
import java.io.FileInputStream;
45
import java.io.FileOutputStream;
46
import java.io.IOException;
47
import java.io.InputStream;
48
import java.io.InputStreamReader;
49
import java.io.OutputStream;
50
import java.math.BigInteger;
51
import java.net.HttpURLConnection;
52
import java.net.MalformedURLException;
53
import java.net.URL;
54
import java.net.URLConnection;
55
import java.nio.channels.FileChannel;
56
import java.security.MessageDigest;
57
import java.security.NoSuchAlgorithmException;
58
import java.util.ArrayList;
59
import java.util.List;
60
import org.apache.tools.ant.BuildException;
61
import org.apache.tools.ant.DirectoryScanner;
62
import org.apache.tools.ant.Project;
63
import org.apache.tools.ant.Task;
64
import org.apache.tools.ant.types.FileSet;
65
import org.apache.tools.ant.util.FileUtils;
66
67
/**
68
 * Task to retrieve named files (generally large binaries such as ZIPs) from a repository.
69
 * Similar to a very simplified version of Ivy, but correctly handles binaries
70
 * with missing or irrelevant version numbers, since it is based on a hash of contents.
71
 * You keep one or more manifests under version control which enumerate files and their SHA-1 hashes.
72
 * Then just run this task to download any missing files.
73
 * Remember to specify the binaries as "ignorable" to your version control system.
74
 * You can also run it in a clean mode which will remove the binaries.
75
 * At the end of this source file is a sample CGI script and matching form which you can run on the server
76
 * to permit people to upload files to the correct repository paths.
77
 * Motivation: http://wiki.netbeans.org/wiki/view/HgMigration#section-HgMigration-Binaries
78
 */
79
public class DownloadBinaries extends Task {
80
81
    private File cache;
82
    /**
83
     * Location of per-user cache of already downloaded binaries.
84
     * Optional; no cache will be used if unset.
85
     * The directory will be created if it does not yet exist.
86
     */
87
    public void setCache(File cache) {
88
        this.cache = cache;
89
    }
90
91
    private String server;
92
    /**
93
     * URL prefix for the server repository.
94
     * Should generally include a trailing slash.
95
     * You may include multiple server URLs separated by spaces
96
     * in which case they will be tried in order.
97
     * To use a local repository, simply specify e.g. <code>file:/repo/</code> as the URL.
98
     */
99
    public void setServer(String server) {
100
        this.server = server;
101
    }
102
103
    private final List<FileSet> manifests = new ArrayList<FileSet>();
104
    /**
105
     * Add one or more manifests of files to download.
106
     * Each manifest is a text file; lines beginning with # are ignored.
107
     * All other lines must be entries of the form
108
     * <pre>
109
     * 0123456789ABCDEF something-1.0.jar
110
     * </pre>
111
     * consisting of an SHA-1 hash followed by a filename.
112
     * The filename is relative to the manifest, usually a simple basename.
113
     * If the file exists and has the specified hash, nothing is done.
114
     * If it has the wrong hash, the task aborts with an error message.
115
     * If it is missing, it is downloaded from the server (or copied from cache)
116
     * using a filename derived from the basename of the file in the manifest and its hash.
117
     * For example, the above line with a server of <code>http://nowhere.net/repo/</code>
118
     * would try to download
119
     * <pre>
120
     * http://nowhere.net/repo/0123456789ABCDEF-something-1.0.jar
121
     * </pre>
122
     * Any version number etc. in the filename is purely informational;
123
     * the "up to date" check is entirely based on the hash.
124
     */
125
    public void addManifest(FileSet manifest) {
126
        manifests.add(manifest);
127
    }
128
129
    private boolean clean;
130
    /**
131
     * If true, rather than creating binary files, will delete them.
132
     * Any cache is ignored in this case.
133
     * If a binary does not match its hash, the build is aborted:
134
     * the file might be a precious customized version and should not be blindly deleted.
135
     */
136
    public void setClean(boolean clean) {
137
        this.clean = clean;
138
    }
139
140
    @Override
141
    public void execute() throws BuildException {
142
        for (FileSet fs : manifests) {
143
            DirectoryScanner scanner = fs.getDirectoryScanner(getProject());
144
            File basedir = scanner.getBasedir();
145
            for (String include : scanner.getIncludedFiles()) {
146
                File manifest = new File(basedir, include);
147
                log("Scanning: " + manifest, Project.MSG_VERBOSE);
148
                try {
149
                    BufferedReader r = new BufferedReader(new InputStreamReader(new FileInputStream(manifest), "UTF-8"));
150
                    String line;
151
                    while ((line = r.readLine()) != null) {
152
                        if (line.startsWith("#")) {
153
                            continue;
154
                        }
155
                        if (line.trim().length() == 0) {
156
                            continue;
157
                        }
158
                        String[] hashAndFile = line.split(" ", 2);
159
                        if (hashAndFile.length < 2) {
160
                            throw new BuildException("Bad line '" + line + "' in " + manifest, getLocation());
161
                        }
162
                        File f = new File(manifest.getParentFile(), hashAndFile[1]);
163
                        if (!clean) {
164
                            if (!f.exists()) {
165
                                log("Creating " + f);
166
                                String cacheName = hashAndFile[0] + "-" + f.getName();
167
                                if (cache != null) {
168
                                    cache.mkdirs();
169
                                    File cacheFile = new File(cache, cacheName);
170
                                    if (!cacheFile.exists()) {
171
                                        download(cacheName, cacheFile);
172
                                    }
173
                                    try {
174
                                        FileUtils.getFileUtils().copyFile(cacheFile, f);
175
                                    } catch (IOException x) {
176
                                        throw new BuildException("Could not copy " + cacheFile + " to " + f + ": " + x, x, getLocation());
177
                                    }
178
                                } else {
179
                                    download(cacheName, f);
180
                                }
181
                            }
182
                            String actualHash = hash(f);
183
                            if (!actualHash.equals(hashAndFile[0])) {
184
                                throw new BuildException("File " + f + " requested by " + manifest + " to have hash " +
185
                                        hashAndFile[0] + " actually had hash " + actualHash, getLocation());
186
                            }
187
                            log("Have " + f + " with expected hash", Project.MSG_VERBOSE);
188
                        } else {
189
                            if (f.exists()) {
190
                                String actualHash = hash(f);
191
                                if (!actualHash.equals(hashAndFile[0])) {
192
                                    throw new BuildException("File " + f + " requested by " + manifest + " to have hash " +
193
                                            hashAndFile[0] + " actually had hash " + actualHash, getLocation());
194
                                }
195
                                log("Deleting " + f);
196
                                f.delete();
197
                            }
198
                        }
199
                    }
200
                } catch (IOException x) {
201
                    throw new BuildException("Could not open " + manifest + ": " + x, x, getLocation());
202
                }
203
            }
204
        }
205
    }
206
207
    private void download(String cacheName, File destination) {
208
        if (server == null) {
209
            throw new BuildException("Must specify a server to download files from", getLocation());
210
        }
211
        for (String prefix : server.split(" ")) {
212
            URL url;
213
            try {
214
                url = new URL(prefix + cacheName);
215
            } catch (MalformedURLException x) {
216
                throw new BuildException(x, getLocation());
217
            }
218
            try {
219
                URLConnection conn = url.openConnection();
220
                conn.connect();
221
                int code = HttpURLConnection.HTTP_OK;
222
                if (conn instanceof HttpURLConnection) {
223
                    code = ((HttpURLConnection) conn).getResponseCode();
224
                }
225
                if (code != HttpURLConnection.HTTP_OK) {
226
                    log("Skipping download from " + url + " due to response code " + code, Project.MSG_VERBOSE);
227
                }
228
                log("Downloading: " + url);
229
                InputStream is = conn.getInputStream();
230
                try {
231
                    OutputStream os = new FileOutputStream(destination);
232
                    try {
233
                        byte[] buf = new byte[4096];
234
                        int read;
235
                        while ((read = is.read(buf)) != -1) {
236
                            os.write(buf, 0, read);
237
                        }
238
                        os.close();
239
                    } catch (IOException x) {
240
                        destination.delete();
241
                        throw x;
242
                    }
243
                } finally {
244
                    is.close();
245
                }
246
                return;
247
            } catch (IOException x) {
248
                log("Could not download " + url + " to " + destination + ": " + x, Project.MSG_WARN);
249
            }
250
        }
251
        throw new BuildException("Could not download " + cacheName + " from " + server, getLocation());
252
    }
253
254
    private String hash(File f) {
255
        MessageDigest digest;
256
        try {
257
            digest = MessageDigest.getInstance("SHA-1");
258
        } catch (NoSuchAlgorithmException x) {
259
            throw new BuildException(x, getLocation());
260
        }
261
        try {
262
            FileInputStream is = new FileInputStream(f);
263
            try {
264
                digest.update(is.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, f.length()));
265
            } finally {
266
                is.close();
267
            }
268
        } catch (IOException x) {
269
            throw new BuildException("Could not get hash for " + f + ": " + x, x, getLocation());
270
        }
271
        return String.format("%040X", new BigInteger(1, digest.digest()));
272
    }
273
274
}
275
276
/*
277
278
Sample upload script (edit repository location as needed):
279
280
#!/usr/bin/env ruby
281
repository = '/tmp/repository'
282
require 'cgi'
283
require 'digest/sha1'
284
require 'date'
285
cgi = CGI.new
286
begin
287
  if cgi.request_method == 'POST'
288
    value = cgi['file']
289
    content = value.read
290
    name = value.original_filename.gsub(/\.\.|[^a-zA-Z0-9._+-]/, '_')
291
    sha1 = Digest::SHA1.hexdigest(content).upcase
292
    open("#{repository}/#{sha1}-#{name}", "w") do |f|
293
      f.write content
294
    end
295
    open("#{repository}/log", "a") do |f|
296
      f << "#{DateTime.now.to_s} #{sha1}-#{name} #{cgi.remote_user}\n"
297
    end
298
    cgi.out do <<RESPONSE
299
<html>
300
<head>
301
<title>Uploaded #{name}</title>
302
</head>
303
<body>
304
<p>Uploaded. Add to your manifest:</p>
305
<pre>#{sha1} #{name}</pre>
306
</body>
307
</html>
308
RESPONSE
309
    end
310
  else
311
    cgi.out do <<FORM
312
<html>
313
<head>
314
<title>Upload a Binary</title>
315
</head>
316
<body>
317
<form method="POST" action="" enctype="multipart/form-data">
318
<input type="file" name="file">
319
<input type="submit" value="Upload">
320
</form>
321
</body>
322
</html>
323
FORM
324
    end
325
  end
326
rescue
327
  cgi.out do
328
    "Caught an exception: #{$!}"
329
  end
330
end
331
332
 */
(-)a/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/RegisterExternalHook.java (-284 lines)
 Lines 1-284    Link Here 
1
/*
2
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3
 *
4
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
5
 *
6
 * The contents of this file are subject to the terms of either the GNU
7
 * General Public License Version 2 only ("GPL") or the Common
8
 * Development and Distribution License("CDDL") (collectively, the
9
 * "License"). You may not use this file except in compliance with the
10
 * License. You can obtain a copy of the License at
11
 * http://www.netbeans.org/cddl-gplv2.html
12
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
13
 * specific language governing permissions and limitations under the
14
 * License.  When distributing the software, include this License Header
15
 * Notice in each file and include the License file at
16
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
17
 * particular file as subject to the "Classpath" exception as provided
18
 * by Sun in the GPL Version 2 section of the License file that
19
 * accompanied this code. If applicable, add the following below the
20
 * License Header, with the fields enclosed by brackets [] replaced by
21
 * your own identifying information:
22
 * "Portions Copyrighted [year] [name of copyright owner]"
23
 *
24
 * If you wish your version of this file to be governed by only the CDDL
25
 * or only the GPL Version 2, indicate your decision by adding
26
 * "[Contributor] elects to include this software in this distribution
27
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
28
 * single choice of license, a recipient has the option to distribute
29
 * your version of this file under either the CDDL, the GPL Version 2 or
30
 * to extend the choice of license to its licensees as provided above.
31
 * However, if you add GPL Version 2 code and therefore, elected the GPL
32
 * Version 2 license, then the option applies only if the new code is
33
 * made subject to such option by the copyright holder.
34
 *
35
 * Contributor(s):
36
 *
37
 * Portions Copyrighted 2008 Sun Microsystems, Inc.
38
 */
39
40
package org.netbeans.nbbuild.extlibs;
41
42
import java.io.BufferedReader;
43
import java.io.ByteArrayOutputStream;
44
import java.io.File;
45
import java.io.FileInputStream;
46
import java.io.FileOutputStream;
47
import java.io.IOException;
48
import java.io.InputStream;
49
import java.io.InputStreamReader;
50
import java.io.OutputStream;
51
import java.util.ArrayList;
52
import java.util.Arrays;
53
import java.util.List;
54
import java.util.regex.Matcher;
55
import java.util.regex.Pattern;
56
import org.apache.tools.ant.BuildException;
57
import org.apache.tools.ant.Project;
58
import org.apache.tools.ant.Task;
59
import org.netbeans.nbbuild.HgExec;
60
61
/**
62
 * Registers external.py as a Mercurial encode/decode hook, if appropriate.
63
 */
64
public class RegisterExternalHook extends Task {
65
66
    public RegisterExternalHook() {}
67
68
    private File root;
69
    /** Location of NB source root. */
70
    public void setRoot(File root) {
71
        this.root = root;
72
    }
73
74
    private File hook;
75
    /** Extension hook to register. */
76
    public void setHook(File hook) {
77
        this.hook = hook;
78
    }
79
80
    private static final String BINARIES = "*/external/*.{zip,jar,gz,bz2,gem,dll}";
81
82
    @Override
83
    public void execute() throws BuildException {
84
        File dotHg = new File(root, ".hg");
85
        if (!dotHg.isDirectory()) {
86
            log(root + " is not a Mercurial repository", Project.MSG_VERBOSE);
87
            return;
88
        }
89
        List<String> hgExecutable = HgExec.hgExecutable();
90
        try {
91
            List<String> commandAndArgs = new ArrayList<String>(hgExecutable);
92
            commandAndArgs.add("showconfig");
93
            Process p = new ProcessBuilder(commandAndArgs).directory(root).start();
94
            BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()));
95
            String line;
96
            while ((line = r.readLine()) != null) {
97
                if (line.matches("(en|de)code\\.\\*\\*=.*")) {
98
                    // XXX try to fix it instead
99
                    // Tricky because no trivial way to find where Mercurial.ini is.
100
                    // If Hg installed to default Python path, could use:
101
                    // python -c 'from mercurial import util; print util.rcpath()'
102
                    // ({}** chosen because it sorts after */external/*.ext so will not take precedence.)
103
                    throw new BuildException(
104
                            "An existing global encode/decode hook will conflict with " + hook.getName() + "\n" +
105
                            "You must edit your Mercurial.ini and change '** = ...' in encode/decode sections to '{}** = ...'", getLocation());
106
                }
107
            }
108
        } catch (IOException x) {
109
            log("Could not verify Hg configuration: " + x, Project.MSG_WARN);
110
        }
111
        File[] repos = {
112
            root,
113
            new File(root, "contrib"),
114
        };
115
        for (File repo : repos) {
116
            dotHg = new File(repo, ".hg");
117
            if (!dotHg.isDirectory()) {
118
                log(repo + " is not a Mercurial repository", Project.MSG_VERBOSE);
119
                continue;
120
            }
121
            try {
122
                File hgrc = new File(dotHg, "hgrc");
123
                ByteArrayOutputStream baos = new ByteArrayOutputStream();
124
                if (hgrc.isFile()) {
125
                    InputStream is = new FileInputStream(hgrc);
126
                    try {
127
                        byte[] buf = new byte[4096];
128
                        int read;
129
                        while ((read = is.read(buf)) != -1) {
130
                            baos.write(buf, 0, read);
131
                        }
132
                    } finally {
133
                        is.close();
134
                    }
135
                }
136
                String config = baos.toString();
137
                if (Pattern.compile("(?m)^external *= *" + Pattern.quote(hook.getAbsolutePath()) + "$").matcher(config).find()) {
138
                    log("Hook already registered in " + hgrc, Project.MSG_VERBOSE);
139
                    continue;
140
                }
141
                if (Pattern.compile("(?m)^external *=").matcher(config).find()) {
142
                    log("Hook already registered in " + hgrc + " in wrong location, correcting...", Project.MSG_WARN);
143
                    config = config.replaceFirst("(?m)^external *=.*$", "external = " + hook.toString().replaceAll("\\\\", "\\\\\\\\"));
144
                } else {
145
                    log("Registering hook in " + hgrc);
146
                    String nl = System.getProperty("line.separator");
147
                    if (config.length() > 0) {
148
                        config += nl;
149
                    }
150
                    config += "[extensions]" + nl + "external = " + hook + nl + "[encode]" + nl;
151
                    Matcher m = Pattern.compile("https://[a-zA-Z0-9_]+(:[^@]+)?@hg.netbeans.org/").matcher(config);
152
                    if (m.find()) {
153
                        config += BINARIES + " = upload: " + m.group() + "binaries/upload" + nl;
154
                    } else {
155
                        config += "# To preauthenticate, use: https://jhacker:secret@hg.netbeans.org/binaries/upload" + nl +
156
                                BINARIES + " = upload: https://hg.netbeans.org/binaries/upload" + nl;
157
                    }
158
                    config += "[decode]" + nl + BINARIES + " = download: http://hg.netbeans.org/binaries/" + nl;
159
                }
160
                // XXX should also register hooks.pretxncommit.crlf=python:hgext.win32text.forbidcrlf (for Hg 1.0+ only)
161
                OutputStream os = new FileOutputStream(hgrc);
162
                try {
163
                    os.write(config.getBytes());
164
                } finally {
165
                    os.close();
166
                }
167
            } catch (IOException x) {
168
                throw new BuildException(x, getLocation());
169
            }
170
            try {
171
                log("Looking for external binaries in " + repo + " which need to be checked out again using decoder");
172
                List<String> commandAndArgs = new ArrayList<String>(hgExecutable);
173
                commandAndArgs.add("locate");
174
                commandAndArgs.add(BINARIES);
175
                Process p = new ProcessBuilder(commandAndArgs).directory(repo).start();
176
                BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()));
177
                boolean doCheckout = false;
178
                String binary;
179
                while ((binary = r.readLine()) != null) {
180
                    File f = new File(repo, binary);
181
                    log("Examining: " + f, Project.MSG_DEBUG);
182
                    if (!f.isFile()) {
183
                        log("No such file checked out: " + f, Project.MSG_VERBOSE);
184
                        doCheckout = true; // in case raw files got deleted last time
185
                        continue;
186
                    }
187
                    byte[] rawheader = "<<<EXTERNAL ".getBytes();
188
                    byte[] buf = new byte[rawheader.length];
189
                    InputStream is = new FileInputStream(f);
190
                    try {
191
                        is.read(buf);
192
                    } finally {
193
                        is.close();
194
                    }
195
                    if (Arrays.equals(buf, rawheader)) {
196
                        if (!f.delete()) {
197
                            throw new IOException("Failed to delete " + f);
198
                        }
199
                        doCheckout = true;
200
                    } else {
201
                        log("Does not look like a raw file: " + f, Project.MSG_VERBOSE);
202
                    }
203
                }
204
                if (doCheckout) {
205
                    log("Will check out external binaries fresh");
206
                    log("(This could take a while as binaries are downloaded into your cache.)");
207
                    HgExec x = new HgExec();
208
                    x.setProject(getProject());
209
                    x.createArg().setValue("checkout");
210
                    x.setDir(repo);
211
                    x.setFailonerror(true);
212
                    x.init();
213
                    x.setLocation(getLocation());
214
                    x.execute();
215
                }
216
            } catch (IOException x) {
217
                throw new BuildException(x + "; failed to run:\n" +
218
                        "hg -R " + repo + " locate \"" + BINARIES + "\"\n" +
219
                        "Please run this yourself, delete any matching files, then run:\n" +
220
                        "hg -R " + repo + " checkout\n" +
221
                        "and restart the build.", getLocation());
222
            }
223
        }
224
    }
225
226
}
227
228
/*
229
230
Sample upload script (edit repository location as needed):
231
232
#!/usr/bin/env ruby
233
repository = '/tmp/repository'
234
require 'cgi'
235
require 'digest/sha1'
236
require 'date'
237
cgi = CGI.new
238
begin
239
  if cgi.request_method == 'POST'
240
    value = cgi['file']
241
    content = value.read
242
    name = value.original_filename.gsub(/\.\.|[^a-zA-Z0-9._+-]/, '_')
243
    sha1 = Digest::SHA1.hexdigest(content).upcase
244
    open("#{repository}/#{sha1}-#{name}", "w") do |f|
245
      f.write content
246
    end
247
    open("#{repository}/log", "a") do |f|
248
      f << "#{DateTime.now.to_s} #{sha1}-#{name} #{cgi.remote_user}\n"
249
    end
250
    cgi.out do <<RESPONSE
251
<html>
252
<head>
253
<title>Uploaded #{name}</title>
254
</head>
255
<body>
256
<p>Uploaded as:</p>
257
<pre>#{sha1} #{name}</pre>
258
</body>
259
</html>
260
RESPONSE
261
    end
262
  else
263
    cgi.out do <<FORM
264
<html>
265
<head>
266
<title>Upload a Binary</title>
267
</head>
268
<body>
269
<form method="POST" action="" enctype="multipart/form-data">
270
<input type="file" name="file">
271
<input type="submit" value="Upload">
272
</form>
273
</body>
274
</html>
275
FORM
276
    end
277
  end
278
rescue
279
  cgi.out do
280
    "Caught an exception: #{$!}"
281
  end
282
end
283
284
 */
(-)a/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/external.py (-222 lines)
 Lines 1-222    Link Here 
1
import os, re, urllib2, sha, inspect, sys
2
from mercurial import util, httprepo, ui
3
4
if sys.version_info < (2, 4, 4):
5
    ui.ui().warn('Warning: external hook requires Python 2.4.4+ (2.5.1 preferred)\n')
6
7
# Workaround for a Python bug (in linecache.py?):
8
# http://bugs.python.org/issue1728
9
# http://bugs.python.org/issue1665
10
# https://bugs.launchpad.net/ubuntu/+source/python-defaults/+bug/70902
11
# http://mail.python.org/pipermail/python-bugs-list/2007-March/037472.html
12
_inspect_findsource_orig = inspect.findsource
13
def _inspect_findsource_robust(object):
14
    try:
15
        return _inspect_findsource_orig(object)
16
    except IndexError:
17
        raise IOError('workaround for linecache bug')
18
inspect.findsource=_inspect_findsource_robust
19
20
# Compatibility for Hg not including f8ad3b76e923:
21
def _findparamvalue(function, param):
22
    for framerec in inspect.getouterframes(inspect.currentframe()):
23
        if framerec[3] == function:
24
            funcvars = inspect.getargvalues(framerec[0])[3]
25
            if not param in funcvars:
26
                raise util.Abort('No parameter named %s in function %s' % (param, function))
27
            return funcvars[param]
28
    raise util.Abort('No function named %s in stack' % function)
29
def _filename(f):
30
    if f:
31
        return f
32
    return _findparamvalue('_filter', 'filename')
33
def _ui(u):
34
    if u:
35
        return u
36
    return _findparamvalue('_filter', 'self').ui
37
def _repo(r):
38
    if r:
39
        return r
40
    return _findparamvalue('_filter', 'self')
41
42
def _trim(filename):
43
    return os.path.basename(filename)
44
def _cachedir():
45
    if 'HGEXTERNALCACHE' in os.environ:
46
        d = os.environ['HGEXTERNALCACHE']
47
    else:
48
        d = os.path.expanduser('~/.hgexternalcache')
49
    if not os.path.exists(d):
50
        os.makedirs(d)
51
    return d
52
def _sha1hash(data):
53
    return sha.new(data).hexdigest().upper()
54
55
def _download_to_cache(url, ui, sha1, filename, cachefile):
56
    ui.status('Downloading %s\n' % url)
57
    handle = urllib2.urlopen(url)
58
    data = handle.read()
59
    handle.close()
60
    if sha1 != _sha1hash(data):
61
        raise util.Abort('hash mismatch in %s' % filename)
62
    # XXX acquire write lock, or write to temp file and do atomic rename
63
    handle = open(cachefile, 'wb')
64
    handle.write(data)
65
    handle.close()
66
    return data
67
68
def download(s, cmd, ui=None, filename=None, **kwargs):
69
    # To support dev versions of Hg with f8ad3b76e923 but not f3a8b5360100:
70
    cmd = re.sub(r'^download: *', '', cmd)
71
    filename = _filename(filename)
72
    ui = _ui(ui)
73
    n = _trim(filename)
74
    m = re.match(r'^<<<EXTERNAL (([0-9A-F]{40})-([a-zA-Z0-9._+-]+))>>>\n$', s)
75
    if not m:
76
        raise util.Abort('malformed contents in %s' % filename)
77
    if n != m.group(3):
78
        raise util.Abort('incorrect basename in %s: %s' % (filename, m.group(3)))
79
    cachefile = os.path.join(_cachedir(), m.group(1))
80
    if os.path.exists(cachefile):
81
        handle = open(cachefile, 'rb')
82
        data = handle.read()
83
        handle.close()
84
        if m.group(2) != _sha1hash(data):
85
            try:
86
                os.remove(cachefile)
87
            except:
88
                pass
89
            raise util.Abort('hash mismatch in %s' % filename)
90
    else:
91
        data = _download_to_cache(url = cmd + m.group(1),
92
                                  ui = ui,
93
                                  sha1 = m.group(2),
94
                                  filename = filename,
95
                                  cachefile = cachefile)
96
    return data
97
98
def upload(s, cmd, ui=None, repo=None, filename=None, **kwargs):
99
    cmd = re.sub(r'^upload: *', '', cmd)
100
    filename = _filename(filename)
101
    ui = _ui(ui)
102
    repo = _repo(repo)
103
    n = _trim(filename)
104
    if not re.match(r'[a-zA-Z0-9._+-]+', n):
105
        raise util.Abort('unsupported file basename: %s', filename)
106
    sha1 = _sha1hash(s)
107
    full = '%s-%s' % (sha1, n)
108
    cachefile = os.path.join(_cachedir(), full)
109
    if not os.path.exists(cachefile):
110
        # XXX forces download and upload URLs to be related;
111
        # would be better to get download URL from download filter,
112
        # but that info is not trivially accessible here:
113
        m = re.match(r'(https?://)(([^:@]+)(:([^@]+))?@)?(.+)upload', cmd)
114
        if not m:
115
            raise util.Abort('malformed upload URL: %s' % cmd)
116
        downloadurl = m.group(1) + m.group(6) + full
117
        url = m.group(1) + m.group(6) + 'upload'
118
        try:
119
            _download_to_cache(url = downloadurl,
120
                               ui = ui,
121
                               sha1 = sha1,
122
                               filename = filename,
123
                               cachefile = cachefile)
124
            ui.status('No need to upload %s: %s already exists\n' % (filename, downloadurl))
125
        except IOError:
126
            # probably a 404, this is normal
127
            pm = httprepo.passwordmgr(ui)
128
            pm.add_password(None, url, m.group(3), m.group(5))
129
            ui.status('Uploading %s to %s (%s Kb)\n' % (filename, url, len(s) / 1024))
130
            auth = urllib2.HTTPBasicAuthHandler(pm)
131
            try:
132
                data = {'file': repo.wfile(filename)}
133
                # XXX support proxies; look at httprepo.httprepository.__init__
134
                # or http://www.hackorama.com/python/upload.shtml
135
                urllib2.build_opener(MultipartPostHandler, auth).open(url, data).close()
136
            except IOError, err:
137
                raise util.Abort('Problem uploading %s to %s (try it manually using a web browser): %s\n' % (filename, url, err))
138
            # Now ensure that the upload actually worked, by downloading:
139
            _download_to_cache(url = downloadurl,
140
                               ui = ui,
141
                               sha1 = sha1,
142
                               filename = filename,
143
                               cachefile = cachefile)
144
    return '<<<EXTERNAL %s>>>\n' % full
145
146
def reposetup(ui, repo):
147
    if not repo.local():
148
        return
149
    for name, fn in {'download:': download, 'upload:': upload}.iteritems():
150
        try:
151
            # Hg 0.9.6+ (with f8ad3b76e923):
152
            repo.adddatafilter(name, fn)
153
        except AttributeError:
154
            # Hg 0.9.5:
155
            util.filtertable[name] = fn
156
157
# --- FROM http://odin.himinbi.org/MultipartPostHandler.py ---
158
# (with some bug fixes!)
159
import urllib
160
import urllib2
161
import mimetools, mimetypes
162
import os, stat
163
164
class Callable:
165
    def __init__(self, anycallable):
166
        self.__call__ = anycallable
167
168
# Controls how sequences are uncoded. If true, elements may be given multiple values by
169
#  assigning a sequence.
170
doseq = 1
171
172
class MultipartPostHandler(urllib2.BaseHandler):
173
    handler_order = urllib2.HTTPHandler.handler_order - 10 # needs to run first
174
175
    def http_request(self, request):
176
        data = request.get_data()
177
        if data is not None and type(data) != str:
178
            v_files = []
179
            v_vars = []
180
            for(key, value) in data.items():
181
                try:
182
                     value.name
183
                     v_files.append((key, value))
184
                except AttributeError:
185
                     v_vars.append((key, value))
186
187
            if len(v_files) == 0:
188
                data = urllib.urlencode(v_vars, doseq)
189
            else:
190
                boundary, data = self.multipart_encode(v_vars, v_files)
191
                contenttype = 'multipart/form-data; boundary=%s' % boundary
192
                if(request.has_header('Content-Type')
193
                   and request.get_header('Content-Type').find('multipart/form-data') != 0):
194
                    print "Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data')
195
                request.add_unredirected_header('Content-Type', contenttype)
196
197
            request.add_data(data)
198
        return request
199
200
    def multipart_encode(vars, files, boundary = None, buffer = None):
201
        if boundary is None:
202
            boundary = mimetools.choose_boundary()
203
        if buffer is None:
204
            buffer = ''
205
        for(key, value) in vars:
206
            buffer += '--%s\r\n' % boundary
207
            buffer += 'Content-Disposition: form-data; name="%s"' % key
208
            buffer += '\r\n\r\n' + value + '\r\n'
209
        for(key, fd) in files:
210
            filename = os.path.basename(fd.name)
211
            contenttype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
212
            buffer += '--%s\r\n' % boundary
213
            buffer += 'Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename)
214
            buffer += 'Content-Type: %s\r\n' % contenttype
215
            fd.seek(0)
216
            buffer += '\r\n' + fd.read() + '\r\n'
217
        buffer += '--%s--\r\n\r\n' % boundary
218
        return boundary, buffer
219
    multipart_encode = Callable(multipart_encode)
220
221
    https_request = http_request
222
# ------------------------------------------------------------
(-)a/nbbuild/build.xml (-3 / +18 lines)
 Lines 98-110    Link Here 
98
        </classpath>
98
        </classpath>
99
        <include name="org/netbeans/nbbuild/HgExec.java"/>
99
        <include name="org/netbeans/nbbuild/HgExec.java"/>
100
        <include name="org/netbeans/nbbuild/ValidateHgConfiguration.java"/>
100
        <include name="org/netbeans/nbbuild/ValidateHgConfiguration.java"/>
101
        <include name="org/netbeans/nbbuild/extlibs/RegisterExternalHook.java"/>
101
        <include name="org/netbeans/nbbuild/extlibs/DeregisterExternalHook.java"/>
102
        <include name="org/netbeans/nbbuild/extlibs/DownloadBinaries.java"/>
102
        <compilerarg line="-Xlint -Xlint:-serial"/>
103
        <compilerarg line="-Xlint -Xlint:-serial"/>
103
    </javac>
104
    </javac>
104
    <taskdef name="validate-hg-configuration" classname="org.netbeans.nbbuild.ValidateHgConfiguration" classpath="build/antclasses"/>
105
    <taskdef name="validate-hg-configuration" classname="org.netbeans.nbbuild.ValidateHgConfiguration" classpath="build/antclasses"/>
105
    <validate-hg-configuration root=".."/>
106
    <validate-hg-configuration root=".."/>
106
    <taskdef name="registerexternalhook" classname="org.netbeans.nbbuild.extlibs.RegisterExternalHook" classpath="build/antclasses"/>
107
    <taskdef name="deregisterexternalhook" classname="org.netbeans.nbbuild.extlibs.DeregisterExternalHook" classpath="build/antclasses"/>
107
    <registerexternalhook root=".." hook="antsrc/org/netbeans/nbbuild/extlibs/external.py"/>
108
    <deregisterexternalhook root=".."/>
109
    <taskdef name="downloadbinaries" classname="org.netbeans.nbbuild.extlibs.DownloadBinaries" classpath="build/antclasses"/>
110
    <downloadbinaries cache="${binaries.cache}" server="${binaries.server}">
111
        <manifest dir="${nb_all}">
112
            <include name="*/external/binaries-list"/>
113
            <include name="contrib/*/external/binaries-list"/>
114
        </manifest>
115
    </downloadbinaries>
108
116
109
    <echo message="Bootstrapping NetBeans-specific Ant extensions..."/>
117
    <echo message="Bootstrapping NetBeans-specific Ant extensions..."/>
110
    <path id="bootstrap-cp">
118
    <path id="bootstrap-cp">
 Lines 1371-1376    Link Here 
1371
        <include name="contrib/*/external/build.xml"/>
1379
        <include name="contrib/*/external/build.xml"/>
1372
      </fileset>
1380
      </fileset>
1373
    </subant>
1381
    </subant>
1382
    <taskdef name="downloadbinaries" classname="org.netbeans.nbbuild.extlibs.DownloadBinaries" classpath="nbantext.jar"/>
1383
    <downloadbinaries cache="${binaries.cache}" server="${binaries.server}" clean="true">
1384
        <manifest dir="${nb_all}">
1385
            <include name="*/external/binaries-list"/>
1386
            <include name="contrib/*/external/binaries-list"/>
1387
        </manifest>
1388
    </downloadbinaries>
1374
  </target>
1389
  </target>
1375
1390
1376
  <target name="-real-clean" depends="-cleanall,localclean,-clean-external">
1391
  <target name="-real-clean" depends="-cleanall,localclean,-clean-external">
(-)a/nbbuild/default-properties.xml (+3 lines)
 Lines 73-77    Link Here 
73
  <property file="${moduleCluster.file}"/>
73
  <property file="${moduleCluster.file}"/>
74
74
75
  <property name="test.dist.dir" location="${nb_all}/nbbuild/build/testdist"/>
75
  <property name="test.dist.dir" location="${nb_all}/nbbuild/build/testdist"/>
76
  
77
  <property name="binaries.cache" location="${user.home}/.hgexternalcache"/>
78
  <property name="binaries.server" location="http://hg.netbeans.org/binaries/"/>
76
79
77
</project>
80
</project>
(-)a/nbbuild/templates/projectized.xml (+6 lines)
 Lines 71-76    Link Here 
71
    </target>
71
    </target>
72
72
73
    <target name="build-init" depends="basic-init,jdk-5-check">
73
    <target name="build-init" depends="basic-init,jdk-5-check">
74
        <taskdef name="downloadbinaries" classname="org.netbeans.nbbuild.extlibs.DownloadBinaries" classpath="${nb_all}/nbbuild/nbantext.jar"/>
75
        <downloadbinaries cache="${binaries.cache}" server="${binaries.server}">
76
            <manifest dir=".">
77
                <include name="external/binaries-list"/>
78
            </manifest>
79
        </downloadbinaries>
74
        <property name="public.package.jar.dir" location="${nb_all}/nbbuild/build/public-package-jars"/>
80
        <property name="public.package.jar.dir" location="${nb_all}/nbbuild/build/public-package-jars"/>
75
        <mkdir dir="${public.package.jar.dir}"/>
81
        <mkdir dir="${public.package.jar.dir}"/>
76
        <property name="test.dist.dir" location="${nb_all}/nbbuild/build/testdist"/>
82
        <property name="test.dist.dir" location="${nb_all}/nbbuild/build/testdist"/>

Return to bug 141817
By use of this website, you agree to the NetBeans Policies and Terms of Use. © 2014, Oracle Corporation and/or its affiliates. Sponsored by Oracle logo