# HG changeset patch # User Vladimir Kvashin # Date 1481013893 -10800 # Branch release82 # Node ID 220bce795781ae5a7fd92a1feb23997f8843da1f # Parent 2b4e36d7aeb62955bcd31c172802a4bdec679e75 Remote FS suspend/resume - fixing #247673 - "New Project" wizard works too slowly in FullRemote case diff -r 2b4e36d7aeb6 -r 220bce795781 cnd.makeproject/src/org/netbeans/modules/cnd/makeproject/api/wizards/MakeSampleProjectGenerator.java --- a/cnd.makeproject/src/org/netbeans/modules/cnd/makeproject/api/wizards/MakeSampleProjectGenerator.java Mon Dec 05 19:15:54 2016 +0300 +++ b/cnd.makeproject/src/org/netbeans/modules/cnd/makeproject/api/wizards/MakeSampleProjectGenerator.java Tue Dec 06 11:44:53 2016 +0300 @@ -54,6 +54,7 @@ import java.io.InterruptedIOException; import java.io.OutputStream; import java.io.OutputStreamWriter; +import java.net.ConnectException; import java.net.URL; import java.util.ArrayList; import java.util.Collection; @@ -91,6 +92,7 @@ import org.openide.filesystems.FileObject; import org.openide.filesystems.FileUtil; import org.openide.filesystems.URLMapper; +import org.openide.util.Exceptions; import org.openide.util.Lookup; import org.openide.util.Utilities; import org.openide.xml.XMLUtil; @@ -448,10 +450,20 @@ prjLoc = FileUtil.createFolder(prjParams.getSourceFileSystem().getRoot(), projectFolderPath); } unzip(inputStream, prjLoc); - postProcessProject(prjLoc, prjParams.getProjectName(), prjParams); - customPostProcessProject(prjLoc, prjParams.getProjectName(), prjParams); - - prjLoc.refresh(false); + FileSystemProvider.suspendWritesUpload(prjLoc); + try { + postProcessProject(prjLoc, prjParams.getProjectName(), prjParams); + customPostProcessProject(prjLoc, prjParams.getProjectName(), prjParams); + } finally { + try { + FileSystemProvider.resumeWritesUpload(prjLoc); + } catch (InterruptedException ex) { + InterruptedIOException iioe = new InterruptedIOException(ex.getMessage()); + iioe.setStackTrace(ex.getStackTrace()); + throw iioe; + } + prjLoc.refresh(false); + } return Collections.singleton(prjLoc); } diff -r 2b4e36d7aeb6 -r 220bce795781 dlight.remote.impl/src/org/netbeans/modules/remote/impl/fs/RemoteDirectory.java --- a/dlight.remote.impl/src/org/netbeans/modules/remote/impl/fs/RemoteDirectory.java Mon Dec 05 19:15:54 2016 +0300 +++ b/dlight.remote.impl/src/org/netbeans/modules/remote/impl/fs/RemoteDirectory.java Tue Dec 06 11:44:53 2016 +0300 @@ -67,7 +67,9 @@ import java.util.logging.Level; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; import org.netbeans.api.annotations.common.NonNull; +import org.netbeans.modules.dlight.libs.common.DLightLibsCommonLogger; import org.netbeans.modules.dlight.libs.common.PathUtilities; import org.netbeans.modules.nativeexecution.api.ExecutionEnvironment; import org.netbeans.modules.nativeexecution.api.util.CommonTasksSupport; @@ -246,6 +248,13 @@ } private RemoteFileObject create(String name, boolean directory, RemoteFileObjectBase orig) throws IOException { + if(isSuspendedWritesUpload()) { + // TODO: shouldn't it be done even if not suspended? + RemoteFileObject fo = getFileObject(name, new HashSet()); + if (fo != null) { + return fo; + } + } // Have to comment this out since NB does lots of stuff in the UI thread and I have no way to control this :( // RemoteLogger.assertNonUiThread("Remote file operations should not be done in UI thread"); String path = getPath() + '/' + name; @@ -1461,9 +1470,71 @@ } } - void uploadAndUnzip(InputStream zipStream) throws ConnectException, InterruptedException, IOException { + boolean isSuspendedWritesUpload() { + for(RemoteFileObjectBase fo = this; fo != null; fo = fo.getParent()) { + if (fo.getFlag(MASK_SUSPEND_WRITES)) { + return true; + } + } + return false; + } + + boolean addSuspendedFile(RemotePlainFile fo) { + for(RemoteFileObjectBase parent = this; parent != null; parent = parent.getParent()) { + if (parent.getFlag(MASK_SUSPEND_WRITES)) { + if (parent instanceof RemoteDirectory) { + getFileSystem().addSuspendedFile((RemoteDirectory) parent, fo); + return true; + } + } + } + return false; + } + + void suspendWritesUpload() { + setFlag(MASK_SUSPEND_WRITES, true); + } + + void resumeWritesUpload() throws IOException, InterruptedException, ConnectException { + setFlag(MASK_SUSPEND_WRITES, false); final ExecutionEnvironment env = getExecutionEnvironment(); if (!ConnectionManager.getInstance().isConnectedTo(env)) { + throw RemoteExceptions.createConnectException(RemoteFileSystemUtils.getConnectExceptionMessage(env)); + } + Set files = getFileSystem().removeSuspendedFiles(this); + if (files == null || files.isEmpty()) { + return; + } + File zipFile = File.createTempFile("rfs_local", ".zip"); + try { + try (ZipOutputStream zipStream = new ZipOutputStream(new FileOutputStream(zipFile))) { + for (RemotePlainFile fo : files) { + String path = fo.getPath(); + if (path.startsWith(getPath()) && path.length() > getPath().length()+1 && path.charAt(getPath().length()) == '/') { + String relPath = path.substring(getPath().length() + 1); + ZipEntry entry = new ZipEntry(relPath); + //entry.setTime(file.lastModified()); + zipStream.putNextEntry(entry); + try (FileInputStream fis = new FileInputStream(fo.getCache())) { + FileUtil.copy(fis, zipStream); + } + } else { + // TODO: log it! + } + } + } + uploadAndUnzip(zipFile); + for (RemotePlainFile fo : files) { + fo.setPendingRemoteDelivery(false); + } + } finally { + zipFile.delete(); + } + } + + void uploadAndUnzip(InputStream zipStream) throws ConnectException, InterruptedException, IOException { + final ExecutionEnvironment env = getExecutionEnvironment(); + if (!ConnectionManager.getInstance().isConnectedTo(env)) { zipStream.close(); throw RemoteExceptions.createConnectException(RemoteFileSystemUtils.getConnectExceptionMessage(env)); } @@ -1477,33 +1548,7 @@ } finally { zipStream.close(); } - // Copy local zip file to remote - FileObject remoteZipFO = getFileSystem().createTempFile(this.getOwnerFileObject(), RemoteFileSystem.TEMP_ZIP_PREFIX, ".zip", false); //NOI18N - try (OutputStream os = remoteZipFO.getOutputStream()) { - try (InputStream is = new FileInputStream(localZipFO)) { - FileUtil.copy(is, os); - } - } - StringBuilder script = new StringBuilder("unzip -q ").append(remoteZipFO.getPath()).append(" && rm ").append(remoteZipFO.getPath()); //NOI18N -// if (adjustLineEndings && Utils.isWindows()) { -// script.append(" && (which dos2unix > /dev/null; if [ $? = 0 ]; then find . -name \"*[Mm]akefile*\" -exec dos2unix {} \\; ; else echo \"no_dos2unix\"; fi)"); //NOI18N -// } - ProcessUtils.ExitStatus rc = ProcessUtils.executeInDir(getPath(), env, - "sh", /*"-x",*/ "-c", script.toString()); //NOI18N - if (!rc.isOK()) { - throw new IOException(rc.getErrorString() + " when unzipping and removing " + remoteZipFO.getPath() + " in " + this); //NOI18N - } - getCache().mkdirs(); - try (InputStream is = new FileInputStream(localZipFO)) { - RemoteFileSystemUtils.unpackZipFile(is, getCache()); - } - try { - refreshImpl(trace, null, true, RefreshMode.DEFAULT, 0); - } catch (TimeoutException ex) { - RemoteFileSystemUtils.reportUnexpectedTimeout(ex, this); - } catch (ExecutionException ex) { - throw new IOException(ex); - } + uploadAndUnzip(localZipFO); } finally { if (localZipFO != null) { localZipFO.delete(); @@ -1511,6 +1556,38 @@ } } + @SuppressWarnings("ReplaceStringBufferByString") + private void uploadAndUnzip(File localZipFO) throws InterruptedException, IOException { + final ExecutionEnvironment env = getExecutionEnvironment(); + // Copy local zip file to remote + FileObject remoteZipFO = getFileSystem().createTempFile(this.getOwnerFileObject(), RemoteFileSystem.TEMP_ZIP_PREFIX, ".zip", false); //NOI18N + try (OutputStream os = remoteZipFO.getOutputStream()) { + try (InputStream is = new FileInputStream(localZipFO)) { + FileUtil.copy(is, os); + } + } + StringBuilder script = new StringBuilder("unzip -q -o ").append(remoteZipFO.getPath()).append(" && rm ").append(remoteZipFO.getPath()); //NOI18N +// if (adjustLineEndings && Utils.isWindows()) { +// script.append(" && (which dos2unix > /dev/null; if [ $? = 0 ]; then find . -name \"*[Mm]akefile*\" -exec dos2unix {} \\; ; else echo \"no_dos2unix\"; fi)"); //NOI18N +// } + ProcessUtils.ExitStatus rc = ProcessUtils.executeInDir(getPath(), env, + "sh", "-c", script.toString()); //NOI18N + if (!rc.isOK()) { + throw new IOException(rc.getErrorString() + " when unzipping and removing " + remoteZipFO.getPath() + " in " + this); //NOI18N + } + getCache().mkdirs(); + try (InputStream is = new FileInputStream(localZipFO)) { + RemoteFileSystemUtils.unpackZipFile(is, getCache()); + } + try { + refreshImpl(trace, null, true, RefreshMode.DEFAULT, 0); + } catch (TimeoutException ex) { + RemoteFileSystemUtils.reportUnexpectedTimeout(ex, this); + } catch (ExecutionException ex) { + throw new IOException(ex); + } + } + private boolean ensureChildSyncFromZip(RemotePlainFile child) { File file = new File(getCache(), RemoteFileSystem.CACHE_ZIP_FILE_NAME); if (file.exists()) { diff -r 2b4e36d7aeb6 -r 220bce795781 dlight.remote.impl/src/org/netbeans/modules/remote/impl/fs/RemoteFileObjectBase.java --- a/dlight.remote.impl/src/org/netbeans/modules/remote/impl/fs/RemoteFileObjectBase.java Mon Dec 05 19:15:54 2016 +0300 +++ b/dlight.remote.impl/src/org/netbeans/modules/remote/impl/fs/RemoteFileObjectBase.java Tue Dec 06 11:44:53 2016 +0300 @@ -103,6 +103,7 @@ protected static final byte CONNECTION_ISSUES = 8; protected static final byte MASK_WARMUP = 16; protected static final byte MASK_CYCLIC_LINK = 32; + protected static final byte MASK_SUSPEND_WRITES = 64; protected RemoteFileObjectBase(RemoteFileObject wrapper, RemoteFileSystem fileSystem, ExecutionEnvironment execEnv, RemoteFileObjectBase parent, String remotePath) { @@ -980,5 +981,5 @@ protected RemoteLockSupport getLockSupport() { return fileSystem.getLockSupport(); - } + } } diff -r 2b4e36d7aeb6 -r 220bce795781 dlight.remote.impl/src/org/netbeans/modules/remote/impl/fs/RemoteFileSystem.java --- a/dlight.remote.impl/src/org/netbeans/modules/remote/impl/fs/RemoteFileSystem.java Mon Dec 05 19:15:54 2016 +0300 +++ b/dlight.remote.impl/src/org/netbeans/modules/remote/impl/fs/RemoteFileSystem.java Tue Dec 06 11:44:53 2016 +0300 @@ -57,6 +57,7 @@ import java.io.ObjectOutputStream; import java.net.ConnectException; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; @@ -149,6 +150,8 @@ private final ThreadLocal isInsideVCS = new ThreadLocal<>(); private final ThreadLocal isGettingDirectoryStorage = new ThreadLocal<>(); + private final Map> suspendedFiles = new HashMap<>(); + private final RequestProcessor.Task connectionTask; /** @@ -1151,6 +1154,23 @@ return fo.lastModified().getTime(); } } + + /*package*/ void addSuspendedFile(RemoteDirectory dir, RemotePlainFile fo) { + synchronized(suspendedFiles) { + Set set = suspendedFiles.get(dir); + if (set == null) { + set = new HashSet(); + suspendedFiles.put(dir, set); + } + set.add(fo); + } + } + + /*package*/ Set removeSuspendedFiles(RemoteDirectory dir) { + synchronized(suspendedFiles) { + return suspendedFiles.remove(dir); + } + } private final class StatusImpl implements StatusDecorator, ImageDecorator, LookupListener, FileStatusListener { diff -r 2b4e36d7aeb6 -r 220bce795781 dlight.remote.impl/src/org/netbeans/modules/remote/impl/fs/RemoteFileSystemProvider.java --- a/dlight.remote.impl/src/org/netbeans/modules/remote/impl/fs/RemoteFileSystemProvider.java Mon Dec 05 19:15:54 2016 +0300 +++ b/dlight.remote.impl/src/org/netbeans/modules/remote/impl/fs/RemoteFileSystemProvider.java Tue Dec 06 11:44:53 2016 +0300 @@ -544,7 +544,48 @@ } else { throw new IOException("Unexpected file object class for " + impl + //NOI18N ", expected " + RemoteDirectory.class.getSimpleName() + //NOI18N - " initial FileObjec " + targetFolder); //NOI18N + " initial FileObject " + targetFolder); //NOI18N + } + } + } + + @Override + public void suspendWritesUpload(FileObject folder) throws IOException { + if (folder instanceof RemoteFileObject) { + RemoteFileObjectBase impl = ((RemoteFileObject) folder).getImplementor(); + while (impl instanceof RemoteLinkBase) { + RemoteFileObjectBase delegate = ((RemoteLinkBase) impl).getCanonicalDelegate(); + if (delegate != null) { + impl = delegate; + } + } + if (impl instanceof RemoteDirectory) { + ((RemoteDirectory) impl).suspendWritesUpload(); + } else { + throw new IOException("Unexpected file object class for " + impl + //NOI18N + ", expected " + RemoteDirectory.class.getSimpleName() + //NOI18N + " initial FileObject " + folder); //NOI18N + } + } + } + + @Override + public void resumeWritesUpload(FileObject folder) + throws IOException, InterruptedException, ConnectException { + if (folder instanceof RemoteFileObject) { + RemoteFileObjectBase impl = ((RemoteFileObject) folder).getImplementor(); + while (impl instanceof RemoteLinkBase) { + RemoteFileObjectBase delegate = ((RemoteLinkBase) impl).getCanonicalDelegate(); + if (delegate != null) { + impl = delegate; + } + } + if (impl instanceof RemoteDirectory) { + ((RemoteDirectory) impl).resumeWritesUpload(); + } else { + throw new IOException("Unexpected file object class for " + impl + //NOI18N + ", expected " + RemoteDirectory.class.getSimpleName() + //NOI18N + " initial FileObject " + folder); //NOI18N } } } diff -r 2b4e36d7aeb6 -r 220bce795781 dlight.remote.impl/src/org/netbeans/modules/remote/impl/fs/RemotePlainFile.java --- a/dlight.remote.impl/src/org/netbeans/modules/remote/impl/fs/RemotePlainFile.java Mon Dec 05 19:15:54 2016 +0300 +++ b/dlight.remote.impl/src/org/netbeans/modules/remote/impl/fs/RemotePlainFile.java Tue Dec 06 11:44:53 2016 +0300 @@ -499,7 +499,9 @@ try { delegate.close(); RemotePlainFile.this.setPendingRemoteDelivery(true); - + if (getParentImpl().addSuspendedFile(RemotePlainFile.this)) { + return; + } String pathToRename, pathToUpload; if (RemotePlainFile.this.getParent().canWrite()) { diff -r 2b4e36d7aeb6 -r 220bce795781 dlight.remote/src/org/netbeans/modules/remote/spi/FileSystemProvider.java --- a/dlight.remote/src/org/netbeans/modules/remote/spi/FileSystemProvider.java Mon Dec 05 19:15:54 2016 +0300 +++ b/dlight.remote/src/org/netbeans/modules/remote/spi/FileSystemProvider.java Tue Dec 06 11:44:53 2016 +0300 @@ -674,6 +674,36 @@ noProvidersWarning(targetFolder); } + /** + * NB #1: does NOT support links inside directory! + * NB #2: does NOT support transform file names, they will be exactly the same + */ + public static void suspendWritesUpload(FileObject folder) throws IOException { + DLightLibsCommonLogger.assertTrue(folder.isFolder(), "Not a folder: " + folder); //NOI18N + for (FileSystemProviderImplementation provider : ALL_PROVIDERS) { + if (provider.isMine(folder)) { + provider.suspendWritesUpload(folder); + return; + } + } + noProvidersWarning(folder); + } + + /** + * NB: does NOT support links inside directory! + */ + public static void resumeWritesUpload(FileObject folder) + throws IOException, InterruptedException, ConnectException { + DLightLibsCommonLogger.assertTrue(folder.isFolder(), "Not a folder: " + folder); //NOI18N + for (FileSystemProviderImplementation provider : ALL_PROVIDERS) { + if (provider.isMine(folder)) { + provider.resumeWritesUpload(folder); + return; + } + } + noProvidersWarning(folder); + } + private static void noProvidersWarning(Object object) { if (RemoteLogger.getInstance().isLoggable(Level.FINE)) { if (RemoteLogger.getInstance().isLoggable(Level.FINEST)) { diff -r 2b4e36d7aeb6 -r 220bce795781 dlight.remote/src/org/netbeans/modules/remote/spi/FileSystemProviderImplementation.java --- a/dlight.remote/src/org/netbeans/modules/remote/spi/FileSystemProviderImplementation.java Mon Dec 05 19:15:54 2016 +0300 +++ b/dlight.remote/src/org/netbeans/modules/remote/spi/FileSystemProviderImplementation.java Tue Dec 06 11:44:53 2016 +0300 @@ -114,4 +114,6 @@ FileSystemProvider.Stat getStat(FileObject fo); void uploadAndUnzip(InputStream zipStream, FileObject targetFolder) throws FileNotFoundException, ConnectException, IOException, InterruptedException; + void suspendWritesUpload(FileObject folder) throws IOException; + void resumeWritesUpload(FileObject folder) throws IOException, InterruptedException, ConnectException; } diff -r 2b4e36d7aeb6 -r 220bce795781 dlight.remote/src/org/netbeans/modules/remote/support/LocalFileSystemProvider.java --- a/dlight.remote/src/org/netbeans/modules/remote/support/LocalFileSystemProvider.java Mon Dec 05 19:15:54 2016 +0300 +++ b/dlight.remote/src/org/netbeans/modules/remote/support/LocalFileSystemProvider.java Tue Dec 06 11:44:53 2016 +0300 @@ -565,4 +565,12 @@ } } } + + @Override + public void suspendWritesUpload(FileObject folder) { + } + + @Override + public void resumeWritesUpload(FileObject folder) { + } }