Lines 86-92
Link Here
|
86 |
private static final boolean trace = RemoteLogger.getInstance().isLoggable(Level.FINEST); |
86 |
private static final boolean trace = RemoteLogger.getInstance().isLoggable(Level.FINEST); |
87 |
private static boolean LS_VIA_SFTP = ! Boolean.getBoolean("remote.parse.ls"); |
87 |
private static boolean LS_VIA_SFTP = ! Boolean.getBoolean("remote.parse.ls"); |
88 |
|
88 |
|
89 |
private Reference<DirectoryStorage> storageRef; |
89 |
private Reference<DirectoryStorage> storageRef = new SoftReference<DirectoryStorage>(null);; |
90 |
private static final class RefLock {} |
90 |
private static final class RefLock {} |
91 |
private final Object refLock = new RefLock(); |
91 |
private final Object refLock = new RefLock(); |
92 |
|
92 |
|
Lines 407-416
Link Here
|
407 |
} |
407 |
} |
408 |
} |
408 |
} |
409 |
|
409 |
|
410 |
private DirectoryStorage getDirectoryStorageImpl(boolean force, String expectedName, String childName) throws |
410 |
private DirectoryStorage getDirectoryStorageImpl(boolean forceRefresh, String expectedName, String childName) throws |
411 |
ConnectException, IOException, InterruptedException, CancellationException, ExecutionException { |
411 |
ConnectException, IOException, InterruptedException, CancellationException, ExecutionException { |
412 |
|
412 |
|
413 |
if (force && ! ConnectionManager.getInstance().isConnectedTo(getExecutionEnvironment())) { |
413 |
if (forceRefresh && ! ConnectionManager.getInstance().isConnectedTo(getExecutionEnvironment())) { |
414 |
//RemoteLogger.getInstance().warning("refreshDirectoryStorage is called while host is not connected"); |
414 |
//RemoteLogger.getInstance().warning("refreshDirectoryStorage is called while host is not connected"); |
415 |
//force = false; |
415 |
//force = false; |
416 |
throw new ConnectException(); |
416 |
throw new ConnectException(); |
Lines 422-448
Link Here
|
422 |
|
422 |
|
423 |
// check whether it is cached in memory |
423 |
// check whether it is cached in memory |
424 |
synchronized (refLock) { |
424 |
synchronized (refLock) { |
425 |
if (storageRef != null) { |
425 |
storage = storageRef.get(); |
426 |
storage = storageRef.get(); |
|
|
427 |
} |
428 |
} |
426 |
} |
429 |
if (! force && storage != null) { |
427 |
boolean fromMemOrDiskCache; |
430 |
return storage; |
|
|
431 |
} |
432 |
|
428 |
|
433 |
boolean loaded; |
|
|
434 |
|
435 |
if (storage == null) { |
429 |
if (storage == null) { |
436 |
// try loading from disk |
430 |
// try loading from disk |
437 |
loaded = false; |
431 |
fromMemOrDiskCache = false; |
438 |
storage = new DirectoryStorage(storageFile); |
432 |
storage = new DirectoryStorage(storageFile); |
439 |
if (storageFile.exists()) { |
433 |
if (storageFile.exists()) { |
440 |
Lock lock = RemoteFileSystem.getLock(getCache()).readLock(); |
434 |
Lock readLock = RemoteFileSystem.getLock(getCache()).readLock(); |
441 |
try { |
435 |
try { |
442 |
lock.lock(); |
436 |
readLock.lock(); |
443 |
try { |
437 |
try { |
444 |
storage.load(); |
438 |
storage.load(); |
445 |
loaded = true; |
439 |
fromMemOrDiskCache = true; |
|
|
440 |
// try to keep loaded cache in memory |
441 |
synchronized (refLock) { |
442 |
DirectoryStorage s = storageRef.get(); |
443 |
// it could be cache put in memory by writer (the best content) |
444 |
// or by previous reader => it's the same as loaded |
445 |
if (s != null) { |
446 |
if (trace) { trace("using storage that was kept by other thread"); } // NOI18N |
447 |
storage = s; |
448 |
} else { |
449 |
storageRef = new SoftReference<DirectoryStorage>(storage); |
450 |
} |
451 |
} |
446 |
} catch (FormatException e) { |
452 |
} catch (FormatException e) { |
447 |
Level level = e.isExpexted() ? Level.FINE : Level.WARNING; |
453 |
Level level = e.isExpexted() ? Level.FINE : Level.WARNING; |
448 |
RemoteLogger.getInstance().log(level, "Error reading directory cache", e); // NOI18N |
454 |
RemoteLogger.getInstance().log(level, "Error reading directory cache", e); // NOI18N |
Lines 456-501
Link Here
|
456 |
Exceptions.printStackTrace(e); |
462 |
Exceptions.printStackTrace(e); |
457 |
} |
463 |
} |
458 |
} finally { |
464 |
} finally { |
459 |
lock.unlock(); |
465 |
readLock.unlock(); |
460 |
} |
466 |
} |
461 |
} |
467 |
} |
462 |
} else { |
468 |
} else { |
463 |
loaded = true; |
469 |
if (trace) { trace("use memory cached storage"); } // NOI18N |
|
|
470 |
fromMemOrDiskCache = true; |
464 |
} |
471 |
} |
465 |
|
472 |
|
466 |
if (loaded && !force) { |
473 |
if (fromMemOrDiskCache && !forceRefresh) { |
467 |
synchronized (refLock) { |
474 |
RemoteLogger.assertTrue(storage != null); |
468 |
if (storageRef != null) { |
475 |
if (trace) { trace("returning cached storage"); } // NOI18N |
|
|
476 |
return storage; |
477 |
} |
478 |
// neither memory nor disk cache helped or was request to force refresh |
479 |
// proceed with reading remote content |
480 |
|
481 |
checkConnection(this, true); |
482 |
|
483 |
Lock writeLock = RemoteFileSystem.getLock(getCache()).writeLock(); |
484 |
if (trace) { trace("waiting for lock"); } // NOI18N |
485 |
writeLock.lock(); |
486 |
try { |
487 |
if (!forceRefresh) { |
488 |
// it means we didn't have any storage |
489 |
RemoteLogger.assertFalse(fromMemOrDiskCache); |
490 |
// in case another writer thread already synchronized content while we were waiting for lock |
491 |
synchronized (refLock) { |
469 |
DirectoryStorage s = storageRef.get(); |
492 |
DirectoryStorage s = storageRef.get(); |
470 |
if (s != null) { |
493 |
if (s != null) { |
471 |
if (trace) { trace("returning storage that was loaded by other thread"); } // NOI18N |
494 |
if (trace) { trace("got storage from mem cache after waiting on writeLock: {0} expectedName={1}", getPath(), expectedName); } // NOI18N |
472 |
return s; |
495 |
return s; |
473 |
} |
496 |
} |
474 |
} |
497 |
} |
475 |
storageRef = new SoftReference<DirectoryStorage>(storage); |
|
|
476 |
} |
477 |
if (trace) { trace("returning just loaded storage"); } // NOI18N |
478 |
return storage; |
479 |
} |
480 |
|
481 |
// neither memory nor disk cache helped |
482 |
checkConnection(this, true); |
483 |
|
484 |
Lock lock = RemoteFileSystem.getLock(getCache()).writeLock(); |
485 |
if (trace) { trace("waiting for lock"); } // NOI18N |
486 |
lock.lock(); |
487 |
try { |
488 |
if (!force) { |
489 |
// in case another thread synchronized content while we were waiting for lock |
490 |
synchronized (refLock) { |
491 |
if (storageRef != null) { |
492 |
DirectoryStorage stor = storageRef.get(); |
493 |
if (trace) { trace("got storage: {0} -> {1}", storageRef, stor); } // NOI18N |
494 |
if (stor != null) { |
495 |
return stor; |
496 |
} |
497 |
} |
498 |
} |
499 |
} |
498 |
} |
500 |
if (!getCache().exists()) { |
499 |
if (!getCache().exists()) { |
501 |
getCache().mkdirs(); |
500 |
getCache().mkdirs(); |
Lines 506-538
Link Here
|
506 |
DirectoryReader directoryReader = getLsViaSftp() ? |
505 |
DirectoryReader directoryReader = getLsViaSftp() ? |
507 |
new DirectoryReaderSftp(getExecutionEnvironment(), getPath()) : new DirectoryReaderLs(getExecutionEnvironment(), getPath()); |
506 |
new DirectoryReaderSftp(getExecutionEnvironment(), getPath()) : new DirectoryReaderLs(getExecutionEnvironment(), getPath()); |
508 |
if (trace) { trace("synchronizing"); } // NOI18N |
507 |
if (trace) { trace("synchronizing"); } // NOI18N |
|
|
508 |
Exception problem = null; |
509 |
try { |
509 |
try { |
510 |
directoryReader.readDirectory(); |
510 |
directoryReader.readDirectory(); |
511 |
} catch (FileNotFoundException ex) { |
511 |
} catch (FileNotFoundException ex) { |
512 |
throw ex; |
512 |
throw ex; |
513 |
} catch (IOException ex) { |
513 |
} catch (IOException ex) { |
|
|
514 |
problem = ex; |
515 |
} catch (ExecutionException ex) { |
516 |
problem = ex; |
517 |
} |
518 |
if (problem != null) { |
514 |
if (!ConnectionManager.getInstance().isConnectedTo(getExecutionEnvironment())) { |
519 |
if (!ConnectionManager.getInstance().isConnectedTo(getExecutionEnvironment())) { |
515 |
// connection was broken while we read directory content - |
520 |
// connection was broken while we read directory content - add notification |
516 |
// add notification and return cache if available |
|
|
517 |
getFileSystem().getRemoteFileSupport().addPendingFile(this); |
521 |
getFileSystem().getRemoteFileSupport().addPendingFile(this); |
518 |
if (loaded && !force && storage != null) { |
522 |
// valid cache can not be available |
519 |
return storage; |
523 |
RemoteLogger.assertFalse(fromMemOrDiskCache && !forceRefresh && storage != null); |
520 |
} else { |
524 |
throw new ConnectException(problem.getMessage()); |
521 |
throw new ConnectException(ex.getMessage()); |
|
|
522 |
} |
523 |
} |
525 |
} |
524 |
|
|
|
525 |
} catch (ExecutionException ex) { |
526 |
if (!ConnectionManager.getInstance().isConnectedTo(getExecutionEnvironment())) { |
527 |
// connection was broken while we read directory content - |
528 |
// add notification and return cache if available |
529 |
getFileSystem().getRemoteFileSupport().addPendingFile(this); |
530 |
if (loaded && !force && storage != null) { |
531 |
return storage; |
532 |
} else { |
533 |
throw ex; |
534 |
} |
535 |
} |
536 |
} |
526 |
} |
537 |
getFileSystem().incrementDirSyncCount(); |
527 |
getFileSystem().incrementDirSyncCount(); |
538 |
Map<String, List<DirEntry>> dupLowerNames = new HashMap<String, List<DirEntry>>(); |
528 |
Map<String, List<DirEntry>> dupLowerNames = new HashMap<String, List<DirEntry>>(); |
Lines 545-557
Link Here
|
545 |
Set<DirEntry> keepCacheNames = new HashSet<DirEntry>(); |
535 |
Set<DirEntry> keepCacheNames = new HashSet<DirEntry>(); |
546 |
List<DirEntry> entriesToFireChanged = new ArrayList<DirEntry>(); |
536 |
List<DirEntry> entriesToFireChanged = new ArrayList<DirEntry>(); |
547 |
List<DirEntry> entriesToFireCreated = new ArrayList<DirEntry>(); |
537 |
List<DirEntry> entriesToFireCreated = new ArrayList<DirEntry>(); |
|
|
538 |
List<FileObject> filesToFireDeleted = new ArrayList<FileObject>(); |
548 |
for (DirEntry newEntry : newEntries.values()) { |
539 |
for (DirEntry newEntry : newEntries.values()) { |
549 |
String cacheName; |
540 |
String cacheName; |
550 |
DirEntry oldEntry = storage.getEntry(newEntry.getName()); |
541 |
DirEntry oldEntry = storage.getEntry(newEntry.getName()); |
551 |
if (oldEntry == null) { |
542 |
if (oldEntry == null) { |
552 |
changed = true; |
543 |
changed = true; |
553 |
cacheName = RemoteFileSystemUtils.escapeFileName(newEntry.getName()); |
544 |
cacheName = RemoteFileSystemUtils.escapeFileName(newEntry.getName()); |
554 |
if (loaded || newEntry.getName().equals(expectedName)) { |
545 |
if (fromMemOrDiskCache || newEntry.getName().equals(expectedName)) { |
555 |
entriesToFireCreated.add(newEntry); |
546 |
entriesToFireCreated.add(newEntry); |
556 |
} |
547 |
} |
557 |
} else { |
548 |
} else { |
Lines 565-575
Link Here
|
565 |
File entryCache = new File(getCache(), oldEntry.getCache()); |
556 |
File entryCache = new File(getCache(), oldEntry.getCache()); |
566 |
if (entryCache.exists()) { |
557 |
if (entryCache.exists()) { |
567 |
if (trace) { trace("removing cache for updated file {0}", entryCache.getAbsolutePath()); } // NOI18N |
558 |
if (trace) { trace("removing cache for updated file {0}", entryCache.getAbsolutePath()); } // NOI18N |
568 |
entryCache.delete(); |
559 |
entryCache.delete(); // TODO: We must just mark it as invalid instead of physically deleting cache file... |
569 |
} |
560 |
} |
570 |
} |
561 |
} |
571 |
} else if (!equals(newEntry.getLinkTarget(), oldEntry.getLinkTarget())) { |
562 |
} else if (!equals(newEntry.getLinkTarget(), oldEntry.getLinkTarget())) { |
572 |
changed = fire = true; |
563 |
changed = fire = true; // TODO: we forgot old link path, probably should be passed to change event |
573 |
getFileSystem().getFactory().setLink(this, getPath() + '/' + newEntry.getName(), newEntry.getLinkTarget()); |
564 |
getFileSystem().getFactory().setLink(this, getPath() + '/' + newEntry.getName(), newEntry.getLinkTarget()); |
574 |
} else if (!newEntry.getAccessAsString().equals(oldEntry.getAccessAsString())) { |
565 |
} else if (!newEntry.getAccessAsString().equals(oldEntry.getAccessAsString())) { |
575 |
changed = fire = true; |
566 |
changed = fire = true; |
Lines 578-591
Link Here
|
578 |
} else if (!newEntry.isSameGroup(oldEntry)) { |
569 |
} else if (!newEntry.isSameGroup(oldEntry)) { |
579 |
changed = fire = true; |
570 |
changed = fire = true; |
580 |
} else if (newEntry.getSize() != oldEntry.getSize()) { |
571 |
} else if (newEntry.getSize() != oldEntry.getSize()) { |
581 |
changed = fire = true; |
572 |
changed = fire = true;// TODO: shouldn't it be the same as time stamp change? |
582 |
} |
573 |
} |
583 |
if (fire) { |
574 |
if (fire) { |
584 |
entriesToFireChanged.add(newEntry); |
575 |
entriesToFireChanged.add(newEntry); |
585 |
} |
576 |
} |
586 |
} else { |
577 |
} else { |
587 |
changed = true; |
578 |
changed = true; |
588 |
invalidate(oldEntry); |
579 |
FileObject removedFO = invalidate(oldEntry); |
|
|
580 |
// remove old |
581 |
if (removedFO != null) { |
582 |
filesToFireDeleted.add(removedFO); |
583 |
} |
584 |
// add new |
585 |
entriesToFireCreated.add(newEntry); |
589 |
cacheName = RemoteFileSystemUtils.escapeFileName(newEntry.getName()); |
586 |
cacheName = RemoteFileSystemUtils.escapeFileName(newEntry.getName()); |
590 |
} |
587 |
} |
591 |
} |
588 |
} |
Lines 606-618
Link Here
|
606 |
// Check for removal |
603 |
// Check for removal |
607 |
for (DirEntry oldEntry : storage.list()) { |
604 |
for (DirEntry oldEntry : storage.list()) { |
608 |
if (!newEntries.containsKey(oldEntry.getName())) { |
605 |
if (!newEntries.containsKey(oldEntry.getName())) { |
609 |
changed = true; |
606 |
FileObject removedFO = invalidate(oldEntry); |
610 |
invalidate(oldEntry); |
607 |
if (removedFO != null) { |
|
|
608 |
filesToFireDeleted.add(removedFO); |
609 |
} |
611 |
} |
610 |
} |
612 |
} |
611 |
} |
613 |
} |
|
|
614 |
|
615 |
if (changed) { |
616 |
if (hasDups) { |
612 |
if (hasDups) { |
617 |
for (Map.Entry<String, List<DirEntry>> mapEntry : |
613 |
for (Map.Entry<String, List<DirEntry>> mapEntry : |
618 |
new ArrayList<Map.Entry<String, List<DirEntry>>>(dupLowerNames.entrySet())) { |
614 |
new ArrayList<Map.Entry<String, List<DirEntry>>>(dupLowerNames.entrySet())) { |
Lines 621-629
Link Here
|
621 |
if (dupEntries.size() > 1) { |
617 |
if (dupEntries.size() > 1) { |
622 |
for (int i = 0; i < dupEntries.size(); i++) { |
618 |
for (int i = 0; i < dupEntries.size(); i++) { |
623 |
DirEntry entry = dupEntries.get(i); |
619 |
DirEntry entry = dupEntries.get(i); |
624 |
if (keepCacheNames.contains(entry) || i == 0) { |
620 |
if (keepCacheNames.contains(entry)) { |
625 |
continue; // keep the one that already exists or otherwise 0-th one |
621 |
continue; // keep the one that already exists |
626 |
} |
622 |
} |
|
|
623 |
// all duplicates will have postfix |
627 |
for (int j = 0; j < Integer.MAX_VALUE; j++) { |
624 |
for (int j = 0; j < Integer.MAX_VALUE; j++) { |
628 |
String cacheName = mapEntry.getKey() + '_' + j; |
625 |
String cacheName = mapEntry.getKey() + '_' + j; |
629 |
String lowerCacheName = cacheName.toLowerCase(); |
626 |
String lowerCacheName = cacheName.toLowerCase(); |
Lines 644-655
Link Here
|
644 |
} else { |
641 |
} else { |
645 |
storage.touch(); |
642 |
storage.touch(); |
646 |
} |
643 |
} |
|
|
644 |
// always put new content in cache |
645 |
// do it before firing events, to give liseners real content |
647 |
synchronized (refLock) { |
646 |
synchronized (refLock) { |
648 |
storageRef = new SoftReference<DirectoryStorage>(storage); |
647 |
storageRef = new SoftReference<DirectoryStorage>(storage); |
649 |
} |
648 |
} |
650 |
storageFile.setLastModified(System.currentTimeMillis()); |
649 |
storageFile.setLastModified(System.currentTimeMillis()); |
651 |
if (trace) { trace("set lastModified to {0}", storageFile.lastModified()); } // NOI18N |
650 |
if (trace) { trace("set lastModified to {0}", storageFile.lastModified()); } // NOI18N |
|
|
651 |
// fire all event under lock |
652 |
if (changed) { |
652 |
if (changed) { |
|
|
653 |
for (FileObject deleted : filesToFireDeleted) { |
654 |
fireFileDeletedEvent(getListeners(), new FileEvent(deleted)); |
655 |
} |
653 |
for (DirEntry entry : entriesToFireCreated) { |
656 |
for (DirEntry entry : entriesToFireCreated) { |
654 |
RemoteFileObjectBase fo = createFileObject(entry); |
657 |
RemoteFileObjectBase fo = createFileObject(entry); |
655 |
fireRemoteFileObjectCreated(fo); |
658 |
fireRemoteFileObjectCreated(fo); |
Lines 662-668
Link Here
|
662 |
} |
665 |
} |
663 |
} |
666 |
} |
664 |
} finally { |
667 |
} finally { |
665 |
lock.unlock(); |
668 |
writeLock.unlock(); |
666 |
} |
669 |
} |
667 |
return storage; |
670 |
return storage; |
668 |
} |
671 |
} |
Lines 752-764
Link Here
|
752 |
throw new IOException(getPath()); |
755 |
throw new IOException(getPath()); |
753 |
} |
756 |
} |
754 |
|
757 |
|
755 |
private void invalidate(DirEntry oldEntry) { |
758 |
private FileObject invalidate(DirEntry oldEntry) { |
756 |
FileObject fo = getFileSystem().getFactory().invalidate(getPath() + '/' + oldEntry.getName()); |
759 |
FileObject fo = getFileSystem().getFactory().invalidate(getPath() + '/' + oldEntry.getName()); |
757 |
File oldEntryCache = new File(getCache(), oldEntry.getCache()); |
760 |
File oldEntryCache = new File(getCache(), oldEntry.getCache()); |
758 |
removeFile(oldEntryCache); |
761 |
removeFile(oldEntryCache); |
759 |
if (fo != null) { |
762 |
return fo; |
760 |
fireFileDeletedEvent(getListeners(), new FileEvent(fo)); |
|
|
761 |
} |
762 |
} |
763 |
} |
763 |
|
764 |
|
764 |
private void removeFile(File cache) { |
765 |
private void removeFile(File cache) { |