exo/checkDataIntegrity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

checkDataIntegrity

Browse Source 
git-svn-id: svn://home0/checkDataIntegrity@3073 71dcc7fd-b1f2-4e5d-bb8e-274146e81173
This commit is contained in:
(no author)
2023-05-31 19:13:08 +00:00
parent 4783acc880
commit 0cee423a0d
4 changed files with 51 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
src/checkDataIntegrity/Main.java
View File

@@ -5,12 +5,17 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Stream;
public class Main {
@@ -23,6 +28,7 @@ public class Main {
final String path1 = args[1];
final int skip = Integer.parseInt(args[2]);
final int limit = Integer.parseInt(args[3]);
final int max = skip + limit;
System.out.println("Path0: " + path0);
System.out.println("Path1: " + path1);
System.out.println("Skip: " + skip);
@@ -30,18 +36,51 @@ public class Main {
final long start = System.currentTimeMillis();
final AtomicLong count = new AtomicLong();
System.out.println("Scanning Tree");
final List<Path> paths = new ArrayList<>(limit);
Files.walkFileTree(Path.of(path0), new FileVisitor<>() {
@Override
public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) throws IOException {
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
if (attrs.isRegularFile() && count.incrementAndGet() > skip) {
paths.add(file);
}
return count.get() < max ? FileVisitResult.CONTINUE : FileVisitResult.TERMINATE;
}
@Override
public FileVisitResult visitFileFailed(final Path file, final IOException exc) throws IOException {
// System.out.println(exc.getClass().getSimpleName() + " " + exc.getMessage());
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException {
// if (exc != null) {
// System.out.println(exc.getClass().getSimpleName() + " " + exc.getMessage());
// }
return FileVisitResult.CONTINUE;
}
});
count.set(0);
final AtomicLong countErr = new AtomicLong();
final ThreadLocal<byte[]> bufs0 = ThreadLocal.withInitial(() -> new byte[8192]);
final ThreadLocal<byte[]> bufs1 = ThreadLocal.withInitial(() -> new byte[8192]);
try (Stream<Path> walk = Files.walk(Path.of(path0))) {
walk
try {
paths
.stream()
.parallel()
.filter(Files::isRegularFile)
.skip(skip)
.limit(limit)
.forEachOrdered(p0 -> {
long cnt = count.incrementAndGet();
.forEach(p0 -> {
final long cnt = count.incrementAndGet();
if (cnt % 1000 == 0) {
System.out.println(cnt);
}
@@ -66,11 +105,10 @@ public class Main {
countErr.incrementAndGet();
break;
}
}
while (len0 > -1);
} while (len0 > -1);
}
catch (NoSuchFileException | FileNotFoundException nsfExc) {
// System.out.println(rel + ": NoSuchFileException");
// System.out.println(rel + ": NoSuchFileException");
}
catch (final Exception ex) {
System.out.println(rel + ": Exception: " + ex.getClass().getName() + " " + ex.getMessage());
@@ -78,12 +116,12 @@ public class Main {
}
});
}
catch (Exception ex) {
catch (final Exception ex) {
System.out.println("General Exception: " + ex.getClass().getName() + " " + ex.getMessage());
countErr.incrementAndGet();
}
System.out.println("Took " + (System.currentTimeMillis() - start) + "ms");
System.out.println("Took " + Duration.ofMillis(System.currentTimeMillis() - start).toString());
System.out.println(((count.get() > 0) ? count.get() : "No") + " files checked");
System.out.println(countErr.get() + " errors");