exo/checkDataIntegrity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scrubber;

Browse Source 
git-svn-id: svn://home0/checkDataIntegrity@3060 71dcc7fd-b1f2-4e5d-bb8e-274146e81173
This commit is contained in:
(no author)
2023-05-23 19:47:52 +00:00
parent 4dbfbde6f5
commit 0b2bd8a437
3 changed files with 17 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
bin/checkDataIntegrity/Main.class
View File

Binary file not shown.

BIN
checkDataIntegrity.jar
View File

Binary file not shown.

28
src/checkDataIntegrity/Main.java
View File

@@ -1,5 +1,8 @@
package checkDataIntegrity; package checkDataIntegrity;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.nio.file.Files; import java.nio.file.Files;
@@ -12,14 +15,18 @@ import java.util.stream.Stream;
public class Main { public class Main {
public static void main(final String[] args) throws IOException { public static void main(final String[] args) throws IOException {
if (args.length < 2) { if (args.length < 4) {
System.err.println("Usage: checkDataIntegrity.java <Path0> <Path1>"); System.err.println("Usage: checkDataIntegrity.java <Path0> <Path1> <skip> <limit>");
return; return;
} }
final String path0 = args[0]; final String path0 = args[0];
final String path1 = args[1]; final String path1 = args[1];
final int skip = Integer.parseInt(args[2]);
final int limit = Integer.parseInt(args[3]);
System.out.println("Path0: " + path0); System.out.println("Path0: " + path0);
System.out.println("Path1: " + path1); System.out.println("Path1: " + path1);
System.out.println("Skip: " + skip);
System.out.println("Limit: " + limit);
final long start = System.currentTimeMillis(); final long start = System.currentTimeMillis();
final AtomicLong count = new AtomicLong(); final AtomicLong count = new AtomicLong();
@@ -31,9 +38,11 @@ public class Main {
walk walk
.parallel() .parallel()
.filter(Files::isRegularFile) .filter(Files::isRegularFile)
.forEach(p0 -> { .skip(skip)
.limit(limit)
.forEachOrdered(p0 -> {
long cnt = count.incrementAndGet(); long cnt = count.incrementAndGet();
if (cnt % 10000 == 0) { if (cnt % 1000 == 0) {
System.out.println(cnt); System.out.println(cnt);
} }
@@ -42,7 +51,7 @@ public class Main {
final byte[] buf0 = bufs0.get(); final byte[] buf0 = bufs0.get();
final byte[] buf1 = bufs1.get(); final byte[] buf1 = bufs1.get();
try (InputStream inputStream0 = Files.newInputStream(p0); InputStream inputStream1 = Files.newInputStream(p1);) { try (InputStream inputStream0 = new BufferedInputStream(new FileInputStream(p0.toFile()), 65536); InputStream inputStream1 = new BufferedInputStream(new FileInputStream(p1.toFile()), 65536)) {
int len0, len1; int len0, len1;
do { do {
len0 = inputStream0.read(buf0); len0 = inputStream0.read(buf0);
@@ -52,10 +61,7 @@ public class Main {
countErr.incrementAndGet(); countErr.incrementAndGet();
break; break;
} }
if (len0 == -1) { if (len0 > -1 && !Arrays.equals(buf0, 0, len0, buf1, 0, len1)) {
break;
}
if (!Arrays.equals(buf0, 0, len0, buf1, 0, len1)) {
System.out.println(rel + ": Data mismatch"); System.out.println(rel + ": Data mismatch");
countErr.incrementAndGet(); countErr.incrementAndGet();
break; break;
@@ -63,7 +69,7 @@ public class Main {
} }
while (len0 > -1); while (len0 > -1);
} }
catch (NoSuchFileException nsfExc) { catch (NoSuchFileException | FileNotFoundException nsfExc) {
// System.out.println(rel + ": NoSuchFileException"); // System.out.println(rel + ": NoSuchFileException");
} }
catch (final Exception ex) { catch (final Exception ex) {
@@ -74,7 +80,7 @@ public class Main {
} }
System.out.println("Took " + (System.currentTimeMillis() - start) + "ms"); System.out.println("Took " + (System.currentTimeMillis() - start) + "ms");
System.out.println(count.get() + " files"); System.out.println(count.get() + " files checked");
System.out.println(countErr.get() + " errors"); System.out.println(countErr.get() + " errors");
System.exit(countErr.get() > 0 ? 1 : 0); System.exit(countErr.get() > 0 ? 1 : 0);