Create de-dup files based on key matches

Author: terrywbrady

core/src/main/gov/nara/nwts/ftapp/importer/CountKey.java

@@ -15,6 +15,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.StringTokenizer;
 import java.util.TreeMap;
 import java.util.Vector;
 
@@ -41,6 +42,7 @@ public static enum Generator implements StatsGenerator {
 	public static final String DELIM = "Delimiter";
 	public static final String HEADROW = "HeadRow";
 	public static final String COL = "COL";
+    public static final String DEDUP = "DeDup";
 	public CountKey(FTDriver dt) {
 		super(dt);
 		this.ftprops.add(new FTPropEnum(dt, this.getClass().getName(), DELIM, "delim",
@@ -49,6 +51,8 @@ public CountKey(FTDriver dt) {
 				"Treat first row as header", YN.values(), YN.Y));
 		this.ftprops.add(new FTPropString(dt, this.getClass().getName(), COL, COL,
 				"Key Column starting at 1", "1"));
+        this.ftprops.add(new FTPropEnum(dt, this.getClass().getName(), DEDUP, DEDUP,
+                "Create De-duplicated file set", YN.values(), YN.N));
 	}
 
 	public ActionResult importFile(File selectedFile) throws IOException {
@@ -67,27 +71,75 @@ public ActionResult importFile(File selectedFile) throws IOException {
 
 		DelimitedFileReader dfr = new DelimitedFileReader(selectedFile, fileSeparator.separator);
 		boolean firstRow = (YN)getProperty(HEADROW) == YN.Y;
+        boolean dedup = (YN)getProperty(DEDUP) == YN.Y;
+        DelimitedFileWriter bwDedup = null;
+        DelimitedFileWriter bwDup = null;
+        if (dedup) {
+            bwDedup = new DelimitedFileWriter(getNewFile(selectedFile, "dedup"), fileSeparator.separator);
+            bwDup = new DelimitedFileWriter(getNewFile(selectedFile, "dup-drop"), fileSeparator.separator);
+        }
 
 		firstRow = (YN)getProperty(HEADROW) == YN.Y;
 		dfr = new DelimitedFileReader(selectedFile, fileSeparator.separator);
 		for(Vector<String> cols = dfr.getRow(); cols != null; cols = dfr.getRow()){
 			if (firstRow) {
 				firstRow = false;
+				if (dedup) {
+				    bwDedup.writeRow(cols);
+                    bwDup.writeRow(cols);
+				}
 				continue;
 			}
 			String key = cols.get(col < cols.size() ? col : 0);
 			Stats stats = types.get(key);
-			if (stats == null) {
+            if (stats == null) {
 				stats = Generator.INSTANCE.create(key);
 				stats.setVal(CountStatsItems.Count, 1);
 				stats.setVal(CountStatsItems.Stat, MULT.ONE);
 				types.put(key, stats);
+                if (dedup) {
+                    bwDedup.writeRow(cols);
+                }
 			} else {
 				stats.sumVal(CountStatsItems.Count, 1);
 				stats.setVal(CountStatsItems.Stat, MULT.MANY);
+                if (dedup) {
+                    bwDup.writeRow(cols);
+                }
 			}
 		}
-		
+
+        if (dedup) {
+            bwDedup.close();
+            bwDup.close();
+
+            DelimitedFileWriter bwNoDup = new DelimitedFileWriter(getNewFile(selectedFile, "no-dup"), fileSeparator.separator);
+            DelimitedFileWriter bwAllDup = new DelimitedFileWriter(getNewFile(selectedFile, "all-dup"), fileSeparator.separator);
+            dfr = new DelimitedFileReader(selectedFile, fileSeparator.separator);
+            firstRow = (YN)getProperty(HEADROW) == YN.Y;
+            for(Vector<String> cols = dfr.getRow(); cols != null; cols = dfr.getRow()){
+                if (firstRow) {
+                    firstRow = false;
+                    if (dedup) {
+                        bwNoDup.writeRow(cols);
+                        bwAllDup.writeRow(cols);
+                    }
+                    continue;
+                }
+                String key = cols.get(col < cols.size() ? col : 0);
+                Stats stats = types.get(key);
+                if (stats == null) {
+                } else if (stats.getVal(CountStatsItems.Stat) == MULT.ONE) {
+                    bwNoDup.writeRow(cols);
+                } else {
+                    bwAllDup.writeRow(cols);
+                }
+            }
+            bwNoDup.close();
+            bwAllDup.close();
+        }
+
+
 		return new ActionResult(selectedFile, selectedFile.getName(), this.toString(), details, types, true, timer.getDuration());
 	}
 
@@ -100,4 +152,22 @@ public String getDescription() {
 	public String getShortName() {
 		return "Key";
 	}
+    public String getExt(File f) {
+        String ext = "";
+        StringTokenizer st = new StringTokenizer(f.getName(), ".");
+        while(st.hasMoreElements()) {
+            ext = st.nextElement().toString();
+        }   
+        return ext;
+    }
+    
+    public String getNewName(File f, String suff) {
+        String ext = getExt(f);
+        String bname = f.getName();
+        return (ext.isEmpty()) ? bname + "." + suff : bname.replaceFirst(ext + "$", suff + "." + ext);
+    }
+    public File getNewFile(File f, String suff) {
+        return new File(f.getParentFile(), getNewName(f, suff));
+    }
+
 }