# encoding: ascii
# api: csharp
# title: Findup
# description: Findup – Find duplicates C# version. Compares files sizes and SHA512 hashes to identify duplicates. New regex Include/Exclude feature. Should be compiled with Visual Studio 11 (beta as of now), as older Visual Studio C# compilers seem to have a bug that causes crashes on long file names.
# version: 2.0
# type: class
# author: James Gentile
# license: CC0
# x-poshcode-id: 3370
# x-derived-from-id: 3377
# x-archived: 2016-03-17T14:34:49
# x-published: 2013-04-18T12:58:00
#
#
using System;
using System.Collections;
using System.Collections.Generic;
using System.Text;
using System.Security.Cryptography;
using System.Runtime.InteropServices;
using Microsoft.Win32;
using System.IO;
using System.Text.RegularExpressions;
namespace Findup
{
public class FileLengthComparer : IComparer<FileInfo>
{
public int Compare(FileInfo x, FileInfo y)
{
return (x.Length.CompareTo(y.Length));
}
}
class Findup
{
public static Dictionary<string, List<string>> optionspaths = new Dictionary<string, List<string>>
{ {"/x", new List<string>()},{"/i",new List<string>()},{"/xf",new List<string>()},{"/if",new List<string>()},
{"/xd",new List<string>()},{"/id",new List<string>()},{"/paths",new List<string>()} };
public static Dictionary<string, List<Regex>> optionsregex = new Dictionary<string, List<Regex>>
{ {"/xr", new List<Regex>()},{"/ir",new List<Regex>()},{"/xfr",new List<Regex>()},{"/ifr",new List<Regex>()},
{"/xdr",new List<Regex>()},{"/idr",new List<Regex>()} };
public static Dictionary<string, Boolean> optionsbools = new Dictionary<string, bool> { { "/recurse", false }, { "/noerr", false }, {"/delete",false}, {"/xj", false}};
public static long numOfDupes, dupeBytes, bytesrecovered, deletedfiles = 0; // number of duplicate files found, bytes in duplicates, bytes recovered from deleting dupes, number of deleted dupes.
public static long errors = 0;
public static string delconfirm;
public static void Main(string[] args)
{
DateTime startTime = DateTime.Now;
Console.WriteLine("Findup.exe v2.0 - By James Gentile - JamesRaymondGentile@gmail.com - 2012.");
Console.WriteLine("Findup.exe matches sizes, then SHA512 hashes to identify duplicate files.");
Console.WriteLine(" ");
string optionskey = "/paths";
List<FileInfo> files = new List<FileInfo>();
int i = 0;
for (i = 0; i < args.Length; i++)
{
string argitem = args[i].ToLower();
if ((System.String.Compare(argitem, "/help", true) == 0) || (System.String.Compare(argitem, "/h", true) == 0))
{
Console.WriteLine("Usage: findup.exe <file/directory #1..#N> [/recurse] [/noerr] [/x /i /xd /id /xf /if + [r]] <files/directories/regex> [/delete]");
Console.WriteLine(" ");
Console.WriteLine("Options: /help - displays this help message.");
Console.WriteLine(" /recurse - recurses through subdirectories when directories or file specifications (e.g. *.txt) are specified.");
Console.WriteLine(" /noerr - discards error messages.");
Console.WriteLine(" /delete - delete each duplicate file with confirmation.");
Console.WriteLine(" /x - eXcludes if full file path starts with (or RegEx matches if /xr) one of the items following this switch until another switch is used.");
Console.WriteLine(" /i - include if full file path starts with (or Regex matches if /ir) one of the items following this switch until another switch is used.");
Console.WriteLine(" /xd - eXcludes all directories - minus drive/files - (using RegEx if /xdr) including subdirs following this switch until another switch is used.");
Console.WriteLine(" /id - Include only directories - minus drive/files - (using RegEx if /idr) including subdirs following this switch until another switch is used.");
Console.WriteLine(" /xf - eXcludes all files - minus drive/directories - (using RegEx if /xfr) following this switch until another switch is used.");
Console.WriteLine(" /if - Include only files - minus drive/directories - (using RegEx if /ifr) following this switch until another switch is used.");
Console.WriteLine(" [r] - Use regex for include/exclude by appending an 'r' to the option, e.g. -ir, -ifr, -idr, -xr, -xfr, -xdr.");
Console.WriteLine(" /paths - not needed unless you want to specify files/dirs after an include/exclude without using another non-exclude/non-include option.");
Console.WriteLine(" /xj - Exclude File and Directory Junctions.");
Console.WriteLine(" ");
Console.WriteLine("Examples: findup.exe c:\\finances /recurse /noerr /delete");
Console.WriteLine(" - Find dupes in c:\\finance.");
Console.WriteLine(" - recurse all subdirs of c:\\finance.");
Console.WriteLine(" - suppress error messages.");
Console.WriteLine(" - deletes duplicates after consent is given.");
Console.WriteLine(" findup.exe c:\\users\\alice\\plan.txt d:\\data /recurse /x d:\\data\\webpics");
Console.WriteLine(" - Find dupes in c:\\users\\alice\\plan.txt, d:\\data");
Console.WriteLine(" - recurse subdirs in d:\\data.");
Console.WriteLine(" - exclude any files in d:\\data\\webpics and subdirs.");
Console.WriteLine(" findup.exe c:\\data *.txt c:\\reports\\quarter.doc /xfr \"(jim)\"");
Console.WriteLine(" - Find dupes in c:\\data, *.txt in current directory and c:\\reports\\quarter.doc");
Console.WriteLine(" - exclude any file with 'jim' in the name as specified by the Regex item \"(jim)\"");
Console.WriteLine(" findup.exe c:\\data *.txt c:\\reports\\*quarter.doc /xr \"[bf]\" /ir \"(smith)\"");
Console.WriteLine(" - Find dupes in c:\\data, *.txt in current directory and c:\\reports\\*quarter.doc");
Console.WriteLine(" - Include only files with 'smith' and exclude any file with letters b or f in the path name as specified by the Regex items \"[bf]\",\"(smith)\"");
Console.WriteLine("Note: Exclude takes precedence over Include.");
return;
}
if (optionsbools.ContainsKey(argitem))
{
optionsbools[argitem] = true;
optionskey = "/paths";
continue;
}
if (optionspaths.ContainsKey(argitem) || optionsregex.ContainsKey(argitem))
{
optionskey = argitem;
continue;
}
if (optionspaths.ContainsKey(optionskey))
optionspaths[optionskey].Add(args[i]);
else
{
try {
Regex rgx = new Regex(args[i], RegexOptions.Compiled);
optionsregex[optionskey].Add(rgx);
}
catch (Exception e) {WriteErr("Regex compilation failed: " + e.Message);}
}
}
if (optionspaths["/paths"].Count == 0)
{
WriteErr("No files, file specifications, or directories specified. Try findup.exe -help. Assuming current directory.");
optionspaths["/paths"].Add(".");
}
Console.Write("Getting file info and sorting file list...");
getFiles(optionspaths["/paths"], "*.*", files, optionsbools["/recurse"], optionsbools["/xj"]);
if (files.Count < 2)
{
WriteErr("\nFindup.exe needs at least 2 files to compare. Try findup.exe -help");
Console.WriteLine("\n");
return;
}
files.Sort(new FileLengthComparer());
Console.WriteLine("Completed!");
Console.WriteLine("Building dictionary of file sizes, SHA512 hashes and full paths...");
var SizeHashFile = new Dictionary<long, Dictionary<string,List<FileInfo>>>();
long filesize = 0;
for (i = 0; i < (files.Count - 1); i++)
{
if (files[i].Length != files[i + 1].Length) continue;
var breakout = false;
while (true)
{
filesize = (files[i].Length);
try
{
var _SHA512 = SHA512.Create();
using (var fstream = File.OpenRead(files[i].FullName))
{
_SHA512.ComputeHash(fstream);
}
string SHA512string = Hash2String(_SHA512.Hash);
if (!SizeHashFile.ContainsKey(filesize))
SizeHashFile.Add(filesize, new Dictionary<string,List<FileInfo>>());
if (!SizeHashFile[filesize].ContainsKey(SHA512string))
SizeHashFile[filesize][SHA512string] = new List<FileInfo>() {};
SizeHashFile[filesize][SHA512string].Add(files[i]);
}
catch (Exception e) { WriteErr("Hash error: " + e.Message); }
if (breakout == true) {break;}
i++;
if (i == (files.Count - 1)) { breakout = true; continue; }
breakout = (files[i].Length != files[i + 1].Length);
}
if (SizeHashFile.ContainsKey(filesize))
{
foreach (var HG in SizeHashFile[filesize])
{
if (HG.Value.Count > 1)
{
Console.WriteLine("{0:N0} Duplicate files. {1:N0} Bytes each. {2:N0} Bytes total : ", HG.Value.Count, filesize, filesize * HG.Value.Count);
foreach (var finfo in HG.Value)
{
Console.WriteLine(finfo.FullName);
numOfDupes++;
dupeBytes += finfo.Length;
if (optionsbools["/delete"])
if (DeleteDupe(finfo)) { bytesrecovered += finfo.Length; deletedfiles++; }
}
}
}
}
}
Console.WriteLine("\n ");
Console.WriteLine("Files checked : {0:N0}", files.Count); // display statistics and return to OS.
Console.WriteLine("Duplicate files : {0:N0}", numOfDupes);
Console.WriteLine("Duplicate bytes : {0:N0}", dupeBytes);
Console.WriteLine("Deleted duplicates : {0:N0}", deletedfiles);
Console.WriteLine("Bytes recovered : {0:N0}", bytesrecovered);
Console.WriteLine("Errors : {0:N0}", errors);
Console.WriteLine("Execution time : " + (DateTime.Now - startTime));
}
private static void WriteErr(string Str)
{
errors++;
if (!optionsbools["/noerr"])
Console.WriteLine(Str);
}
private static string Hash2String(Byte[] hasharray)
{
string SHA512string = "";
foreach (var c in hasharray)
{
SHA512string += String.Format("{0:x2}", c);
}
return SHA512string;
}
private static Boolean DeleteDupe(FileInfo Filenfo)
{
Console.Write("Delete this file <y/N> <ENTER>?");
delconfirm = Console.ReadLine();
if ((delconfirm[0] == 'Y') || (delconfirm[0] == 'y'))
{
try
{
Filenfo.Delete();
Console.WriteLine("File Successfully deleted.");
return true;
}
catch (Exception e) { Console.WriteLine("File could not be deleted: " + e.Message); }
}
return false;
}
private static Boolean CheckNames(string fullname)
{
var filePart = Path.GetFileName(fullname); // get file name only (e.g. "d:\temp\data.txt" -> "data.txt")
var dirPart = Path.GetDirectoryName(fullname).Substring(fullname.IndexOf(Path.VolumeSeparatorChar)+2); // remove drive & file (e.g. "d:\temp\data.txt" -> "temp")
if (CheckNamesWorker(fullname, "/x", "/xr", true))
return false;
if (CheckNamesWorker(filePart, "/xf", "/xfr", true))
return false;
if (CheckNamesWorker(dirPart, "/xd", "/xdr", true))
return false;
if (CheckNamesWorker(fullname, "/i", "/ir", false))
return false;
if (CheckNamesWorker(filePart, "/if", "/ifr", false))
return false;
if (CheckNamesWorker(dirPart, "/id", "/idr", false))
return false;
return true;
}
private static Boolean CheckNamesWorker(string filestr, string pathskey, string rgxkey, Boolean CheckType)
{
foreach (var filepath in optionspaths[pathskey])
{
if (filestr.ToLower().StartsWith(filepath.ToLower()) == CheckType)
return true;
}
foreach (var rgx in optionsregex[rgxkey])
{
if (rgx.IsMatch(filestr) == CheckType)
return true;
}
return false;
}
private static void getFiles(List<string> pathRec, string searchPattern, List<FileInfo> returnList, Boolean recursiveFlag = true, Boolean xj = true)
{
foreach (string d in pathRec) { getFiles(d, searchPattern, returnList, recursiveFlag, xj); }
}
private static void getFiles(string[] pathRec, string searchPattern, List<FileInfo> returnList, Boolean recursiveFlag = true, Boolean xj = true)
{
foreach (string d in pathRec) { getFiles(d, searchPattern, returnList, recursiveFlag, xj); }
}
private static void getFiles(string pathRec, string searchPattern, List<FileInfo> returnList, Boolean recursiveFlag = true, Boolean xj = true)
{
string dirPart;
string filePart;
if (File.Exists(pathRec))
{
try
{
FileInfo addf = (new FileInfo(pathRec));
if (((addf.Attributes & FileAttributes.ReparsePoint) == 0) || !xj)
if (CheckNames(addf.FullName))
returnList.Add(addf);
}
catch (Exception e) { WriteErr("Add file error: " + e.Message); }
}
else if (Directory.Exists(pathRec))
{
try
{
DirectoryInfo Dir = new DirectoryInfo(pathRec);
if (((Dir.Attributes & FileAttributes.ReparsePoint) == 0) || !xj)
foreach (FileInfo addf in (Dir.GetFiles(searchPattern)))
{
if (((addf.Attributes & FileAttributes.ReparsePoint) == 0) || !xj)
if (CheckNames(addf.FullName))
returnList.Add(addf);
}
}
catch (Exception e) { WriteErr("Add files from Directory error: " + e.Message); }
if (recursiveFlag)
{
try { getFiles((Directory.GetDirectories(pathRec)), searchPattern, returnList, recursiveFlag, xj); }
catch (Exception e) { WriteErr("Add Directory error: " + e.Message); }
}
}
else
{
try
{
filePart = Path.GetFileName(pathRec);
dirPart = Path.GetDirectoryName(pathRec);
}
catch (Exception e)
{
WriteErr("Parse error on: " + pathRec);
WriteErr(@"Make sure you don't end a directory with a \ when using quotes. The console arg parser doesn't accept that.");
WriteErr("Exception: " + e.Message);
return;
}
if (filePart.IndexOfAny(new char[] {'?','*'}) >= 0)
{
if ((dirPart == null) || (dirPart == ""))
dirPart = Directory.GetCurrentDirectory();
if (Directory.Exists(dirPart))
{
getFiles(dirPart, filePart, returnList, recursiveFlag, xj);
return;
}
}
WriteErr("Invalid file path, directory path, file specification, or program option specified: " + pathRec);
}
}
}
}