Have --catch-up=SESSION add the new hash files to SESSION

After the --catch-up=SESSION run finished, add its input files to
SESSION's recovery file so that SESSION can simply be resumed for
continuing with its old files as well as the new ones.

This can disabled with a new --no-catch-up-add option.

Author: magnumripper

doc/NEWS

@@ -210,7 +210,9 @@ Major changes from 1.9.0-jumbo-1 (May 2019) in this bleeding-edge version:
 
 - Add new option --catch-up=NAME, for running a new session only until it
   reaches the candidates tried count of a different, existing and paused
-  session, then exit.  [magnum; 2021]
+  session, then exit.  Unless the --no-catch-up-add option is also given,
+  the new input hashes (file names) used for the new session will be added
+  to the old session file once caught up.  [magnum; 2021, 2026]
 
 - Add BestCrypt Volume Encryption V4 format.  [Jean-Christophe Delaunay; 2021]
 

doc/OPTIONS

@@ -229,9 +229,11 @@ numbers printed on the status line mean?"
 Limit the session to running only until it reaches the candidates tried
 count of a different, existing and paused session NAME.  This can be
 used for adding more hashes to an existing job:  After the new session
-finishes, just concatenate the new hashes to the older session's hash
-file and resume the old session.  This will end up more or less as if
-both sets of hashes were in the original session to start with.
+finishes, the new hashes (file names) will be added to the older session
+file so you can just resume that session.  This will end up just as if both
+sets of hashes were in the original session to start with, with no wasted
+resources or time (assuming salted hashes).  If you do not wish the original
+session file to be touched, also add the --no-catch-up-add option.
 
 --make-charset=FILE		make a charset, overwriting FILE
 

src/john.c

@@ -1954,10 +1954,15 @@ static void john_done(void)
 				        mask_iter_warn);
 		}
 		if (event_abort && options.catchup && john_max_cands && status.cands >= john_max_cands) {
+			if (options.catchup_add)
+				rec_add_files(options.catchup);
 			event_abort = 0;
-			log_event("Done catching up with '%s'", options.catchup);
-			if (john_main_process)
-				fprintf(stderr, "Done catching up with '%s'\n", options.catchup);
+			log_event("Done catching up with '%s'%s", options.catchup,
+			          options.catchup_add ? ", and added this session's password file(s) to it" : "");
+			if (john_main_process) {
+				fprintf(stderr, "Done catching up with '%s'%s\n", options.catchup,
+				        options.catchup_add ? ", and added this session's password file(s) to it" : "");
+			}
 		}
 		if (event_abort) {
 			char *abort_msg = (aborted_by_timer) ?

src/options.c

@@ -139,6 +139,7 @@ static struct opt_entry opt_list[] = {
 	{"restore", FLG_RESTORE_SET, FLG_RESTORE_CHK, 0, ~FLG_RESTORE_SET & ~GETOPT_FLAGS, OPT_FMT_STR_ALLOC, &options.session},
 	{"session", FLG_SESSION, FLG_SESSION, FLG_CRACKING_SUP, OPT_REQ_PARAM, OPT_FMT_STR_ALLOC, &options.session},
 	{"catch-up", FLG_ONCE, 0, 0, OPT_REQ_PARAM, OPT_FMT_STR_ALLOC, &options.catchup},
+	{"catch-up-add", FLG_ONCE, 0, 0, USUAL_REQ_CLR | OPT_TRISTATE, NULL, &options.catchup_add},
 	{"status", FLG_STATUS_SET, FLG_STATUS_CHK, 0, ~FLG_STATUS_SET & ~GETOPT_FLAGS, OPT_FMT_STR_ALLOC, &options.session},
 	{"make-charset", FLG_MAKECHR_SET, FLG_MAKECHR_CHK, 0, FLG_CRACKING_CHK | FLG_SESSION | OPT_REQ_PARAM, OPT_FMT_STR_ALLOC, &options.charset},
 	{"show", FLG_SHOW_SET, FLG_SHOW_CHK, 0, FLG_CRACKING_SUP | FLG_MAKECHR_CHK, OPT_FMT_STR_ALLOC, &show_uncracked_str},
@@ -332,6 +333,8 @@ JOHN_USAGE_FORK \
 "--no-log                   Disables creation and writing to john.log file\n"  \
 "--bare-always-valid=Y      Treat bare hashes as valid (Y/N)\n" \
 "--catch-up=NAME            Catch up with existing (paused) session NAME\n" \
+"--no-catch-up-add          Do not add the new hashes (files) to NAME after\n" \
+"                           --catch-up=NAME caught up\n" \
 "--config=FILE              Use FILE instead of john.conf or john.ini\n" \
 "--encoding=NAME            Input encoding (eg. UTF-8, ISO-8859-1). See also\n" \
 "                           doc/ENCODINGS.\n" \
@@ -623,6 +626,9 @@ void opt_init(char *name, int argc, char **argv)
 	if (options.catchup && options.max_cands)
 		error_msg("Can't combine --max-candidates and --catch-up options\n");
 
+	if ((options.catchup_add != -1) && !options.catchup)
+		error_msg("The --no-catch-up-add option can only be used with --catch-up\n");
+
 	if (options.flags & FLG_STATUS_CHK) {
 #if OS_FORK
 		char *rec_name_orig = rec_name;

src/options.h

@@ -457,6 +457,8 @@ struct options_main {
 	int crack_status;
 /* --catch-up=oldsession */
 	char *catchup;
+/* --catch-up-add (tri-state) */
+	int catchup_add;
 #if defined(HAVE_OPENCL) || defined(HAVE_ZTEX)
 /* --mask-internal-target=N */
 	int req_int_cand_target;

src/recovery.c

@@ -763,3 +763,72 @@ uint64_t rec_read_cands(char *session)
 
 	return ret;
 }
+
+void rec_add_files(char *session)
+{
+	char *other_name;
+	FILE *other_file = NULL;
+	int64_t other_size = 0;
+
+	if (!john_main_process && options.node_min) {
+		char suffix[1 + 20 + sizeof(RECOVERY_SUFFIX)];
+
+		sprintf(suffix, ".%u%s", options.node_min, RECOVERY_SUFFIX);
+		other_name = path_session(session, suffix);
+	} else {
+		other_name = path_session(session, RECOVERY_SUFFIX);
+	}
+
+	if (!(other_file = fopen(other_name, "r+")))
+		pexit("fopen catch-up file: '%s'", other_name);
+
+#if !(__MINGW32__ || _MSC_VER)
+	if (jtr_lock(fileno(other_file), F_SETLK, F_RDLCK, other_name))
+		error_msg("Catch-Up session file '%s' is locked\n", other_name);
+#endif
+
+	jtr_fseek64(other_file, 0, SEEK_END);
+	if ((other_size = jtr_ftell64(other_file)) == -1)
+		pexit("ftell");
+	jtr_fseek64(other_file, 0, SEEK_SET);
+	if (other_size == 0) {
+		error_msg("Error, %s is empty\n", other_name);
+	}
+
+	char *other_data = mem_alloc_tiny(other_size + 1, MEM_ALIGN_NONE);
+	memset(other_data, 0, other_size + 1);
+
+	if (fread(other_data, 1, (size_t)other_size, other_file) != other_size) {
+		if (ferror(other_file))
+			pexit("fread");
+		error_msg("fread: Unexpected EOF\n");
+	}
+
+	char magic[16];
+	int argc;
+	unsigned offset;
+	sscanf(other_data, "%15s\n%d\n%nu", magic, &argc, &offset);
+
+	jtr_fseek64(other_file, 0, SEEK_SET);
+
+	fprintf(other_file, "%s\n%d\n", magic, argc + (int)options.passwd->count);
+
+	struct list_entry *current;
+	if ((current = options.passwd->head)) {
+		do {
+			fprintf(other_file, "%s\n", current->data);
+		} while ((current = current->next));
+	}
+
+	fprintf(other_file, "%s", other_data + offset);
+	fclose(other_file);
+
+	if (unlink(path_expand(rec_name)))
+		pexit("unlink: %s", path_expand(rec_name));
+
+	if (rec_file) {
+		if (fclose(rec_file))
+			pexit("fclose");
+		rec_file = NULL;
+	}
+}

src/recovery.h

@@ -118,4 +118,10 @@ extern void rec_restore_mode(int (*restore_mode)(FILE *file));
  */
 extern uint64_t rec_read_cands(char *session);
 
+/*
+ * Add the current session's password files to the file we caught up with.
+ * --catch-up=SESSION does this after successfully catching up.
+ */
+extern void rec_add_files(char *session);
+
 #endif