sha1-file: create shared-cache directory if it doesn't exist

The config variable `gvfs.sharedCache` contains the pathname to an alternate
<odb> that will be used by `gvfs-helper` to store dynamically-fetched missing
objects.  If this directory does not exist on disk, `prepare_alt_odb()` omits
this directory from the in-memory list of alternates.  This causes `git`
commands (and `gvfs-helper` in particular) to fall-back to `.git/objects` for
storage of these objects.  This disables the shared-cache and leads to poorer
performance.

Teach `alt_obj_usable()` and `prepare_alt_odb()`, match up the directory
named in `gvfs.sharedCache` with an entry in `.git/objects/info/alternates`
and force-create the `<odb>` root directory (and the associated `<odb>/pack`
directory) if necessary.

If the value of `gvfs.sharedCache` refers to a directory that is NOT listed
as an alternate, create an in-memory alternate entry in the odb-list.  (This
is similar to how GIT_ALTERNATE_OBJECT_DIRECTORIES works.)

This work happens the first time that `prepare_alt_odb()` is called.

Furthermore, teach the `--shared-cache=<odb>` command line option in
`gvfs-helper` (which is runs after the first call to `prepare_alt_odb()`)
to override the inherited shared-cache (and again, create the ODB directory
if necessary).

Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com>

Author: jeffhostetler

environment.c

@@ -125,7 +125,7 @@ int protect_hfs = PROTECT_HFS_DEFAULT;
 int protect_ntfs = PROTECT_NTFS_DEFAULT;
 int core_use_gvfs_helper;
 char *gvfs_cache_server_url;
-const char *gvfs_shared_cache_pathname;
+struct strbuf gvfs_shared_cache_pathname = STRBUF_INIT;
 
 /*
  * The character that begins a commented line in user-editable file
@@ -721,19 +721,17 @@ static int git_default_gvfs_config(const char *var, const char *value)
 	}
 
 	if (!strcmp(var, "gvfs.sharedcache") && value && *value) {
-		struct strbuf buf = STRBUF_INIT;
-		strbuf_addstr(&buf, value);
-		if (strbuf_normalize_path(&buf) < 0) {
+		strbuf_setlen(&gvfs_shared_cache_pathname, 0);
+		strbuf_addstr(&gvfs_shared_cache_pathname, value);
+		if (strbuf_normalize_path(&gvfs_shared_cache_pathname) < 0) {
 			/*
 			 * Pretend it wasn't set.  This will cause us to
 			 * fallback to ".git/objects" effectively.
 			 */
-			strbuf_release(&buf);
+			strbuf_release(&gvfs_shared_cache_pathname);
 			return 0;
 		}
-		strbuf_trim_trailing_dir_sep(&buf);
-
-		gvfs_shared_cache_pathname = strbuf_detach(&buf, NULL);
+		strbuf_trim_trailing_dir_sep(&gvfs_shared_cache_pathname);
 		return 0;
 	}
 

environment.h

@@ -166,7 +166,7 @@ extern int protect_hfs;
 extern int protect_ntfs;
 extern int core_use_gvfs_helper;
 extern char *gvfs_cache_server_url;
-extern const char *gvfs_shared_cache_pathname;
+extern struct strbuf gvfs_shared_cache_pathname;
 
 extern int core_apply_sparse_checkout;
 extern int core_sparse_checkout_cone;

gvfs-helper-client.c

@@ -1,5 +1,7 @@
 #define USE_THE_REPOSITORY_VARIABLE
 #include "git-compat-util.h"
+#include "dir.h"
+#include "environment.h"
 #include "gvfs-helper-client.h"
 #include "hex.h"
 #include "object-file.h"
@@ -181,13 +183,32 @@ static int gh_client__get__receive_response(
 	return err;
 }
 
+/*
+ * Select the preferred ODB for fetching missing objects.
+ * This should be the alternate with the same directory
+ * name as set in `gvfs.sharedCache`.
+ *
+ * Fallback to .git/objects if necessary.
+ */
 static void gh_client__choose_odb(void)
 {
+	struct odb_source *odb;
+
 	if (gh_client__chosen_odb)
 		return;
 
 	odb_prepare_alternates(the_repository->objects);
 	gh_client__chosen_odb = the_repository->objects->sources;
+
+	if (!gvfs_shared_cache_pathname.len)
+		return;
+
+	for (odb = the_repository->objects->sources->next; odb; odb = odb->next) {
+		if (!fspathcmp(odb->path, gvfs_shared_cache_pathname.buf)) {
+			gh_client__chosen_odb = odb;
+			return;
+		}
+	}
 }
 
 static int gh_client__get(enum gh_client__created *p_ghc)

gvfs-helper.c

@@ -81,10 +81,11 @@
 //
 //                 Fetch 1 or more objects.  If a cache-server is configured,
 //                 try it first.  Optionally fallback to the main Git server.
+//
 //                 Create 1 or more loose objects and/or packfiles in the
-//                 requested shared-cache directory (given on the command
-//                 line and which is reported at the beginning of the
-//                 response).
+//                 shared-cache ODB.  (The pathname of the selected ODB is
+//                 reported at the beginning of the response; this should
+//                 match the pathname given on the command line).
 //
 //                 git> get
 //                 git> <oid>
@@ -641,26 +642,88 @@ static int option_parse_cache_server_mode(const struct option *opt,
 }
 
 /*
- * Let command line args override "gvfs.sharedcache" config setting.
+ * Let command line args override "gvfs.sharedcache" config setting
+ * and override the value set by git_default_config().
+ *
+ * The command line is parsed *AFTER* the config is loaded, so
+ * prepared_alt_odb() has already been called any default or inherited
+ * shared-cache has already been set.
  *
- * It would be nice to move this to parse-options.c as an
- * OPTION_PATHNAME handler.  And maybe have flags for exists()
- * and is_directory().
+ * We have a chance to override it here.
  */
 static int option_parse_shared_cache_directory(const struct option *opt,
 					       const char *arg, int unset)
 {
+	struct strbuf buf_arg = STRBUF_INIT;
+
 	if (unset) /* should not happen */
 		return error(_("missing value for switch '%s'"),
 			     opt->long_name);
 
-	if (!is_directory(arg))
-		return error(_("value for switch '%s' is not a directory: '%s'"),
-			     opt->long_name, arg);
+	strbuf_addstr(&buf_arg, arg);
+	if (strbuf_normalize_path(&buf_arg) < 0) {
+		/*
+		 * Pretend command line wasn't given.  Use whatever
+		 * settings we already have from the config.
+		 */
+		strbuf_release(&buf_arg);
+		return 0;
+	}
+	strbuf_trim_trailing_dir_sep(&buf_arg);
+
+	if (!strbuf_cmp(&buf_arg, &gvfs_shared_cache_pathname)) {
+		/*
+		 * The command line argument matches what we got from
+		 * the config, so we're already setup correctly. (And
+		 * we have already verified that the directory exists
+		 * on disk.)
+		 */
+		strbuf_release(&buf_arg);
+		return 0;
+	}
+
+	else if (!gvfs_shared_cache_pathname.len) {
+		/*
+		 * A shared-cache was requested and we did not inherit one.
+		 * Try it, but let alt_odb_usable() secretly disable it if
+		 * it cannot create the directory on disk.
+		 */
+		strbuf_addbuf(&gvfs_shared_cache_pathname, &buf_arg);
 
-	gvfs_shared_cache_pathname = arg;
+		odb_add_to_alternates_memory(the_repository->objects, buf_arg.buf);
 
-	return 0;
+		strbuf_release(&buf_arg);
+		return 0;
+	}
+
+	else {
+		/*
+		 * The requested shared-cache is different from the one
+		 * we inherited.  Replace the inherited value with this
+		 * one, but smartly fallback if necessary.
+		 */
+		struct strbuf buf_prev = STRBUF_INIT;
+
+		strbuf_addbuf(&buf_prev, &gvfs_shared_cache_pathname);
+
+		strbuf_setlen(&gvfs_shared_cache_pathname, 0);
+		strbuf_addbuf(&gvfs_shared_cache_pathname, &buf_arg);
+
+		odb_add_to_alternates_memory(the_repository->objects, buf_arg.buf);
+
+		/*
+		 * alt_odb_usable() releases gvfs_shared_cache_pathname
+		 * if it cannot create the directory on disk, so fallback
+		 * to the previous choice when it fails.
+		 */
+		if (!gvfs_shared_cache_pathname.len)
+			strbuf_addbuf(&gvfs_shared_cache_pathname,
+				      &buf_prev);
+
+		strbuf_release(&buf_arg);
+		strbuf_release(&buf_prev);
+		return 0;
+	}
 }
 
 /*
@@ -961,24 +1024,20 @@ static void approve_cache_server_creds(void)
 }
 
 /*
- * Select the ODB directory where we will write objects that we
- * download.  If was given on the command line or define in the
- * config, use the local ODB (in ".git/objects").
+ * Get the pathname to the ODB where we write objects that we download.
  */
 static void select_odb(void)
 {
-	const char *odb_path = NULL;
+	odb_prepare_alternates(the_repository->objects);
 
 	strbuf_init(&gh__global.buf_odb_path, 0);
 
-	if (gvfs_shared_cache_pathname && *gvfs_shared_cache_pathname)
-		odb_path = gvfs_shared_cache_pathname;
-	else {
-		odb_prepare_alternates(the_repository->objects);
-		odb_path = the_repository->objects->sources->path;
-	}
-
-	strbuf_addstr(&gh__global.buf_odb_path, odb_path);
+	if (gvfs_shared_cache_pathname.len)
+		strbuf_addbuf(&gh__global.buf_odb_path,
+			      &gvfs_shared_cache_pathname);
+	else
+		strbuf_addstr(&gh__global.buf_odb_path,
+			      the_repository->objects->sources->path);
 }
 
 /*

odb.c

@@ -94,17 +94,66 @@ int odb_mkstemp(struct object_database *odb,
 	return xmkstemp_mode(temp_filename->buf, mode);
 }
 
+static int gvfs_matched_shared_cache_to_alternate;
+
 /*
  * Return non-zero iff the path is usable as an alternate object database.
  */
 static bool odb_is_source_usable(struct object_database *o, const char *path)
 {
+	extern struct strbuf gvfs_shared_cache_pathname;
 	int r;
 	struct strbuf normalized_objdir = STRBUF_INIT;
 	bool usable = false;
 
 	strbuf_realpath(&normalized_objdir, o->sources->path, 1);
 
+	if (!strcmp(path, gvfs_shared_cache_pathname.buf)) {
+		/*
+		 * `gvfs.sharedCache` is the preferred alternate that we
+		 * will use with `gvfs-helper.exe` to dynamically fetch
+		 * missing objects.  It is set during git_default_config().
+		 *
+		 * Make sure the directory exists on disk before we let the
+		 * stock code discredit it.
+		 */
+		struct strbuf buf_pack_foo = STRBUF_INIT;
+		enum scld_error scld;
+
+		/*
+		 * Force create the "<odb>" and "<odb>/pack" directories, if
+		 * not present on disk.  Append an extra bogus directory to
+		 * get safe_create_leading_directories() to see "<odb>/pack"
+		 * as a leading directory of something deeper (which it
+		 * won't create).
+		 */
+		strbuf_addf(&buf_pack_foo, "%s/pack/foo", path);
+
+		scld = safe_create_leading_directories(o->repo, buf_pack_foo.buf);
+		if (scld != SCLD_OK && scld != SCLD_EXISTS) {
+			error_errno(_("could not create shared-cache ODB '%s'"),
+				    gvfs_shared_cache_pathname.buf);
+
+			strbuf_release(&buf_pack_foo);
+
+			/*
+			 * Pretend no shared-cache was requested and
+			 * effectively fallback to ".git/objects" for
+			 * fetching missing objects.
+			 */
+			strbuf_release(&gvfs_shared_cache_pathname);
+			return 0;
+		}
+
+		/*
+		 * We know that there is an alternate (either from
+		 * .git/objects/info/alternates or from a memory-only
+		 * entry) associated with the shared-cache directory.
+		 */
+		gvfs_matched_shared_cache_to_alternate++;
+		strbuf_release(&buf_pack_foo);
+	}
+
 	/* Detect cases where alternate disappeared */
 	if (!is_directory(path)) {
 		error(_("object directory %s does not exist; "
@@ -622,6 +671,7 @@ int odb_for_each_alternate(struct object_database *odb,
 
 void odb_prepare_alternates(struct object_database *odb)
 {
+	extern struct strbuf gvfs_shared_cache_pathname;
 	struct strvec sources = STRVEC_INIT;
 
 	if (odb->loaded_alternates)
@@ -632,6 +682,35 @@ void odb_prepare_alternates(struct object_database *odb)
 	for (size_t i = 0; i < sources.nr; i++)
 		odb_add_alternate_recursively(odb, sources.v[i], 0);
 
+	if (gvfs_shared_cache_pathname.len &&
+	    !gvfs_matched_shared_cache_to_alternate) {
+		/*
+		 * There is no entry in .git/objects/info/alternates for
+		 * the requested shared-cache directory.  Therefore, the
+		 * odb-list does not contain this directory.
+		 *
+		 * Force this directory into the odb-list as an in-memory
+		 * alternate.  Implicitly create the directory on disk, if
+		 * necessary.
+		 *
+		 * See GIT_ALTERNATE_OBJECT_DIRECTORIES for another example
+		 * of this kind of usage.
+		 *
+		 * Note: This has the net-effect of allowing Git to treat
+		 * `gvfs.sharedCache` as an unofficial alternate.  This
+		 * usage should be discouraged for compatbility reasons
+		 * with other tools in the overall Git ecosystem (that
+		 * won't know about this trick).  It would be much better
+		 * for us to update .git/objects/info/alternates instead.
+		 * The code here is considered a backstop.
+		 */
+		parse_alternates(gvfs_shared_cache_pathname.buf, '\n', NULL, &sources);
+		odb_source_read_alternates(odb->sources, &sources);
+		for (size_t i = 0; i < sources.nr; i++)
+			odb_add_alternate_recursively(odb, sources.v[i], 0);
+
+	}
+
 	odb->loaded_alternates = 1;
 
 	strvec_clear(&sources);