gvfs: allow "virtualizing" objects

The idea is to allow blob objects to be missing from the local repository,
and to load them lazily on demand.

After discussing this idea on the mailing list, we will rename the feature
to "lazy clone" and work more on this.

Signed-off-by: Ben Peart <Ben.Peart@microsoft.com>
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>

Author: Ben Peart

connected.c

@@ -1,6 +1,7 @@
 #define USE_THE_REPOSITORY_VARIABLE
 
 #include "git-compat-util.h"
+#include "environment.h"
 #include "gettext.h"
 #include "gvfs.h"
 #include "hex.h"
@@ -52,6 +53,8 @@ int check_connected(oid_iterate_fn fn, void *cb_data,
 	 */
 	if (gvfs_config_is_set(the_repository, GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK))
 		return 0;
+	if (gvfs_virtualize_objects(the_repository))
+		return 0;
 
 	if (!opt)
 		opt = &defaults;

environment.c

@@ -80,6 +80,7 @@ int core_sparse_checkout_cone;
 int sparse_expect_files_outside_of_patterns;
 int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */
 unsigned long pack_size_limit_cfg;
+int core_virtualize_objects;
 int max_allowed_tree_depth =
 #ifdef _MSC_VER
 	/*

environment.h

@@ -215,5 +215,6 @@ extern int auto_comment_line_char;
 extern bool warn_on_auto_comment_char;
 #endif /* !WITH_BREAKING_CHANGES */
 
+extern int core_virtualize_objects;
 # endif /* USE_THE_REPOSITORY_VARIABLE */
 #endif /* ENVIRONMENT_H */

gvfs.c

@@ -16,6 +16,8 @@ static int early_core_gvfs_config(const char *var, const char *value,
 	if (!strcmp(var, "core.gvfs"))
 		core_gvfs = git_config_bool_or_int("core.gvfs", value, ctx->kvi,
 						   &core_gvfs_is_bool);
+	if (!strcmp(var, "core.virtualizeobjects"))
+		core_virtualize_objects = git_config_bool(var, value);
 	return 0;
 }
 
@@ -27,11 +29,16 @@ static void gvfs_load_config_value(struct repository *r)
 	if (r) {
 		repo_config_get_bool_or_int(r, "core.gvfs",
 					    &core_gvfs_is_bool, &core_gvfs);
-	} else if (startup_info->have_repository == 0)
+		repo_config_get_bool(r, "core.virtualizeobjects",
+				     &core_virtualize_objects);
+	} else if (startup_info->have_repository == 0) {
 		read_early_config(the_repository, early_core_gvfs_config, NULL);
-	else
+	} else {
 		repo_config_get_bool_or_int(the_repository, "core.gvfs",
 					    &core_gvfs_is_bool, &core_gvfs);
+		repo_config_get_bool(the_repository, "core.virtualizeobjects",
+				     &core_virtualize_objects);
+	}
 
 	/* Turn on all bits if a bool was set in the settings */
 	if (core_gvfs_is_bool && core_gvfs)
@@ -46,3 +53,9 @@ int gvfs_config_is_set(struct repository *r, int mask)
 	gvfs_load_config_value(r);
 	return (core_gvfs & mask) == mask;
 }
+
+int gvfs_virtualize_objects(struct repository *r)
+{
+	gvfs_load_config_value(r);
+	return core_virtualize_objects;
+}

gvfs.h

@@ -18,5 +18,6 @@ struct repository;
 #define GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS      (1 << 6)
 
 int gvfs_config_is_set(struct repository *r, int mask);
+int gvfs_virtualize_objects(struct repository *r);
 
 #endif /* GVFS_H */

odb.c

@@ -6,7 +6,9 @@
 #include "dir.h"
 #include "environment.h"
 #include "gettext.h"
+#include "gvfs.h"
 #include "hex.h"
+#include "hook.h"
 #include "khash.h"
 #include "lockfile.h"
 #include "loose.h"
@@ -26,6 +28,7 @@
 #include "submodule.h"
 #include "tmp-objdir.h"
 #include "trace2.h"
+#include "trace.h"
 #include "write-or-die.h"
 
 KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
@@ -656,6 +659,20 @@ void disable_obj_read_lock(void)
 	pthread_mutex_destroy(&obj_read_mutex);
 }
 
+static int run_read_object_hook(struct repository *r, const struct object_id *oid)
+{
+	struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT;
+	int ret;
+	uint64_t start;
+
+	start = getnanotime();
+	strvec_push(&opt.args, oid_to_hex(oid));
+	ret = run_hooks_opt(r, "read-object", &opt);
+	trace_performance_since(start, "run_read_object_hook");
+
+	return ret;
+}
+
 int fetch_if_missing = 1;
 
 static int register_all_submodule_sources(struct object_database *odb)
@@ -682,13 +699,15 @@ static int do_oid_object_info_extended(struct object_database *odb,
 	const struct cached_object *co;
 	const struct object_id *real = oid;
 	int already_retried = 0;
+	int tried_hook = 0;
 
 	if (flags & OBJECT_INFO_LOOKUP_REPLACE)
 		real = lookup_replace_object(odb->repo, oid);
 
 	if (is_null_oid(real))
 		return -1;
 
+retry:
 	co = find_cached_object(odb, real);
 	if (co) {
 		if (oi) {
@@ -725,6 +744,11 @@ static int do_oid_object_info_extended(struct object_database *odb,
 			for (source = odb->sources; source; source = source->next)
 				if (!packfile_store_read_object_info(source->packfiles, real, oi, flags))
 					return 0;
+			if (gvfs_virtualize_objects(odb->repo) && !tried_hook) {
+				tried_hook = 1;
+				if (!run_read_object_hook(odb->repo, oid))
+					goto retry;
+			}
 		}
 
 		/*