[PATCH 3/7] odb/streaming: support streaming arbitrary object types

The object database supports the ability to write object streams into
it. This functionality is used when we encounter a blob that is larger
than "core.bigFileThreshold" so that we don't have to soak large files
into memory.

As we only ever write large files, the infrastructure doesn't support
specifying any other object type than "blob". This limitation is quite
artificial though: there is no reason why we shouldn't support writing
arbitrary large objects with a stream. While it's very unlikely that we
encounter a huge object other than a blob, users are known to be
creative and sometimes like to inflict pain on themselves by creating
commits or trees that are huge.

Extend the infrastructure to support streaming arbitrary object types.
For now we don't use this functionality anywhere, but it brings us a bit
closer to unify `struct odb_read_stream` and `struct odb_write_stream`.

Signed-off-by: Patrick Steinhardt <[email protected]>
---
 builtin/unpack-objects.c      |  1 +
 object-file.c                 | 31 +++++++++++++++----------------
 odb/source-inmemory.c         |  2 +-
 odb/source-loose.c            |  2 +-
 odb/streaming.c               |  3 ++-
 odb/streaming.h               |  3 ++-
 t/unit-tests/u-odb-inmemory.c |  7 +++++--
 7 files changed, 27 insertions(+), 22 deletions(-)

diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c
index b7c486ea94..7439ec53be 100644
--- a/builtin/unpack-objects.c
+++ b/builtin/unpack-objects.c
@@ -393,6 +393,7 @@ static void stream_blob(unsigned long size, unsigned nr)
 		.read = feed_input_zstream,
 		.data = &data,
 		.size = size,
+		.type = OBJ_BLOB,
 	};
 	struct obj_info *info = &obj_list[nr];
 
diff --git a/object-file.c b/object-file.c
index 317c09dff8..699a6a008c 100644
--- a/object-file.c
+++ b/object-file.c
@@ -702,9 +702,9 @@ static void prepare_packfile_transaction(struct odb_transaction_files *transacti
 		die_errno("unable to write pack header");
 }
 
-static int hash_blob_stream(struct odb_write_stream *stream,
-			    const struct git_hash_algo *hash_algo,
-			    struct object_id *result_oid)
+static int hash_stream(struct odb_write_stream *stream,
+		       const struct git_hash_algo *hash_algo,
+		       struct object_id *result_oid)
 {
 	unsigned char buf[16384];
 	struct git_hash_ctx ctx;
@@ -712,7 +712,7 @@ static int hash_blob_stream(struct odb_write_stream *stream,
 	size_t bytes_hashed = 0;
 
 	header_len = format_object_header((char *)buf, sizeof(buf),
-					  OBJ_BLOB, stream->size);
+					  stream->type, stream->size);
 	git_hash_init(&ctx, hash_algo);
 	git_hash_update(&ctx, buf, header_len);
 
@@ -740,9 +740,9 @@ static int hash_blob_stream(struct odb_write_stream *stream,
  * Read the contents from the stream provided, streaming it to the
  * packfile in state while updating the hash in ctx.
  */
-static void stream_blob_to_pack(struct transaction_packfile *state,
-				struct git_hash_ctx *ctx,
-				struct odb_write_stream *stream)
+static void stream_to_pack(struct transaction_packfile *state,
+			   struct git_hash_ctx *ctx,
+			   struct odb_write_stream *stream)
 {
 	git_zstream s;
 	unsigned char ibuf[16384];
@@ -755,7 +755,7 @@ static void stream_blob_to_pack(struct transaction_packfile *state,
 
 	git_deflate_init(&s, cfg->pack_compression_level);
 
-	hdrlen = encode_in_pack_object_header(obuf, sizeof(obuf), OBJ_BLOB, stream->size);
+	hdrlen = encode_in_pack_object_header(obuf, sizeof(obuf), stream->type, stream->size);
 	s.next_out = obuf + hdrlen;
 	s.avail_out = sizeof(obuf) - hdrlen;
 
@@ -764,7 +764,7 @@ static void stream_blob_to_pack(struct transaction_packfile *state,
 			ssize_t rsize = odb_write_stream_read(stream, ibuf,
 							      sizeof(ibuf));
 			if (rsize < 0)
-				die("failed to read blob data");
+				die("failed to read object data");
 			if (!rsize)
 				is_finished = true;
 
@@ -797,7 +797,7 @@ static void stream_blob_to_pack(struct transaction_packfile *state,
 	}
 
 	if (bytes_read != stream->size)
-		die("read %" PRIuMAX " bytes of blob data, but expected %" PRIuMAX " bytes",
+		die("read %" PRIuMAX " bytes of object data, but expected %" PRIuMAX " bytes",
 		    (uintmax_t)bytes_read, (uintmax_t)stream->size);
 
 	git_deflate_end(&s);
@@ -868,7 +868,7 @@ static void flush_packfile_transaction(struct odb_transaction_files *transaction
  * result, which we need to know beforehand when writing a git object.
  * Since the primary motivation for trying to stream from the working
  * tree file and to avoid mmaping it in core is to deal with large
- * binary blobs, they generally do not want to get any conversion, and
+ * objects, they generally do not want to get any conversion, and
  * callers should avoid this code path when filters are requested.
  */
 static int odb_transaction_files_write_object_stream(struct odb_transaction *base,
@@ -886,7 +886,7 @@ static int odb_transaction_files_write_object_stream(struct odb_transaction *bas
 	struct pack_idx_entry *idx;
 
 	header_len = format_object_header((char *)obuf, sizeof(obuf),
-					  OBJ_BLOB, stream->size);
+					  stream->type, stream->size);
 	git_hash_init(&ctx, transaction->base.source->odb->repo->hash_algo);
 	git_hash_update(&ctx, obuf, header_len);
 
@@ -911,7 +911,7 @@ static int odb_transaction_files_write_object_stream(struct odb_transaction *bas
 	hashfile_checkpoint(state->f, &checkpoint);
 	idx->offset = state->offset;
 	crc32_begin(state->f);
-	stream_blob_to_pack(state, &ctx, stream);
+	stream_to_pack(state, &ctx, stream);
 	git_hash_final_oid(result_oid, &ctx);
 
 	idx->crc32 = crc32_end(state->f);
@@ -953,7 +953,7 @@ int index_fd(struct index_state *istate, struct object_id *oid,
 				 type, path, flags);
 	} else {
 		struct odb_write_stream stream;
-		odb_write_stream_from_fd(&stream, fd, xsize_t(st->st_size));
+		odb_write_stream_from_fd(&stream, fd, xsize_t(st->st_size), OBJ_BLOB);
 
 		if (flags & INDEX_WRITE_OBJECT) {
 			struct object_database *odb = the_repository->objects;
@@ -968,8 +968,7 @@ int index_fd(struct index_state *istate, struct object_id *oid,
 			if (!inflight)
 				odb_transaction_commit(transaction);
 		} else {
-			ret = hash_blob_stream(&stream,
-					       the_repository->hash_algo, oid);
+			ret = hash_stream(&stream, the_repository->hash_algo, oid);
 		}
 
 		odb_write_stream_release(&stream);
diff --git a/odb/source-inmemory.c b/odb/source-inmemory.c
index 01bb81c63c..4f76db5496 100644
--- a/odb/source-inmemory.c
+++ b/odb/source-inmemory.c
@@ -293,7 +293,7 @@ static int odb_source_inmemory_write_object_stream(struct odb_source *source,
 	hash_object_file(source->odb->repo->hash_algo, data, total_read, OBJ_BLOB, oid);
 
 	ret = odb_source_inmemory_write_object(source, data, stream->size,
-					       OBJ_BLOB, oid, NULL, NULL, 0);
+					       stream->type, oid, NULL, NULL, 0);
 	if (ret < 0)
 		goto out;
 
diff --git a/odb/source-loose.c b/odb/source-loose.c
index 361b4e2a2a..5681a38f03 100644
--- a/odb/source-loose.c
+++ b/odb/source-loose.c
@@ -868,7 +868,7 @@ static int odb_source_loose_write_object_stream(struct odb_source *source,
 
 	/* Since oid is not determined, save tmp file to odb path. */
 	strbuf_addf(&filename, "%s/", loose->base.path);
-	hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, in_stream->size);
+	hdrlen = format_object_header(hdr, sizeof(hdr), in_stream->type, in_stream->size);
 
 	/*
 	 * Common steps for write_loose_object and stream_loose_object to
diff --git a/odb/streaming.c b/odb/streaming.c
index 912e75e682..0918cad426 100644
--- a/odb/streaming.c
+++ b/odb/streaming.c
@@ -324,7 +324,7 @@ static ssize_t read_object_fd(struct odb_write_stream *stream,
 }
 
 void odb_write_stream_from_fd(struct odb_write_stream *stream, int fd,
-			      size_t size)
+			      size_t size, enum object_type type)
 {
 	struct read_object_fd_data *data;
 
@@ -335,4 +335,5 @@ void odb_write_stream_from_fd(struct odb_write_stream *stream, int fd,
 	stream->data = data;
 	stream->read = read_object_fd;
 	stream->size = size;
+	stream->type = type;
 }
diff --git a/odb/streaming.h b/odb/streaming.h
index 5e8e6e532e..3c8ed55129 100644
--- a/odb/streaming.h
+++ b/odb/streaming.h
@@ -56,6 +56,7 @@ struct odb_write_stream {
 	ssize_t (*read)(struct odb_write_stream *, unsigned char *, size_t);
 	void *data;
 	size_t size;
+	enum object_type type;
 };
 
 /*
@@ -92,6 +93,6 @@ int odb_stream_blob_to_fd(struct object_database *odb,
  * Sets up an ODB write stream that reads from an fd.
  */
 void odb_write_stream_from_fd(struct odb_write_stream *stream, int fd,
-			      size_t size);
+			      size_t size, enum object_type type);
 
 #endif /* STREAMING_H */
diff --git a/t/unit-tests/u-odb-inmemory.c b/t/unit-tests/u-odb-inmemory.c
index 4437140ed0..1ab07af6d6 100644
--- a/t/unit-tests/u-odb-inmemory.c
+++ b/t/unit-tests/u-odb-inmemory.c
@@ -297,8 +297,11 @@ void test_odb_inmemory__write_object_stream(void)
 	struct odb_source_inmemory *source = odb_source_inmemory_new(odb);
 	const char data[] = "foobar";
 	struct membuf_write_stream stream = {
-		.base.read = membuf_write_stream_read,
-		.base.size = strlen(data),
+		.base = {
+			.read = membuf_write_stream_read,
+			.size = strlen(data),
+			.type = OBJ_BLOB,
+		},
 		.buf = data,
 	};
 	struct object_id written_oid;

-- 
2.55.0.679.g6767b8d81c.dirty