Re: Strange FS download behaviour
Alessio Vanni <[email protected]> Sun, 9 Feb 2025 16:29:19 +0100
| Newsgroups | gmane.network.gnunet.devel |
|---|---|
| Message-ID | <[email protected]> |
This is a multi-part message in MIME format. --------------08048EB8EFF71E62F27DD918 Content-Type: text/plain; charset=utf-8; format=flowed Content-Transfer-Encoding: 7bit Hello, I added the proposed wrapper. It required moving around a few parts of the FS service header, so I'm submitting it for review to ensure I didn't make any error without noticing. The patch contains changes to gnunet-publish too and a simple test for the new API. Thanks, A.V. On 02/07/25 21:48, Martin Schanzenbach wrote: > I see. > > After a brief look at gnunet-publish and the fs service API I think > adding this is fine. > > > BR > > Martin --------------08048EB8EFF71E62F27DD918 Content-Type: text/x-patch; charset=UTF-8; name="0001-FS-create-file-information-from-a-share-tree-item.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="0001-FS-create-file-information-from-a-share-tree-item.patch" From fc5564e0fc8706817472b65008b67c0a1469ff0e Mon Sep 17 00:00:00 2001 From: Alessio Vanni <[email protected]> Date: Sun, 9 Feb 2025 16:19:12 +0100 Subject: [PATCH] FS: create file information from a share tree item NEWS: File information for publishing can now be created from share tree items --- src/cli/fs/gnunet-publish.c | 12 +- src/include/gnunet_fs_service.h | 4242 +++++++++++---------- src/service/fs/fs_file_information.c | 24 + src/service/fs/test_fs_file_information.c | 56 +- 4 files changed, 2216 insertions(+), 2118 deletions(-) diff --git a/src/cli/fs/gnunet-publish.c b/src/cli/fs/gnunet-publish.c index 999318c3c..3e1e90af7 100644 --- a/src/cli/fs/gnunet-publish.c +++ b/src/cli/fs/gnunet-publish.c @@ -552,13 +552,11 @@ get_file_information (struct GNUNET_FS_ShareTreeItem *item) } else { - fi = GNUNET_FS_file_information_create_from_file (fs_handle, - NULL, - item->filename, - item->ksk_uri, - item->meta, - ! do_insert, - &bo); + fi = GNUNET_FS_file_information_create_from_share_tree_item (fs_handle, + NULL, + item, + ! do_insert, + &bo); } return fi; } diff --git a/src/include/gnunet_fs_service.h b/src/include/gnunet_fs_service.h index d9bd3a4dd..f8561d5c1 100644 --- a/src/include/gnunet_fs_service.h +++ b/src/include/gnunet_fs_service.h @@ -558,1013 +558,1049 @@ GNUNET_FS_uri_ksk_create_from_meta_data (const struct GNUNET_FS_MetaData *md); -/* ******************** command-line option parsing API *********************** */ + +/* ****************** metadata ******************* */ + /** - * Allow user to specify keywords. - * - * @param shortName short name of the option - * @param name long name of the option - * @param argumentHelp help text for the option argument - * @param description long help text for the option - * @param[out] topKeywords set to the desired value + * Meta data processing + * @defgroup metadata Metadata + * GNU libextractor key-value pairs */ -struct GNUNET_GETOPT_CommandLineOption -GNUNET_FS_GETOPT_KEYWORDS (char shortName, - const char *name, - const char *argumentHelp, - const char *description, - struct GNUNET_FS_Uri **topKeywords); + /** - * Allow user to specify metadata. + * @ingroup metadata + * Create a fresh meta data container. * - * @param shortName short name of the option - * @param name long name of the option - * @param argumentHelp help text for the option argument - * @param description long help text for the option - * @param[out] meta metadata set to the desired value + * @return empty meta-data container */ -struct GNUNET_GETOPT_CommandLineOption -GNUNET_FS_GETOPT_METADATA (char shortName, - const char *name, - const char *argumentHelp, - const char *description, - struct GNUNET_FS_MetaData **meta); +struct GNUNET_FS_MetaData * +GNUNET_FS_meta_data_create (void); + /** - * Command-line option parser function that allows the user to specify - * one or more '-m' options with metadata. Each specified entry of - * the form "type=value" will be added to the metadata. A pointer to - * the metadata must be passed as the "scls" argument. + * @ingroup metadata + * Duplicate a MetaData token. * - * @param ctx command line processor context - * @param scls must be of type `struct GNUNET_FS_MetaData **` - * @param option name of the option (typically 'k') - * @param value command line argument given - * @return #GNUNET_OK on success + * @param md what to duplicate + * @return duplicate meta-data container */ -int -GNUNET_FS_getopt_set_metadata (struct - GNUNET_GETOPT_CommandLineProcessorContext *ctx, - void *scls, - const char *option, - const char *value); - - -/* ************************* sharing API ***************** */ +struct GNUNET_FS_MetaData * +GNUNET_FS_meta_data_duplicate ( + const struct GNUNET_FS_MetaData *md); /** - * Possible status codes used in the callback for the - * various file-sharing operations. On each file (or search), - * the callback is guaranteed to be called once with "START" - * and once with STOPPED; calls with PROGRESS, ERROR or COMPLETED - * are optional and depend on the circumstances; parent operations - * will be STARTED before child-operations and STOPPED after - * their respective child-operations. START and STOP signals - * are typically generated either due to explicit client requests - * or because of suspend/resume operations. + * @ingroup metadata + * Free meta data. + * + * @param md what to free */ -enum GNUNET_FS_Status -{ - /** - * Notification that we have started to publish a file structure. - */ - GNUNET_FS_STATUS_PUBLISH_START = 0, +void +GNUNET_FS_meta_data_destroy (struct GNUNET_FS_MetaData *md); - /** - * Notification that we have resumed sharing a file structure. - */ - GNUNET_FS_STATUS_PUBLISH_RESUME = 1, - /** - * Notification that we have suspended sharing a file structure. - */ - GNUNET_FS_STATUS_PUBLISH_SUSPEND = 2, +/** + * @ingroup metadata + * Test if two MDs are equal. We consider them equal if + * the meta types, formats and content match (we do not + * include the mime types and plugins names in this + * consideration). + * + * @param md1 first value to check + * @param md2 other value to check + * @return #GNUNET_YES if they are equal + */ +int +GNUNET_FS_meta_data_test_equal ( + const struct GNUNET_FS_MetaData *md1, + const struct GNUNET_FS_MetaData *md2); - /** - * Notification that we are making progress sharing a file structure. - */ - GNUNET_FS_STATUS_PUBLISH_PROGRESS = 3, - /** - * Notification that an error was encountered sharing a file structure. - * The application will continue to receive resume/suspend events for - * this structure until "GNUNET_FS_publish_stop" is called. - */ - GNUNET_FS_STATUS_PUBLISH_ERROR = 4, +/** + * @ingroup metadata + * Extend metadata. + * + * @param md metadata to extend + * @param plugin_name name of the plugin that produced this value; + * special values can be used (e.g. '<zlib>' for zlib being + * used in the main libextractor library and yielding + * meta data). + * @param type libextractor-type describing the meta data + * @param format basic format information about data + * @param data_mime_type mime-type of data (not of the original file); + * can be NULL (if mime-type is not known) + * @param data actual meta-data found + * @param data_size number of bytes in data + * @return #GNUNET_OK on success, #GNUNET_SYSERR if this entry already exists + * data_mime_type and plugin_name are not considered for "exists" checks + */ +int +GNUNET_FS_meta_data_insert (struct GNUNET_FS_MetaData *md, + const char *plugin_name, + enum EXTRACTOR_MetaType type, + enum EXTRACTOR_MetaFormat format, + const char *data_mime_type, + const char *data, + size_t data_size); - /** - * Notification that we completed sharing a file structure. - * The application will continue to receive resume/suspend events for - * this structure until "GNUNET_FS_publish_stop" is called. - */ - GNUNET_FS_STATUS_PUBLISH_COMPLETED = 5, - /** - * Notification that we have stopped - * the process of uploading a file structure; no - * further events will be generated for this action. - */ - GNUNET_FS_STATUS_PUBLISH_STOPPED = 6, +/** + * @ingroup metadata + * Extend metadata. Merges the meta data from the second argument + * into the first, discarding duplicate key-value pairs. + * + * @param md metadata to extend + * @param in metadata to merge + */ +void +GNUNET_FS_meta_data_merge (struct GNUNET_FS_MetaData *md, + const struct GNUNET_FS_MetaData *in); - /** - * Notification that we have started this download. - */ - GNUNET_FS_STATUS_DOWNLOAD_START = 7, - /** - * Notification that this download is being resumed. - */ - GNUNET_FS_STATUS_DOWNLOAD_RESUME = 8, +/** + * @ingroup metadata + * Remove an item. + * + * @param md metadata to manipulate + * @param type type of the item to remove + * @param data specific value to remove, NULL to remove all + * entries of the given type + * @param data_size number of bytes in data + * @return #GNUNET_OK on success, #GNUNET_SYSERR if the item does not exist in md + */ +int +GNUNET_FS_meta_data_delete (struct GNUNET_FS_MetaData *md, + enum EXTRACTOR_MetaType type, + const char *data, + size_t data_size); - /** - * Notification that this download was suspended. - */ - GNUNET_FS_STATUS_DOWNLOAD_SUSPEND = 9, - /** - * Notification about progress with this download. - */ - GNUNET_FS_STATUS_DOWNLOAD_PROGRESS = 10, +/** + * @ingroup metadata + * Remove all items in the container. + * + * @param md metadata to manipulate + */ +void +GNUNET_FS_meta_data_clear (struct GNUNET_FS_MetaData *md); - /** - * Notification that this download encountered an error. - */ - GNUNET_FS_STATUS_DOWNLOAD_ERROR = 11, - /** - * Notification that this download completed. Note that for - * directories, completion does not imply completion of all files in - * the directory. - */ - GNUNET_FS_STATUS_DOWNLOAD_COMPLETED = 12, +/** + * @ingroup metadata + * Add the current time as the publication date + * to the meta-data. + * + * @param md metadata to modify + */ +void +GNUNET_FS_meta_data_add_publication_date ( + struct GNUNET_FS_MetaData *md); - /** - * Notification that this download was stopped - * (final event with respect to this action). - */ - GNUNET_FS_STATUS_DOWNLOAD_STOPPED = 13, - /** - * Notification that this download is now actively being - * pursued (as opposed to waiting in the queue). - */ - GNUNET_FS_STATUS_DOWNLOAD_ACTIVE = 14, +/** + * @ingroup metadata + * Iterate over MD entries. + * + * @param md metadata to inspect + * @param iter function to call on each entry, return 0 to continue to iterate + * and 1 to abort iteration in this function (GNU libextractor API!) + * @param iter_cls closure for @a iter + * @return number of entries + */ +int +GNUNET_FS_meta_data_iterate (const struct GNUNET_FS_MetaData *md, + EXTRACTOR_MetaDataProcessor iter, + void *iter_cls); - /** - * Notification that this download is no longer actively - * being pursued (back in the queue). - */ - GNUNET_FS_STATUS_DOWNLOAD_INACTIVE = 15, - /** - * Notification that this download is no longer part of a - * recursive download or search but now a 'stand-alone' - * download (and may thus need to be moved in the GUI - * into a different category). - */ - GNUNET_FS_STATUS_DOWNLOAD_LOST_PARENT = 16, - - /** - * First event generated when a client requests - * a search to begin or when a namespace result - * automatically triggers the search for updates. - */ - GNUNET_FS_STATUS_SEARCH_START = 17, - - /** - * Last event when a search is being resumed; - * note that "GNUNET_FS_SEARCH_START" will not - * be generated in this case. - */ - GNUNET_FS_STATUS_SEARCH_RESUME = 18, - - /** - * Event generated for each search result - * when the respective search is resumed. - */ - GNUNET_FS_STATUS_SEARCH_RESUME_RESULT = 19, - - /** - * Last event when a search is being suspended; - * note that "GNUNET_FS_SEARCH_STOPPED" will not - * be generated in this case. - */ - GNUNET_FS_STATUS_SEARCH_SUSPEND = 20, - - /** - * This search has yielded a result. - */ - GNUNET_FS_STATUS_SEARCH_RESULT = 21, - - /** - * We have discovered a new namespace. - */ - GNUNET_FS_STATUS_SEARCH_RESULT_NAMESPACE = 22, - - /** - * We have additional data about the quality - * or availability of a search result. - */ - GNUNET_FS_STATUS_SEARCH_UPDATE = 23, - - /** - * Signals a problem with this search. - */ - GNUNET_FS_STATUS_SEARCH_ERROR = 24, - - /** - * Signals that this search was paused. - */ - GNUNET_FS_STATUS_SEARCH_PAUSED = 25, - - /** - * Signals that this search was continued (unpaused). - */ - GNUNET_FS_STATUS_SEARCH_CONTINUED = 26, - - /** - * Event generated for each search result - * when the respective search is stopped. - */ - GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED = 27, - - /** - * Event generated for each search result - * when the respective search is suspended. - */ - GNUNET_FS_STATUS_SEARCH_RESULT_SUSPEND = 28, - - /** - * Last message from a search; this signals - * that there will be no further events associated - * with this search. - */ - GNUNET_FS_STATUS_SEARCH_STOPPED = 29, - - /** - * Notification that we started to unindex a file. - */ - GNUNET_FS_STATUS_UNINDEX_START = 30, +/** + * @ingroup metadata + * Get the first MD entry of the given type. Caller + * is responsible for freeing the return value. + * Also, only meta data items that are strings (0-terminated) + * are returned by this function. + * + * @param md metadata to inspect + * @param type type to look for + * @return NULL if no entry was found + */ +char * +GNUNET_FS_meta_data_get_by_type ( + const struct GNUNET_FS_MetaData *md, + enum EXTRACTOR_MetaType type); - /** - * Notification that we resumed unindexing of a file. - */ - GNUNET_FS_STATUS_UNINDEX_RESUME = 31, - /** - * Notification that we suspended unindexing a file. - */ - GNUNET_FS_STATUS_UNINDEX_SUSPEND = 32, +/** + * @ingroup metadata + * Get the first matching MD entry of the given types. Caller is + * responsible for freeing the return value. Also, only meta data + * items that are strings (0-terminated) are returned by this + * function. + * + * @param md metadata to inspect + * @param ... -1-terminated list of types + * @return NULL if we do not have any such entry, + * otherwise client is responsible for freeing the value! + */ +char * +GNUNET_FS_meta_data_get_first_by_types ( + const struct GNUNET_FS_MetaData *md, + ...); - /** - * Notification that we made progress unindexing a file. - */ - GNUNET_FS_STATUS_UNINDEX_PROGRESS = 33, +/** + * @ingroup metadata + * Get a thumbnail from the meta-data (if present). Only matches meta + * data with mime type "image" and binary format. + * + * @param md metadata to inspect + * @param thumb will be set to the thumbnail data. Must be + * freed by the caller! + * @return number of bytes in thumbnail, 0 if not available + */ +size_t +GNUNET_FS_meta_data_get_thumbnail ( + const struct GNUNET_FS_MetaData *md, + unsigned char **thumb); - /** - * Notification that we encountered an error unindexing - * a file. - */ - GNUNET_FS_STATUS_UNINDEX_ERROR = 34, +/** + * @ingroup metadata + * Options for metadata serialization. + */ +enum GNUNET_FS_MetaDataSerializationOptions +{ /** - * Notification that the unindexing of this file - * was completed. + * @ingroup metadata + * Serialize all of the data. */ - GNUNET_FS_STATUS_UNINDEX_COMPLETED = 35, + GNUNET_FS_META_DATA_SERIALIZE_FULL = 0, /** - * Notification that the unindexing of this file - * was stopped (final event for this action). + * @ingroup metadata + * If not enough space is available, it is acceptable + * to only serialize some of the metadata. */ - GNUNET_FS_STATUS_UNINDEX_STOPPED = 36, + GNUNET_FS_META_DATA_SERIALIZE_PART = 1, /** - * Notification that we are making progress sharing a directory. + * @ingroup metadata + * Speed is of the essence, do not allow compression. */ - GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY = 37 + GNUNET_FS_META_DATA_SERIALIZE_NO_COMPRESS = 2 }; /** - * Handle for controlling an upload. + * @ingroup metadata + * Serialize meta-data to target. + * + * @param md metadata to serialize + * @param target where to write the serialized metadata; + * *target can be NULL, in which case memory is allocated + * @param max maximum number of bytes available + * @param opt is it ok to just write SOME of the + * meta-data to match the size constraint, + * possibly discarding some data? + * @return number of bytes written on success, + * -1 on error (typically: not enough + * space) */ -struct GNUNET_FS_PublishContext; +ssize_t +GNUNET_FS_meta_data_serialize ( + const struct GNUNET_FS_MetaData *md, + char **target, + size_t max, + enum GNUNET_FS_MetaDataSerializationOptions opt); /** - * Handle for controlling an unindexing operation. + * @ingroup metadata + * Get the size of the full meta-data in serialized form. + * + * @param md metadata to inspect + * @return number of bytes needed for serialization, -1 on error */ -struct GNUNET_FS_UnindexContext; +ssize_t +GNUNET_FS_meta_data_get_serialized_size ( + const struct GNUNET_FS_MetaData *md); /** - * Handle for controlling a search. + * @ingroup metadata + * Deserialize meta-data. Initializes md. + * + * @param input serialized meta-data. + * @param size number of bytes available + * @return MD on success, NULL on error (e.g. + * bad format) */ -struct GNUNET_FS_SearchContext; - +struct GNUNET_FS_MetaData * +GNUNET_FS_meta_data_deserialize (const char *input, size_t size); /** - * Result from a search. Opaque handle to refer to the search - * (typically used when starting a download associated with the search - * result). + * Write a metadata container. + * + * @param h the IO handle to write to + * @param what what is being written (for error message creation) + * @param m metadata to write + * @return #GNUNET_OK on success, #GNUNET_SYSERR on error */ -struct GNUNET_FS_SearchResult; - +enum GNUNET_GenericReturnValue +GNUNET_FS_write_meta_data (struct GNUNET_BIO_WriteHandle *h, + const char *what, + const struct GNUNET_FS_MetaData *m); /** - * Context for controlling a download. + * Create the specification to read a metadata container. + * + * @param what describes what is being read (for error message creation) + * @param result the buffer to store a pointer to the (allocated) metadata + * @return the read spec */ -struct GNUNET_FS_DownloadContext; +struct GNUNET_BIO_ReadSpec +GNUNET_FS_read_spec_meta_data (const char *what, + struct GNUNET_FS_MetaData **result); + /** - * Handle for detail information about a file that is being published. - * Specifies metadata, keywords, how to get the contents of the file - * (i.e. data-buffer in memory, filename on disk) and other options. + * Create the specification to write a metadata container. + * + * @param what what is being written (for error message creation) + * @param m metadata to write + * @return the write spec */ -struct GNUNET_FS_FileInformation; - +struct GNUNET_BIO_WriteSpec +GNUNET_FS_write_spec_meta_data (const char *what, + const struct GNUNET_FS_MetaData *m); /** - * Argument given to the progress callback with - * information about what is going on. + * Read a metadata container. + * + * @param h handle to an open file + * @param what describes what is being read (for error message creation) + * @param result the buffer to store a pointer to the (allocated) metadata + * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure */ -struct GNUNET_FS_ProgressInfo -{ - /** - * Values that depend on the event type. - */ - union - { - /** - * Values for all "GNUNET_FS_STATUS_PUBLISH_*" events. - */ - struct GNUNET_FS_PublishStatusEvent - { - /** - * Context for controlling the upload. - */ - struct GNUNET_FS_PublishContext *pc; - - /** - * Information about the file that is being published. - */ - const struct GNUNET_FS_FileInformation *fi; - - /** - * Client context pointer (set the last time by the client for - * this operation; initially NULL on START/RESUME events). - */ - void *cctx; +enum GNUNET_GenericReturnValue +GNUNET_FS_read_meta_data (struct GNUNET_BIO_ReadHandle *h, + const char *what, + struct GNUNET_FS_MetaData **result); - /** - * Client context pointer for the parent operation - * (if this is a file in a directory or a subdirectory). - */ - void *pctx; - /** - * Name of the file being published; can be NULL. - */ - const char *filename; +/* ******************** command-line option parsing API *********************** */ - /** - * How large is the file overall? For directories, - * this is only the size of the directory itself, - * not of the other files contained within the - * directory. - */ - uint64_t size; +/** + * Allow user to specify keywords. + * + * @param shortName short name of the option + * @param name long name of the option + * @param argumentHelp help text for the option argument + * @param description long help text for the option + * @param[out] topKeywords set to the desired value + */ +struct GNUNET_GETOPT_CommandLineOption +GNUNET_FS_GETOPT_KEYWORDS (char shortName, + const char *name, + const char *argumentHelp, + const char *description, + struct GNUNET_FS_Uri **topKeywords); - /** - * At what time do we expect to finish the upload? - * (will be a value in the past for completed - * uploads). - */ - struct GNUNET_TIME_Relative eta; +/** + * Allow user to specify metadata. + * + * @param shortName short name of the option + * @param name long name of the option + * @param argumentHelp help text for the option argument + * @param description long help text for the option + * @param[out] meta metadata set to the desired value + */ +struct GNUNET_GETOPT_CommandLineOption +GNUNET_FS_GETOPT_METADATA (char shortName, + const char *name, + const char *argumentHelp, + const char *description, + struct GNUNET_FS_MetaData **meta); - /** - * How long has this upload been actively running - * (excludes times where the upload was suspended). - */ - struct GNUNET_TIME_Relative duration; +/** + * Command-line option parser function that allows the user to specify + * one or more '-m' options with metadata. Each specified entry of + * the form "type=value" will be added to the metadata. A pointer to + * the metadata must be passed as the "scls" argument. + * + * @param ctx command line processor context + * @param scls must be of type `struct GNUNET_FS_MetaData **` + * @param option name of the option (typically 'k') + * @param value command line argument given + * @return #GNUNET_OK on success + */ +int +GNUNET_FS_getopt_set_metadata (struct + GNUNET_GETOPT_CommandLineProcessorContext *ctx, + void *scls, + const char *option, + const char *value); - /** - * How many bytes have we completed? - */ - uint64_t completed; - /** - * What anonymity level is used for this upload? - */ - uint32_t anonymity; +/* ******************** Directory API *********************** */ - /** - * Additional values for specific events. - */ - union - { - /** - * These values are only valid for - * #GNUNET_FS_STATUS_PUBLISH_PROGRESS events. - */ - struct GNUNET_FS_PublishProgressEvent - { - /** - * Data block we just published. - */ - const void *data; - /** - * At what offset in the file is "data"? - */ - uint64_t offset; +#define GNUNET_FS_DIRECTORY_MIME "application/gnunet-directory" +#define GNUNET_FS_DIRECTORY_MAGIC "\211GND\r\n\032\n" +#define GNUNET_FS_DIRECTORY_EXT ".gnd" - /** - * Length of the data block. - */ - uint64_t data_len; +/** + * Does the meta-data claim that this is a directory? + * Checks if the mime-type is that of a GNUnet directory. + * + * @return #GNUNET_YES if it is, #GNUNET_NO if it is not, #GNUNET_SYSERR if + * we have no mime-type information (treat as #GNUNET_NO) + */ +int +GNUNET_FS_meta_data_test_for_directory (const struct GNUNET_FS_MetaData + *md); - /** - * Depth of the given block in the tree; - * 0 would be the lowest level (DBLOCKs). - */ - unsigned int depth; - } progress; - /** - * These values are only valid for - * #GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY events. - */ - struct GNUNET_FS_PublishProgressDirectoryEvent - { - /** - * How far are we along in the overall directory? - */ - uint64_t completed; +/** + * Set the MIMETYPE information for the given + * metadata to "application/gnunet-directory". + * + * @param md metadata to add mimetype to + */ +void +GNUNET_FS_meta_data_make_directory (struct GNUNET_FS_MetaData *md); - /** - * How big do we estimate the entire directory to be? - */ - uint64_t total; - /** - * At what time do we expect to finish the upload of the - * CONTENTS of the directory. (The directory itself will take - * extra time, indicated with the "eta" member at the - * "publish"-level of this struct.) - */ - struct GNUNET_TIME_Relative eta; - } progress_directory; +/** + * Suggest a filename based on given metadata. + * + * @param md given meta data + * @return NULL if meta data is useless for suggesting a filename + */ +char * +GNUNET_FS_meta_data_suggest_filename (const struct + GNUNET_FS_MetaData *md); - /** - * These values are only valid for - * #GNUNET_FS_STATUS_PUBLISH_RESUME events. - */ - struct GNUNET_FS_PublishResumeEvent - { - /** - * Error message, NULL if no error was encountered so far. - */ - const char *message; - /** - * URI of the file (if the download had been completed) - */ - const struct GNUNET_FS_Uri *chk_uri; +/** + * Function used to process entries in a directory. + * + * @param cls closure + * @param filename name of the file in the directory + * @param uri URI of the file + * @param metadata metadata for the file; metadata for + * the directory if everything else is NULL/zero + * @param length length of the available data for the file + * (of type size_t since data must certainly fit + * into memory; if files are larger than size_t + * permits, then they will certainly not be + * embedded with the directory itself). + * @param data data available for the file (length bytes) + */ +typedef void (*GNUNET_FS_DirectoryEntryProcessor) (void *cls, + const char *filename, + const struct GNUNET_FS_Uri * + uri, + const struct + GNUNET_FS_MetaData * + meta, size_t length, + const void *data); - /** - * SKS URI of the file (if the download had been completed) - */ - const struct GNUNET_FS_Uri *sks_uri; - } resume; - /** - * These values are only valid for - * #GNUNET_FS_STATUS_PUBLISH_COMPLETED events. - */ - struct GNUNET_FS_PublishCompletedEvent - { - /** - * CHK URI of the file. - */ - const struct GNUNET_FS_Uri *chk_uri; +/** + * Iterate over all entries in a directory. Note that directories + * are structured such that it is possible to iterate over the + * individual blocks as well as over the entire directory. Thus + * a client can call this function on the buffer in the + * GNUNET_FS_ProgressCallback. Also, directories can optionally + * include the contents of (small) files embedded in the directory + * itself; for those files, the processor may be given the + * contents of the file directly by this function. + * + * @param size number of bytes in data + * @param data pointer to the beginning of the directory + * @param offset offset of data in the directory + * @param dep function to call on each entry + * @param dep_cls closure for @a dep + * @return #GNUNET_OK if this could be a block in a directory, + * #GNUNET_NO if this could be part of a directory (but not 100% OK) + * #GNUNET_SYSERR if 'data' does not represent a directory + */ +int +GNUNET_FS_directory_list_contents (size_t size, const void *data, + uint64_t offset, + GNUNET_FS_DirectoryEntryProcessor dep, + void *dep_cls); - /** - * SKS URI of the file (if the download had been completed) - */ - const struct GNUNET_FS_Uri *sks_uri; - } completed; - /** - * These values are only valid for - * #GNUNET_FS_STATUS_PUBLISH_ERROR events. - */ - struct GNUNET_FS_PublishErrorEvent - { - /** - * Error message, never NULL. - */ - const char *message; - } error; - } specifics; - } publish; +/** + * Opaque handle to a directory builder. + */ +struct GNUNET_FS_DirectoryBuilder; - /** - * Values for all "GNUNET_FS_STATUS_DOWNLOAD_*" events. - */ - struct GNUNET_FS_DownloadStatusEvent - { - /** - * Context for controlling the download. - */ - struct GNUNET_FS_DownloadContext *dc; +/** + * Create a directory builder. + * + * @param mdir metadata for the directory + */ +struct GNUNET_FS_DirectoryBuilder * +GNUNET_FS_directory_builder_create (const struct GNUNET_FS_MetaData + *mdir); - /** - * Client context pointer (set the last time - * by the client for this operation; initially - * NULL on START/RESUME events). - */ - void *cctx; - /** - * Client context pointer for the parent operation - * (if this is a file in a directory or a subdirectory). - */ - void *pctx; +/** + * Add an entry to a directory. + * + * @param bld directory to extend + * @param uri uri of the entry (must not be a KSK) + * @param md metadata of the entry + * @param data raw data of the entry, can be NULL, otherwise + * data must point to exactly the number of bytes specified + * by the uri + */ +void +GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld, + const struct GNUNET_FS_Uri *uri, + const struct GNUNET_FS_MetaData *md, + const void *data); - /** - * Client context pointer for the associated search operation - * (specifically, context pointer for the specific search - * result, not the overall search); only set if this - * download was started from a search result. - */ - void *sctx; - /** - * URI used for this download. - */ - const struct GNUNET_FS_Uri *uri; +/** + * Finish building the directory. Frees the + * builder context and returns the directory + * in-memory. + * + * @param bld directory to finish + * @param rsize set to the number of bytes needed + * @param rdata set to the encoded directory + * @return #GNUNET_OK on success + */ +int +GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld, + size_t *rsize, void **rdata); - /** - * Name of the file that we are downloading. - */ - const char *filename; - /** - * How large is the download overall? This - * is NOT necessarily the size from the - * URI since we may be doing a partial download. - */ - uint64_t size; +/* ******************** DirScanner API *********************** */ - /** - * At what time do we expect to finish the download? - * (will be a value in the past for completed - * uploads). - */ - struct GNUNET_TIME_Relative eta; +/** + * Progress reasons of the directory scanner. + */ +enum GNUNET_FS_DirScannerProgressUpdateReason +{ + /** + * We've started processing a file or directory. + */ + GNUNET_FS_DIRSCANNER_FILE_START = 0, - /** - * How long has this download been active? - */ - struct GNUNET_TIME_Relative duration; + /** + * We're having trouble accessing a file (soft-error); it will + * be ignored. + */ + GNUNET_FS_DIRSCANNER_FILE_IGNORED, - /** - * How many bytes have we completed? - */ - uint64_t completed; + /** + * We've found all files (in the pre-pass). + */ + GNUNET_FS_DIRSCANNER_ALL_COUNTED, - /** - * What anonymity level is used for this download? - */ - uint32_t anonymity; + /** + * We've finished extracting meta data from a file. + */ + GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED, - /** - * Is the download currently active. - */ - int is_active; + /** + * Last call to the progress function: we have finished scanning + * the directory. + */ + GNUNET_FS_DIRSCANNER_FINISHED, - /** - * Additional values for specific events. - */ - union - { - /** - * These values are only valid for - * #GNUNET_FS_STATUS_DOWNLOAD_PROGRESS events. - */ - struct GNUNET_FS_DownloadProgressEvent - { - /** - * Data block we just obtained, can be NULL (even if - * data_len > 0) if we found the entire block 'intact' on - * disk. In this case, it is also possible for 'data_len' - * to be larger than an individual (32k) block. - */ - const void *data; + /** + * There was an internal error. Application should abort the scan. + */ + GNUNET_FS_DIRSCANNER_INTERNAL_ERROR +}; - /** - * At what offset in the file is "data"? - */ - uint64_t offset; - /** - * Length of the data block. - */ - uint64_t data_len; +/** + * Function called over time as the directory scanner makes + * progress on the job at hand. + * + * @param cls closure + * @param filename which file we are making progress on + * @param is_directory #GNUNET_YES if this is a directory, + * #GNUNET_NO if this is a file + * #GNUNET_SYSERR if it is neither (or unknown) + * @param reason kind of progress we are making + */ +typedef void (*GNUNET_FS_DirScannerProgressCallback) (void *cls, + const char *filename, + int is_directory, + enum + GNUNET_FS_DirScannerProgressUpdateReason + reason); - /** - * How much time passed between us asking for this block and - * actually getting it? #GNUNET_TIME_UNIT_FOREVER_REL if unknown. - */ - struct GNUNET_TIME_Relative block_download_duration; - /** - * Depth of the given block in the tree; - * 0 would be the lowest level (DBLOCKS). - */ - unsigned int depth; +/** + * A node of a directory tree (produced by dirscanner) + */ +struct GNUNET_FS_ShareTreeItem +{ + /** + * This is a doubly-linked list + */ + struct GNUNET_FS_ShareTreeItem *prev; - /** - * How much respect did we offer for downloading this block? (estimate, - * because we might have the same request pending for multiple clients, - * and of course because a transmission may have failed at a lower - * layer). - */ - uint32_t respect_offered; + /** + * This is a doubly-linked list + */ + struct GNUNET_FS_ShareTreeItem *next; - /** - * How often did we transmit the request? (estimate, - * because we might have the same request pending for multiple clients, - * and of course because a transmission may have failed at a lower - * layer). - */ - uint32_t num_transmissions; - } progress; + /** + * This is a doubly-linked tree + * NULL for top-level entries. + */ + struct GNUNET_FS_ShareTreeItem *parent; - /** - * These values are only valid for - * #GNUNET_FS_STATUS_DOWNLOAD_START events. - */ - struct GNUNET_FS_DownloadStartEvent - { - /** - * Known metadata for the download. - */ - const struct GNUNET_FS_MetaData *meta; - } start; + /** + * This is a doubly-linked tree + * NULL for files and empty directories + */ + struct GNUNET_FS_ShareTreeItem *children_head; - /** - * These values are only valid for - * #GNUNET_FS_STATUS_DOWNLOAD_RESUME events. - */ - struct GNUNET_FS_DownloadResumeEvent - { - /** - * Known metadata for the download. - */ - const struct GNUNET_FS_MetaData *meta; + /** + * This is a doubly-linked tree + * NULL for files and empty directories + */ + struct GNUNET_FS_ShareTreeItem *children_tail; - /** - * Error message, NULL if we have not encountered any error yet. - */ - const char *message; - } resume; + /** + * Metadata for this file or directory + */ + struct GNUNET_FS_MetaData *meta; - /** - * These values are only valid for - * #GNUNET_FS_STATUS_DOWNLOAD_ERROR events. - */ - struct GNUNET_FS_DownloadErrorEvent - { - /** - * Error message. - */ - const char *message; - } error; - } specifics; - } download; + /** + * Keywords for this file or directory (derived from metadata). + */ + struct GNUNET_FS_Uri *ksk_uri; - /** - * Values for all "GNUNET_FS_STATUS_SEARCH_*" events. - */ - struct GNUNET_FS_SearchStatusEvent - { - /** - * Context for controlling the search, NULL for - * searches that were not explicitly triggered - * by the client (i.e., searches for updates in - * namespaces). - */ - struct GNUNET_FS_SearchContext *sc; + /** + * Name of the file/directory + */ + char *filename; - /** - * Client context pointer (set the last time by the client for - * this operation; initially NULL on START/RESUME events). Note - * that this value can only be set on START/RESUME; returning - * non-NULL on RESULT/RESUME_RESULT will actually update the - * private context for "UPDATE" events. - */ - void *cctx; + /** + * Base name of the file/directory. + */ + char *short_filename; - /** - * Client parent-context pointer; NULL for top-level searches, - * refers to the client context of the associated search result - * for automatically triggered searches for updates in - * namespaces. In this case, 'presult' refers to that search - * result. - */ - void *pctx; + /** + * #GNUNET_YES if this is a directory + */ + int is_directory; +}; - /** - * What query is used for this search - * (list of keywords or SKS identifier). - */ - const struct GNUNET_FS_Uri *query; - /** - * How long has this search been actively running - * (excludes times where the search was paused or - * suspended). - */ - struct GNUNET_TIME_Relative duration; +/** + * Opaque handle to an asynchronous directory scanning activity. + */ +struct GNUNET_FS_DirScanner; - /** - * What anonymity level is used for this search? - */ - uint32_t anonymity; - /** - * Additional values for specific events. - */ - union - { - /** - * These values are only valid for - * #GNUNET_FS_STATUS_SEARCH_RESULT events. - */ - struct GNUNET_FS_SearchResultEvent - { - /** - * Metadata for the search result. - */ - const struct GNUNET_FS_MetaData *meta; +/** + * Start a directory scanner. + * + * @param filename name of the directory to scan + * @param disable_extractor #GNUNET_YES to not run libextractor on files (only + * build a tree) + * @param ex if not NULL, must be a list of extra plugins for extractor + * @param cb the callback to call when there are scanning progress messages + * @param cb_cls closure for @a cb + * @return directory scanner object to be used for controlling the scanner + */ +struct GNUNET_FS_DirScanner * +GNUNET_FS_directory_scan_start (const char *filename, + int disable_extractor, + const char *ex, + GNUNET_FS_DirScannerProgressCallback cb, + void *cb_cls); - /** - * URI for the search result. - */ - const struct GNUNET_FS_Uri *uri; - /** - * Handle to the result (for starting downloads). - */ - struct GNUNET_FS_SearchResult *result; +/** + * Abort the scan. Must not be called from within the progress_callback + * function. + * + * @param ds directory scanner structure + */ +void +GNUNET_FS_directory_scan_abort (struct GNUNET_FS_DirScanner *ds); - /** - * Applicability rank (the larger, the better the result - * fits the search criteria). - */ - uint32_t applicability_rank; - } result; - /** - * These values are only valid for - * #GNUNET_FS_STATUS_SEARCH_RESUME_RESULT events. - */ - struct GNUNET_FS_SearchResumeResultEvent - { - /** - * Metadata for the search result. - */ - const struct GNUNET_FS_MetaData *meta; +/** + * Obtain the result of the scan after the scan has signalled + * completion. Must not be called prior to completion. The @a ds is + * freed as part of this call. + * + * @param ds directory scanner structure + * @return the results of the scan (a directory tree) + */ +struct GNUNET_FS_ShareTreeItem * +GNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds); - /** - * URI for the search result. - */ - const struct GNUNET_FS_Uri *uri; - /** - * Handle to the result (for starting downloads). - */ - struct GNUNET_FS_SearchResult *result; +/** + * Process a share item tree, moving frequent keywords up and + * copying frequent metadata up. + * + * @param toplevel toplevel directory in the tree, returned by the scanner + */ +void +GNUNET_FS_share_tree_trim (struct GNUNET_FS_ShareTreeItem *toplevel); - /** - * Current availability rank (negative: - * unavailable, positive: available) - */ - int32_t availability_rank; - /** - * On how many total queries is the given - * availability_rank based? - */ - uint32_t availability_certainty; +/** + * Release memory of a share item tree. + * + * @param toplevel toplevel of the tree to be freed + */ +void +GNUNET_FS_share_tree_free (struct GNUNET_FS_ShareTreeItem *toplevel); - /** - * Updated applicability rank (the larger, - * the better the result fits the search - * criteria). - */ - uint32_t applicability_rank; - } resume_result; - /** - * These values are only valid for - * #GNUNET_FS_STATUS_SEARCH_UPDATE events. - */ - struct GNUNET_FS_SearchUpdateEvent - { - /** - * Private context set for for this result - * during the "RESULT" event. - */ - void *cctx; +/* ************************* sharing API ***************** */ - /** - * Metadata for the search result. - */ - const struct GNUNET_FS_MetaData *meta; - /** - * URI for the search result. - */ - const struct GNUNET_FS_Uri *uri; +/** + * Possible status codes used in the callback for the + * various file-sharing operations. On each file (or search), + * the callback is guaranteed to be called once with "START" + * and once with STOPPED; calls with PROGRESS, ERROR or COMPLETED + * are optional and depend on the circumstances; parent operations + * will be STARTED before child-operations and STOPPED after + * their respective child-operations. START and STOP signals + * are typically generated either due to explicit client requests + * or because of suspend/resume operations. + */ +enum GNUNET_FS_Status +{ + /** + * Notification that we have started to publish a file structure. + */ + GNUNET_FS_STATUS_PUBLISH_START = 0, - /** - * Current availability rank (negative: - * unavailable, positive: available) - */ - int32_t availability_rank; + /** + * Notification that we have resumed sharing a file structure. + */ + GNUNET_FS_STATUS_PUBLISH_RESUME = 1, - /** - * On how many total queries is the given - * availability_rank based? - */ - uint32_t availability_certainty; + /** + * Notification that we have suspended sharing a file structure. + */ + GNUNET_FS_STATUS_PUBLISH_SUSPEND = 2, - /** - * Updated applicability rank (the larger, - * the better the result fits the search - * criteria). - */ - uint32_t applicability_rank; + /** + * Notification that we are making progress sharing a file structure. + */ + GNUNET_FS_STATUS_PUBLISH_PROGRESS = 3, - /** - * How long has the current probe been active? - */ - struct GNUNET_TIME_Relative current_probe_time; - } update; + /** + * Notification that an error was encountered sharing a file structure. + * The application will continue to receive resume/suspend events for + * this structure until "GNUNET_FS_publish_stop" is called. + */ + GNUNET_FS_STATUS_PUBLISH_ERROR = 4, - /** - * These values are only valid for - * #GNUNET_FS_STATUS_SEARCH_RESULT_SUSPEND events. - * These events are automatically triggered for - * each search result before the - * #GNUNET_FS_STATUS_SEARCH_SUSPEND event. This - * happens primarily to give the client a chance - * to clean up the "cctx" (if needed). - */ - struct GNUNET_FS_SearchSuspendResultEvent - { - /** - * Private context set for for this result - * during the "RESULT" event. - */ - void *cctx; + /** + * Notification that we completed sharing a file structure. + * The application will continue to receive resume/suspend events for + * this structure until "GNUNET_FS_publish_stop" is called. + */ + GNUNET_FS_STATUS_PUBLISH_COMPLETED = 5, - /** - * Metadata for the search result. - */ - const struct GNUNET_FS_MetaData *meta; + /** + * Notification that we have stopped + * the process of uploading a file structure; no + * further events will be generated for this action. + */ + GNUNET_FS_STATUS_PUBLISH_STOPPED = 6, - /** - * URI for the search result. - */ - const struct GNUNET_FS_Uri *uri; - } result_suspend; + /** + * Notification that we have started this download. + */ + GNUNET_FS_STATUS_DOWNLOAD_START = 7, - /** - * These values are only valid for - * #GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED events. - * These events are automatically triggered for - * each search result before the - * #GNUNET_FS_STATUS_SEARCH_STOPPED event. This - * happens primarily to give the client a chance - * to clean up the "cctx" (if needed). - */ - struct GNUNET_FS_SearchStoppedResultEvent - { - /** - * Private context set for for this result - * during the "RESULT" event. - */ - void *cctx; + /** + * Notification that this download is being resumed. + */ + GNUNET_FS_STATUS_DOWNLOAD_RESUME = 8, - /** - * Metadata for the search result. - */ - const struct GNUNET_FS_MetaData *meta; + /** + * Notification that this download was suspended. + */ + GNUNET_FS_STATUS_DOWNLOAD_SUSPEND = 9, - /** - * URI for the search result. - */ - const struct GNUNET_FS_Uri *uri; - } result_stopped; + /** + * Notification about progress with this download. + */ + GNUNET_FS_STATUS_DOWNLOAD_PROGRESS = 10, - /** - * These values are only valid for - * #GNUNET_FS_STATUS_SEARCH_RESUME events. - */ - struct GNUNET_GS_SearchResumeEvent - { - /** - * Error message, NULL if we have not encountered any error yet. - */ - const char *message; + /** + * Notification that this download encountered an error. + */ + GNUNET_FS_STATUS_DOWNLOAD_ERROR = 11, - /** - * Is this search currently paused? - */ - int is_paused; - } resume; + /** + * Notification that this download completed. Note that for + * directories, completion does not imply completion of all files in + * the directory. + */ + GNUNET_FS_STATUS_DOWNLOAD_COMPLETED = 12, - /** - * These values are only valid for - * #GNUNET_FS_STATUS_SEARCH_ERROR events. - */ - struct GNUNET_FS_SearchErrorEvent - { - /** - * Error message. - */ - const char *message; - } error; + /** + * Notification that this download was stopped + * (final event with respect to this action). + */ + GNUNET_FS_STATUS_DOWNLOAD_STOPPED = 13, - /** - * Values for #GNUNET_FS_STATUS_SEARCH_RESULT_NAMESPACE events. - */ - struct GNUNET_GS_SearchResultNamespaceEvent - { - /** - * Short, human-readable name of the namespace. - */ - const char *name; + /** + * Notification that this download is now actively being + * pursued (as opposed to waiting in the queue). + */ + GNUNET_FS_STATUS_DOWNLOAD_ACTIVE = 14, - /** - * Root identifier for the namespace, can be NULL. - */ - const char *root; + /** + * Notification that this download is no longer actively + * being pursued (back in the queue). + */ + GNUNET_FS_STATUS_DOWNLOAD_INACTIVE = 15, - /** - * Metadata for the namespace. - */ - const struct GNUNET_FS_MetaData *meta; + /** + * Notification that this download is no longer part of a + * recursive download or search but now a 'stand-alone' + * download (and may thus need to be moved in the GUI + * into a different category). + */ + GNUNET_FS_STATUS_DOWNLOAD_LOST_PARENT = 16, - /** - * Public key of the namespace. - */ - struct GNUNET_CRYPTO_EcdsaPublicKey pseudonym; - } ns; - } specifics; - } search; + /** + * First event generated when a client requests + * a search to begin or when a namespace result + * automatically triggers the search for updates. + */ + GNUNET_FS_STATUS_SEARCH_START = 17, - /** - * Values for all "GNUNET_FS_STATUS_UNINDEX_*" events. - */ - struct GNUNET_FS_UnindexEvent - { - /** - * Context for controlling the unindexing. - */ - struct GNUNET_FS_UnindexContext *uc; + /** + * Last event when a search is being resumed; + * note that "GNUNET_FS_SEARCH_START" will not + * be generated in this case. + */ + GNUNET_FS_STATUS_SEARCH_RESUME = 18, - /** - * Client context pointer (set the last time - * by the client for this operation; initially - * NULL on START/RESUME events). - */ - void *cctx; + /** + * Event generated for each search result + * when the respective search is resumed. + */ + GNUNET_FS_STATUS_SEARCH_RESUME_RESULT = 19, - /** - * Name of the file that is being unindexed. - */ + /** + * Last event when a search is being suspended; + * note that "GNUNET_FS_SEARCH_STOPPED" will not + * be generated in this case. + */ + GNUNET_FS_STATUS_SEARCH_SUSPEND = 20, + + /** + * This search has yielded a result. + */ + GNUNET_FS_STATUS_SEARCH_RESULT = 21, + + /** + * We have discovered a new namespace. + */ + GNUNET_FS_STATUS_SEARCH_RESULT_NAMESPACE = 22, + + /** + * We have additional data about the quality + * or availability of a search result. + */ + GNUNET_FS_STATUS_SEARCH_UPDATE = 23, + + /** + * Signals a problem with this search. + */ + GNUNET_FS_STATUS_SEARCH_ERROR = 24, + + /** + * Signals that this search was paused. + */ + GNUNET_FS_STATUS_SEARCH_PAUSED = 25, + + /** + * Signals that this search was continued (unpaused). + */ + GNUNET_FS_STATUS_SEARCH_CONTINUED = 26, + + /** + * Event generated for each search result + * when the respective search is stopped. + */ + GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED = 27, + + /** + * Event generated for each search result + * when the respective search is suspended. + */ + GNUNET_FS_STATUS_SEARCH_RESULT_SUSPEND = 28, + + /** + * Last message from a search; this signals + * that there will be no further events associated + * with this search. + */ + GNUNET_FS_STATUS_SEARCH_STOPPED = 29, + + /** + * Notification that we started to unindex a file. + */ + GNUNET_FS_STATUS_UNINDEX_START = 30, + + /** + * Notification that we resumed unindexing of a file. + */ + GNUNET_FS_STATUS_UNINDEX_RESUME = 31, + + /** + * Notification that we suspended unindexing a file. + */ + GNUNET_FS_STATUS_UNINDEX_SUSPEND = 32, + + /** + * Notification that we made progress unindexing a file. + */ + GNUNET_FS_STATUS_UNINDEX_PROGRESS = 33, + + /** + * Notification that we encountered an error unindexing + * a file. + */ + GNUNET_FS_STATUS_UNINDEX_ERROR = 34, + + /** + * Notification that the unindexing of this file + * was completed. + */ + GNUNET_FS_STATUS_UNINDEX_COMPLETED = 35, + + /** + * Notification that the unindexing of this file + * was stopped (final event for this action). + */ + GNUNET_FS_STATUS_UNINDEX_STOPPED = 36, + + /** + * Notification that we are making progress sharing a directory. + */ + GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY = 37 +}; + + +/** + * Handle for controlling an upload. + */ +struct GNUNET_FS_PublishContext; + + +/** + * Handle for controlling an unindexing operation. + */ +struct GNUNET_FS_UnindexContext; + + +/** + * Handle for controlling a search. + */ +struct GNUNET_FS_SearchContext; + + +/** + * Result from a search. Opaque handle to refer to the search + * (typically used when starting a download associated with the search + * result). + */ +struct GNUNET_FS_SearchResult; + + +/** + * Context for controlling a download. + */ +struct GNUNET_FS_DownloadContext; + + +/** + * Handle for detail information about a file that is being published. + * Specifies metadata, keywords, how to get the contents of the file + * (i.e. data-buffer in memory, filename on disk) and other options. + */ +struct GNUNET_FS_FileInformation; + + +/** + * Argument given to the progress callback with + * information about what is going on. + */ +struct GNUNET_FS_ProgressInfo +{ + /** + * Values that depend on the event type. + */ + union + { + /** + * Values for all "GNUNET_FS_STATUS_PUBLISH_*" events. + */ + struct GNUNET_FS_PublishStatusEvent + { + /** + * Context for controlling the upload. + */ + struct GNUNET_FS_PublishContext *pc; + + /** + * Information about the file that is being published. + */ + const struct GNUNET_FS_FileInformation *fi; + + /** + * Client context pointer (set the last time by the client for + * this operation; initially NULL on START/RESUME events). + */ + void *cctx; + + /** + * Client context pointer for the parent operation + * (if this is a file in a directory or a subdirectory). + */ + void *pctx; + + /** + * Name of the file being published; can be NULL. + */ const char *filename; /** - * How large is the file overall? + * How large is the file overall? For directories, + * this is only the size of the directory itself, + * not of the other files contained within the + * directory. */ uint64_t size; /** - * At what time do we expect to finish unindexing? + * At what time do we expect to finish the upload? * (will be a value in the past for completed - * unindexing operations). + * uploads). */ struct GNUNET_TIME_Relative eta; @@ -1579,6 +1615,11 @@ struct GNUNET_FS_ProgressInfo */ uint64_t completed; + /** + * What anonymity level is used for this upload? + */ + uint32_t anonymity; + /** * Additional values for specific events. */ @@ -1586,12 +1627,12 @@ struct GNUNET_FS_ProgressInfo { /** * These values are only valid for - * #GNUNET_FS_STATUS_UNINDEX_PROGRESS events. + * #GNUNET_FS_STATUS_PUBLISH_PROGRESS events. */ - struct GNUNET_FS_UnindexProgressEvent + struct GNUNET_FS_PublishProgressEvent { /** - * Data block we just unindexed. + * Data block we just published. */ const void *data; @@ -1607,1628 +1648,1609 @@ struct GNUNET_FS_ProgressInfo /** * Depth of the given block in the tree; - * 0 would be the lowest level (DBLOCKS). + * 0 would be the lowest level (DBLOCKs). */ unsigned int depth; } progress; /** * These values are only valid for - * #GNUNET_FS_STATUS_UNINDEX_RESUME events. + * #GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY events. */ - struct GNUNET_FS_UnindexResumeEvent + struct GNUNET_FS_PublishProgressDirectoryEvent { /** - * Error message, NULL if we have not encountered any error yet. + * How far are we along in the overall directory? */ - const char *message; - } resume; + uint64_t completed; + + /** + * How big do we estimate the entire directory to be? + */ + uint64_t total; + + /** + * At what time do we expect to finish the upload of the + * CONTENTS of the directory. (The directory itself will take + * extra time, indicated with the "eta" member at the + * "publish"-level of this struct.) + */ + struct GNUNET_TIME_Relative eta; + } progress_directory; /** * These values are only valid for - * #GNUNET_FS_STATUS_UNINDEX_ERROR events. + * #GNUNET_FS_STATUS_PUBLISH_RESUME events. */ - struct GNUNET_FS_UnindexErrorEvent + struct GNUNET_FS_PublishResumeEvent { /** - * Error message. + * Error message, NULL if no error was encountered so far. */ const char *message; - } error; - } specifics; - } unindex; - } value; - /** - * Specific status code (determines the event type). - */ - enum GNUNET_FS_Status status; + /** + * URI of the file (if the download had been completed) + */ + const struct GNUNET_FS_Uri *chk_uri; - /** - * File-sharing handle that generated the event. - */ - struct GNUNET_FS_Handle *fsh; -}; + /** + * SKS URI of the file (if the download had been completed) + */ + const struct GNUNET_FS_Uri *sks_uri; + } resume; + /** + * These values are only valid for + * #GNUNET_FS_STATUS_PUBLISH_COMPLETED events. + */ + struct GNUNET_FS_PublishCompletedEvent + { + /** + * CHK URI of the file. + */ + const struct GNUNET_FS_Uri *chk_uri; -/** - * Notification of FS to a client about the progress of an - * operation. Callbacks of this type will be used for uploads, - * downloads and searches. Some of the arguments depend a bit - * in their meaning on the context in which the callback is used. - * - * @param cls closure - * @param info details about the event, specifying the event type - * and various bits about the event - * @return client-context (for the next progress call - * for this operation; should be set to NULL for - * SUSPEND and STOPPED events). The value returned - * will be passed to future callbacks in the respective - * field in the `struct GNUNET_FS_ProgressInfo`. - */ -typedef void * -(*GNUNET_FS_ProgressCallback) (void *cls, - const struct GNUNET_FS_ProgressInfo *info); - + /** + * SKS URI of the file (if the download had been completed) + */ + const struct GNUNET_FS_Uri *sks_uri; + } completed; -/** - * General (global) option flags for file-sharing. - */ -enum GNUNET_FS_Flags -{ - /** - * No special flags set. - */ - GNUNET_FS_FLAGS_NONE = 0, + /** + * These values are only valid for + * #GNUNET_FS_STATUS_PUBLISH_ERROR events. + */ + struct GNUNET_FS_PublishErrorEvent + { + /** + * Error message, never NULL. + */ + const char *message; + } error; + } specifics; + } publish; - /** - * Is persistence of operations desired? - * (will create SUSPEND/RESUME events). - */ - GNUNET_FS_FLAGS_PERSISTENCE = 1, - /** - * Should we automatically trigger probes for search results - * to determine availability? - * (will create #GNUNET_FS_STATUS_SEARCH_UPDATE events). - */ - GNUNET_FS_FLAGS_DO_PROBES = 2 -}; + /** + * Values for all "GNUNET_FS_STATUS_DOWNLOAD_*" events. + */ + struct GNUNET_FS_DownloadStatusEvent + { + /** + * Context for controlling the download. + */ + struct GNUNET_FS_DownloadContext *dc; + /** + * Client context pointer (set the last time + * by the client for this operation; initially + * NULL on START/RESUME events). + */ + void *cctx; -/** - * Options specified in the VARARGs portion of GNUNET_FS_start. - */ -enum GNUNET_FS_OPTIONS -{ - /** - * Last option in the VARARG list. - */ - GNUNET_FS_OPTIONS_END = 0, + /** + * Client context pointer for the parent operation + * (if this is a file in a directory or a subdirectory). + */ + void *pctx; - /** - * Select the desired amount of parallelism (this option should be - * followed by an "unsigned int" giving the desired maximum number - * of parallel downloads). - */ - GNUNET_FS_OPTIONS_DOWNLOAD_PARALLELISM = 1, + /** + * Client context pointer for the associated search operation + * (specifically, context pointer for the specific search + * result, not the overall search); only set if this + * download was started from a search result. + */ + void *sctx; - /** - * Maximum number of requests that should be pending at a given - * point in time (individual downloads may go above this, but - * if we are above this threshold, we should not activate any - * additional downloads. - */ - GNUNET_FS_OPTIONS_REQUEST_PARALLELISM = 2 -}; + /** + * URI used for this download. + */ + const struct GNUNET_FS_Uri *uri; + /** + * Name of the file that we are downloading. + */ + const char *filename; -/** - * Settings for publishing a block (which may of course also - * apply to an entire directory or file). - */ -struct GNUNET_FS_BlockOptions -{ - /** - * At what time should the block expire? Data blocks (DBLOCKS and - * IBLOCKS) may still be used even if they are expired (however, - * they'd be removed quickly from the datastore if we are short on - * space), all other types of blocks will no longer be returned - * after they expire. - */ - struct GNUNET_TIME_Absolute expiration_time; + /** + * How large is the download overall? This + * is NOT necessarily the size from the + * URI since we may be doing a partial download. + */ + uint64_t size; - /** - * At which anonymity level should the block be shared? - * (0: no anonymity, 1: normal GAP, >1: with cover traffic). - */ - uint32_t anonymity_level; + /** + * At what time do we expect to finish the download? + * (will be a value in the past for completed + * uploads). + */ + struct GNUNET_TIME_Relative eta; - /** - * How important is it for us to store the block? If we run - * out of space, the highest-priority, non-expired blocks will - * be kept. - */ - uint32_t content_priority; + /** + * How long has this download been active? + */ + struct GNUNET_TIME_Relative duration; - /** - * How often should we try to migrate the block to other peers? - * Only used if "CONTENT_PUSHING" is set to YES, in which case we - * first push each block to other peers according to their - * replication levels. Once each block has been pushed that many - * times to other peers, blocks are chosen for migration at random. - * Naturally, there is no guarantee that the other peers will keep - * these blocks for any period of time (since they won't have any - * priority or might be too busy to even store the block in the - * first place). - */ - uint32_t replication_level; -}; + /** + * How many bytes have we completed? + */ + uint64_t completed; + /** + * What anonymity level is used for this download? + */ + uint32_t anonymity; -/** - * Handle to the file-sharing service. - */ -struct GNUNET_FS_Handle; + /** + * Is the download currently active. + */ + int is_active; + /** + * Additional values for specific events. + */ + union + { + /** + * These values are only valid for + * #GNUNET_FS_STATUS_DOWNLOAD_PROGRESS events. + */ + struct GNUNET_FS_DownloadProgressEvent + { + /** + * Data block we just obtained, can be NULL (even if + * data_len > 0) if we found the entire block 'intact' on + * disk. In this case, it is also possible for 'data_len' + * to be larger than an individual (32k) block. + */ + const void *data; -/** - * Setup a connection to the file-sharing service. - * - * @param cfg configuration to use - * @param client_name unique identifier for this client - * @param upcb function to call to notify about FS actions - * @param upcb_cls closure for @a upcb - * @param flags specific attributes for fs-operations - * @param ... list of optional options, terminated with #GNUNET_FS_OPTIONS_END - * @return NULL on error - */ -struct GNUNET_FS_Handle * -GNUNET_FS_start (const struct GNUNET_CONFIGURATION_Handle *cfg, - const char *client_name, - GNUNET_FS_ProgressCallback upcb, - void *upcb_cls, - enum GNUNET_FS_Flags flags, - ...); + /** + * At what offset in the file is "data"? + */ + uint64_t offset; + /** + * Length of the data block. + */ + uint64_t data_len; -/** - * Close our connection with the file-sharing service. - * The callback given to #GNUNET_FS_start() will no longer be - * called after this function returns. - * This function MUST NOT be called from within the - * callback itself. - * - * @param h handle that was returned from #GNUNET_FS_start() - */ -void -GNUNET_FS_stop (struct GNUNET_FS_Handle *h); + /** + * How much time passed between us asking for this block and + * actually getting it? #GNUNET_TIME_UNIT_FOREVER_REL if unknown. + */ + struct GNUNET_TIME_Relative block_download_duration; + /** + * Depth of the given block in the tree; + * 0 would be the lowest level (DBLOCKS). + */ + unsigned int depth; -/** - * Function called on entries in a `struct GNUNET_FS_FileInformation` iteration. - * - * @param cls closure - * @param fi the entry in the publish-structure - * @param length length of the file or directory - * @param meta metadata for the file or directory (can be modified) - * @param uri pointer to the keywords that will be used for this entry (can be modified) - * @param bo block options (can be modified) - * @param do_index should we index (can be modified) - * @param client_info pointer to client context set upon creation (can be modified) - * @return #GNUNET_OK to continue, #GNUNET_NO to remove - * this entry from the directory, #GNUNET_SYSERR - * to abort the iteration - */ -typedef int -(*GNUNET_FS_FileInformationProcessor) (void *cls, - struct GNUNET_FS_FileInformation *fi, - uint64_t length, - struct GNUNET_FS_MetaData *meta, - struct GNUNET_FS_Uri **uri, - struct GNUNET_FS_BlockOptions *bo, - int *do_index, - void **client_info); - - -/** - * Obtain the name under which this file information - * structure is stored on disk. Only works for top-level - * file information structures. - * - * @param s structure to get the filename for - * @return NULL on error, otherwise filename that can be used - * to read this fi-struct from disk. - */ -const char * -GNUNET_FS_file_information_get_id (struct GNUNET_FS_FileInformation *s); - - -/** - * Obtain the filename from the file information structure. - * - * @param s structure to get the filename for - * @return "filename" field of the structure (can be NULL) - */ -const char * -GNUNET_FS_file_information_get_filename (const struct - GNUNET_FS_FileInformation *s); - - -/** - * Set the filename in the file information structure. - * If filename was already set, frees it before setting the new one. - * Makes a copy of the argument. - * - * @param s structure to get the filename for - * @param filename filename to set - */ -void -GNUNET_FS_file_information_set_filename (struct GNUNET_FS_FileInformation *s, - const char *filename); - - -/** - * Create an entry for a file in a publish-structure. - * - * @param h handle to the file sharing subsystem - * @param client_info initial client-info value for this entry - * @param filename name of the file or directory to publish - * @param keywords under which keywords should this file be available - * directly; can be NULL - * @param meta metadata for the file - * @param do_index #GNUNET_YES for index, #GNUNET_NO for insertion, - * #GNUNET_SYSERR for simulation - * @param bo block options - * @return publish structure entry for the file - */ -struct GNUNET_FS_FileInformation * -GNUNET_FS_file_information_create_from_file (struct GNUNET_FS_Handle *h, - void *client_info, - const char *filename, - const struct - GNUNET_FS_Uri *keywords, - const struct - GNUNET_FS_MetaData *meta, - int do_index, - const struct - GNUNET_FS_BlockOptions *bo); - - -/** - * Create an entry for a file in a publish-structure. - * - * @param h handle to the file sharing subsystem - * @param client_info initial client-info value for this entry - * @param length length of the file - * @param data data for the file (should not be used afterwards by - * the caller; callee will "free") - * @param keywords under which keywords should this file be available - * directly; can be NULL - * @param meta metadata for the file - * @param do_index #GNUNET_YES for index, #GNUNET_NO for insertion, - * #GNUNET_SYSERR for simulation - * @param bo block options - * @return publish structure entry for the file - */ -struct GNUNET_FS_FileInformation * -GNUNET_FS_file_information_create_from_data (struct GNUNET_FS_Handle *h, - void *client_info, - uint64_t length, - void *data, - const struct - GNUNET_FS_Uri *keywords, - const struct - GNUNET_FS_MetaData *meta, - int do_index, - const struct - GNUNET_FS_BlockOptions *bo); - - -/** - * Function that provides data. - * - * @param cls closure - * @param offset offset to read from; it is possible - * that the caller might need to go backwards - * a bit at times; set to UINT64_MAX to tell - * the reader that we won't be reading for a while - * (used to close the file descriptor but NOT fully - * clean up the reader's state); in this case, - * a value of '0' for max should be ignored - * @param max maximum number of bytes that should be - * copied to buf; readers are not allowed - * to provide less data unless there is an error; - * a value of "0" will be used at the end to allow - * the reader to clean up its internal state - * @param buf where the reader should write the data - * @param emsg location for the reader to store an error message - * @return number of bytes written, usually @a max, 0 on error - */ -typedef size_t -(*GNUNET_FS_DataReader) (void *cls, - uint64_t offset, - size_t max, - void *buf, - char **emsg); - + /** + * How much respect did we offer for downloading this block? (estimate, + * because we might have the same request pending for multiple clients, + * and of course because a transmission may have failed at a lower + * layer). + */ + uint32_t respect_offered; -/** - * Create an entry for a file in a publish-structure. - * - * @param h handle to the file sharing subsystem - * @param client_info initial client-info value for this entry - * @param length length of the file - * @param reader function that can be used to obtain the data for the file - * @param reader_cls closure for @a reader - * @param keywords under which keywords should this file be available - * directly; can be NULL - * @param meta metadata for the file - * @param do_index #GNUNET_YES for index, #GNUNET_NO for insertion, - * #GNUNET_SYSERR for simulation - * @param bo block options - * @return publish structure entry for the file - */ -struct GNUNET_FS_FileInformation * -GNUNET_FS_file_information_create_from_reader (struct GNUNET_FS_Handle *h, - void *client_info, - uint64_t length, - GNUNET_FS_DataReader reader, - void *reader_cls, - const struct GNUNET_FS_Uri - *keywords, - const struct - GNUNET_FS_MetaData *meta, - int do_index, - const struct - GNUNET_FS_BlockOptions *bo); + /** + * How often did we transmit the request? (estimate, + * because we might have the same request pending for multiple clients, + * and of course because a transmission may have failed at a lower + * layer). + */ + uint32_t num_transmissions; + } progress; + /** + * These values are only valid for + * #GNUNET_FS_STATUS_DOWNLOAD_START events. + */ + struct GNUNET_FS_DownloadStartEvent + { + /** + * Known metadata for the download. + */ + const struct GNUNET_FS_MetaData *meta; + } start; -/** - * Create an entry for an empty directory in a publish-structure. - * - * @param h handle to the file sharing subsystem - * @param client_info initial client-info value for this entry - * @param keywords under which keywords should this directory be available - * directly; can be NULL - * @param meta metadata for the directory - * @param bo block options - * @param filename name of the directory; can be NULL - * @return publish structure entry for the directory , NULL on error - */ -struct GNUNET_FS_FileInformation * -GNUNET_FS_file_information_create_empty_directory (struct GNUNET_FS_Handle *h, - void *client_info, - const struct GNUNET_FS_Uri - *keywords, - const struct - GNUNET_FS_MetaData - *meta, - const struct - GNUNET_FS_BlockOptions *bo, - const char *filename); + /** + * These values are only valid for + * #GNUNET_FS_STATUS_DOWNLOAD_RESUME events. + */ + struct GNUNET_FS_DownloadResumeEvent + { + /** + * Known metadata for the download. + */ + const struct GNUNET_FS_MetaData *meta; + /** + * Error message, NULL if we have not encountered any error yet. + */ + const char *message; + } resume; -/** - * Test if a given entry represents a directory. - * - * @param ent check if this FI represents a directory - * @return #GNUNET_YES if so, #GNUNET_NO if not - */ -int -GNUNET_FS_file_information_is_directory (const struct GNUNET_FS_FileInformation - *ent); + /** + * These values are only valid for + * #GNUNET_FS_STATUS_DOWNLOAD_ERROR events. + */ + struct GNUNET_FS_DownloadErrorEvent + { + /** + * Error message. + */ + const char *message; + } error; + } specifics; + } download; + /** + * Values for all "GNUNET_FS_STATUS_SEARCH_*" events. + */ + struct GNUNET_FS_SearchStatusEvent + { + /** + * Context for controlling the search, NULL for + * searches that were not explicitly triggered + * by the client (i.e., searches for updates in + * namespaces). + */ + struct GNUNET_FS_SearchContext *sc; -/** - * Add an entry to a directory in a publish-structure. Clients - * should never modify publish structures that were passed to - * #GNUNET_FS_publish_start() already. - * - * @param dir the directory - * @param ent the entry to add; the entry must not have been - * added to any other directory at this point and - * must not include @a dir in its structure - * @return #GNUNET_OK on success, #GNUNET_SYSERR on error - */ -int -GNUNET_FS_file_information_add (struct GNUNET_FS_FileInformation *dir, - struct GNUNET_FS_FileInformation *ent); + /** + * Client context pointer (set the last time by the client for + * this operation; initially NULL on START/RESUME events). Note + * that this value can only be set on START/RESUME; returning + * non-NULL on RESULT/RESUME_RESULT will actually update the + * private context for "UPDATE" events. + */ + void *cctx; + /** + * Client parent-context pointer; NULL for top-level searches, + * refers to the client context of the associated search result + * for automatically triggered searches for updates in + * namespaces. In this case, 'presult' refers to that search + * result. + */ + void *pctx; -/** - * Inspect a file or directory in a publish-structure. Clients - * should never modify publish structures that were passed to - * #GNUNET_FS_publish_start already. When called on a directory, - * this function will FIRST call @a proc with information about - * the directory itself and then for each of the files in the - * directory (but not for files in subdirectories). When called - * on a file, @a proc will be called exactly once (with information - * about the specific file). - * - * @param dir the directory - * @param proc function to call on each entry - * @param proc_cls closure for @a proc - */ -void -GNUNET_FS_file_information_inspect (struct GNUNET_FS_FileInformation *dir, - GNUNET_FS_FileInformationProcessor proc, - void *proc_cls); + /** + * What query is used for this search + * (list of keywords or SKS identifier). + */ + const struct GNUNET_FS_Uri *query; + /** + * How long has this search been actively running + * (excludes times where the search was paused or + * suspended). + */ + struct GNUNET_TIME_Relative duration; -/** - * Destroy publish-structure. Clients should never destroy publish - * structures that were passed to #GNUNET_FS_publish_start already. - * - * @param fi structure to destroy - * @param cleaner function to call on each entry in the structure - * (useful to clean up client_info); can be NULL; return - * values are ignored - * @param cleaner_cls closure for @a cleaner - */ -void -GNUNET_FS_file_information_destroy (struct GNUNET_FS_FileInformation *fi, - GNUNET_FS_FileInformationProcessor cleaner, - void *cleaner_cls); + /** + * What anonymity level is used for this search? + */ + uint32_t anonymity; + /** + * Additional values for specific events. + */ + union + { + /** + * These values are only valid for + * #GNUNET_FS_STATUS_SEARCH_RESULT events. + */ + struct GNUNET_FS_SearchResultEvent + { + /** + * Metadata for the search result. + */ + const struct GNUNET_FS_MetaData *meta; -/** - * Options for publishing. Compatible options - * can be OR'ed together. - */ -enum GNUNET_FS_PublishOptions -{ - /** - * No options (use defaults for everything). - */ - GNUNET_FS_PUBLISH_OPTION_NONE = 0, + /** + * URI for the search result. + */ + const struct GNUNET_FS_Uri *uri; - /** - * Simulate publishing. With this option, no data will be stored - * in the datastore. Useful for computing URIs from files. - */ - GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY = 1 -}; + /** + * Handle to the result (for starting downloads). + */ + struct GNUNET_FS_SearchResult *result; + /** + * Applicability rank (the larger, the better the result + * fits the search criteria). + */ + uint32_t applicability_rank; + } result; -/** - * Publish a file or directory. - * - * @param h handle to the file sharing subsystem - * @param fi information about the file or directory structure to publish - * @param ns namespace to publish the file in, NULL for no namespace - * @param nid identifier to use for the published content in the namespace - * (can be NULL, must be NULL if namespace is NULL) - * @param nuid update-identifier that will be used for future updates - * (can be NULL, must be NULL if namespace or nid is NULL) - * @param options options for the publication - * @return context that can be used to control the publish operation - */ -struct GNUNET_FS_PublishContext * -GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h, - struct GNUNET_FS_FileInformation *fi, - const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns, - const char *nid, - const char *nuid, - enum GNUNET_FS_PublishOptions options); + /** + * These values are only valid for + * #GNUNET_FS_STATUS_SEARCH_RESUME_RESULT events. + */ + struct GNUNET_FS_SearchResumeResultEvent + { + /** + * Metadata for the search result. + */ + const struct GNUNET_FS_MetaData *meta; + /** + * URI for the search result. + */ + const struct GNUNET_FS_Uri *uri; -/** - * Stop a publication. Will abort incomplete publications (but - * not remove blocks that have already been published) or - * simply clean up the state for completed publications. - * Must NOT be called from within the event callback! - * - * @param pc context for the publication to stop - */ -void -GNUNET_FS_publish_stop (struct GNUNET_FS_PublishContext *pc); + /** + * Handle to the result (for starting downloads). + */ + struct GNUNET_FS_SearchResult *result; + /** + * Current availability rank (negative: + * unavailable, positive: available) + */ + int32_t availability_rank; -/** - * Signature of a function called as the continuation of a KBlock or - * SBlock publication. - * - * @param cls closure - * @param uri URI under which the block is now available, NULL on error - * @param emsg error message, NULL on success - */ -typedef void -(*GNUNET_FS_PublishContinuation) (void *cls, - const struct GNUNET_FS_Uri *uri, - const char *emsg); + /** + * On how many total queries is the given + * availability_rank based? + */ + uint32_t availability_certainty; + /** + * Updated applicability rank (the larger, + * the better the result fits the search + * criteria). + */ + uint32_t applicability_rank; + } resume_result; -/** - * Handle to cancel publish KSK operation. - */ -struct GNUNET_FS_PublishKskContext; + /** + * These values are only valid for + * #GNUNET_FS_STATUS_SEARCH_UPDATE events. + */ + struct GNUNET_FS_SearchUpdateEvent + { + /** + * Private context set for for this result + * during the "RESULT" event. + */ + void *cctx; + /** + * Metadata for the search result. + */ + const struct GNUNET_FS_MetaData *meta; -/** - * Publish a KBlock on GNUnet. - * - * @param h handle to the file sharing subsystem - * @param ksk_uri keywords to use - * @param meta metadata to use - * @param uri URI to refer to in the KBlock - * @param bo block options - * @param options publication options - * @param cont continuation - * @param cont_cls closure for @a cont - * @return NULL on error (@a cont will still be called) - */ -struct GNUNET_FS_PublishKskContext * -GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h, - const struct GNUNET_FS_Uri *ksk_uri, - const struct GNUNET_FS_MetaData *meta, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_BlockOptions *bo, - enum GNUNET_FS_PublishOptions options, - GNUNET_FS_PublishContinuation cont, void *cont_cls); + /** + * URI for the search result. + */ + const struct GNUNET_FS_Uri *uri; + /** + * Current availability rank (negative: + * unavailable, positive: available) + */ + int32_t availability_rank; -/** - * Abort the KSK publishing operation. - * - * @param pkc context of the operation to abort. - */ -void -GNUNET_FS_publish_ksk_cancel (struct GNUNET_FS_PublishKskContext *pkc); + /** + * On how many total queries is the given + * availability_rank based? + */ + uint32_t availability_certainty; + /** + * Updated applicability rank (the larger, + * the better the result fits the search + * criteria). + */ + uint32_t applicability_rank; -/** - * Handle to cancel publish SKS operation. - */ -struct GNUNET_FS_PublishSksContext; + /** + * How long has the current probe been active? + */ + struct GNUNET_TIME_Relative current_probe_time; + } update; + /** + * These values are only valid for + * #GNUNET_FS_STATUS_SEARCH_RESULT_SUSPEND events. + * These events are automatically triggered for + * each search result before the + * #GNUNET_FS_STATUS_SEARCH_SUSPEND event. This + * happens primarily to give the client a chance + * to clean up the "cctx" (if needed). + */ + struct GNUNET_FS_SearchSuspendResultEvent + { + /** + * Private context set for for this result + * during the "RESULT" event. + */ + void *cctx; -/** - * Publish an SBlock on GNUnet. - * - * @param h handle to the file sharing subsystem - * @param ns namespace to publish in - * @param identifier identifier to use - * @param update update identifier to use - * @param meta metadata to use - * @param uri URI to refer to in the SBlock - * @param bo block options - * @param options publication options - * @param cont continuation - * @param cont_cls closure for @a cont - * @return NULL on error (@a cont will still be called) - */ -struct GNUNET_FS_PublishSksContext * -GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h, - const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns, - const char *identifier, - const char *update, - const struct GNUNET_FS_MetaData *meta, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_BlockOptions *bo, - enum GNUNET_FS_PublishOptions options, - GNUNET_FS_PublishContinuation cont, void *cont_cls); + /** + * Metadata for the search result. + */ + const struct GNUNET_FS_MetaData *meta; + /** + * URI for the search result. + */ + const struct GNUNET_FS_Uri *uri; + } result_suspend; -/** - * Abort the SKS publishing operation. - * - * @param psc context of the operation to abort. - */ -void -GNUNET_FS_publish_sks_cancel (struct GNUNET_FS_PublishSksContext *psc); + /** + * These values are only valid for + * #GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED events. + * These events are automatically triggered for + * each search result before the + * #GNUNET_FS_STATUS_SEARCH_STOPPED event. This + * happens primarily to give the client a chance + * to clean up the "cctx" (if needed). + */ + struct GNUNET_FS_SearchStoppedResultEvent + { + /** + * Private context set for for this result + * during the "RESULT" event. + */ + void *cctx; + /** + * Metadata for the search result. + */ + const struct GNUNET_FS_MetaData *meta; -/** - * Type of a function called by #GNUNET_FS_get_indexed_files. - * - * @param cls closure - * @param filename the name of the file, NULL for end of list - * @param file_id hash of the contents of the indexed file - * @return #GNUNET_OK to continue iteration, #GNUNET_SYSERR to abort - */ -typedef int -(*GNUNET_FS_IndexedFileProcessor) (void *cls, - const char *filename, - const struct GNUNET_HashCode *file_id); + /** + * URI for the search result. + */ + const struct GNUNET_FS_Uri *uri; + } result_stopped; + + /** + * These values are only valid for + * #GNUNET_FS_STATUS_SEARCH_RESUME events. + */ + struct GNUNET_GS_SearchResumeEvent + { + /** + * Error message, NULL if we have not encountered any error yet. + */ + const char *message; + /** + * Is this search currently paused? + */ + int is_paused; + } resume; -/** - * Handle to cancel 'GNUNET_FS_get_indexed_files'. - */ -struct GNUNET_FS_GetIndexedContext; + /** + * These values are only valid for + * #GNUNET_FS_STATUS_SEARCH_ERROR events. + */ + struct GNUNET_FS_SearchErrorEvent + { + /** + * Error message. + */ + const char *message; + } error; + /** + * Values for #GNUNET_FS_STATUS_SEARCH_RESULT_NAMESPACE events. + */ + struct GNUNET_GS_SearchResultNamespaceEvent + { + /** + * Short, human-readable name of the namespace. + */ + const char *name; -/** - * Iterate over all indexed files. - * - * @param h handle to the file sharing subsystem - * @param iterator function to call on each indexed file - * @param iterator_cls closure for @a iterator - * @return NULL on error (@a iterator is not called) - */ -struct GNUNET_FS_GetIndexedContext * -GNUNET_FS_get_indexed_files (struct GNUNET_FS_Handle *h, - GNUNET_FS_IndexedFileProcessor iterator, - void *iterator_cls); + /** + * Root identifier for the namespace, can be NULL. + */ + const char *root; + /** + * Metadata for the namespace. + */ + const struct GNUNET_FS_MetaData *meta; -/** - * Cancel iteration over all indexed files. - * - * @param gic operation to cancel - */ -void -GNUNET_FS_get_indexed_files_cancel (struct GNUNET_FS_GetIndexedContext *gic); + /** + * Public key of the namespace. + */ + struct GNUNET_CRYPTO_EcdsaPublicKey pseudonym; + } ns; + } specifics; + } search; + /** + * Values for all "GNUNET_FS_STATUS_UNINDEX_*" events. + */ + struct GNUNET_FS_UnindexEvent + { + /** + * Context for controlling the unindexing. + */ + struct GNUNET_FS_UnindexContext *uc; -/** - * Unindex a file. - * - * @param h handle to the file sharing subsystem - * @param filename file to unindex - * @param cctx initial value for the client context - * @return NULL on error, otherwise handle - */ -struct GNUNET_FS_UnindexContext * -GNUNET_FS_unindex_start (struct GNUNET_FS_Handle *h, - const char *filename, - void *cctx); + /** + * Client context pointer (set the last time + * by the client for this operation; initially + * NULL on START/RESUME events). + */ + void *cctx; + /** + * Name of the file that is being unindexed. + */ + const char *filename; -/** - * Clean up after completion of an unindex operation. - * - * @param uc handle - */ -void -GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc); + /** + * How large is the file overall? + */ + uint64_t size; + /** + * At what time do we expect to finish unindexing? + * (will be a value in the past for completed + * unindexing operations). + */ + struct GNUNET_TIME_Relative eta; -/** - * Function called on updateable identifiers. - * - * @param cls closure - * @param last_id last identifier - * @param last_uri uri used for the content published under the @a last_id - * @param last_meta metadata associated with @a last_uri - * @param next_id identifier that should be used for updates - */ -typedef void (*GNUNET_FS_IdentifierProcessor) (void *cls, - const char *last_id, - const struct - GNUNET_FS_Uri *last_uri, - const struct - GNUNET_FS_MetaData * - last_meta, - const char *next_id); + /** + * How long has this upload been actively running + * (excludes times where the upload was suspended). + */ + struct GNUNET_TIME_Relative duration; + /** + * How many bytes have we completed? + */ + uint64_t completed; -/** - * List all of the identifiers in the namespace for which we could - * produce an update. Namespace updates form a graph where each node - * has a name. Each node can have any number of URI/meta-data entries - * which can each be linked to other nodes. Cycles are possible. - * - * Calling this function with @a next_id NULL will cause the library to - * call @a ip with a root for each strongly connected component of the - * graph (a root being a node from which all other nodes in the Scc - * are reachable). - * - * Calling this function with @a next_id being the name of a node will - * cause the library to call @a ip with all children of the node. Note - * that cycles within an SCC are possible (including self-loops). - * - * @param h fs handle to use - * @param ns namespace to inspect for updateable content - * @param next_id ID to look for; use NULL to look for SCC roots - * @param ip function to call on each updateable identifier - * @param ip_cls closure for @a ip - */ -void -GNUNET_FS_namespace_list_updateable (struct GNUNET_FS_Handle *h, - const struct - GNUNET_CRYPTO_EcdsaPrivateKey *ns, - const char *next_id, - GNUNET_FS_IdentifierProcessor ip, - void *ip_cls); + /** + * Additional values for specific events. + */ + union + { + /** + * These values are only valid for + * #GNUNET_FS_STATUS_UNINDEX_PROGRESS events. + */ + struct GNUNET_FS_UnindexProgressEvent + { + /** + * Data block we just unindexed. + */ + const void *data; + /** + * At what offset in the file is "data"? + */ + uint64_t offset; -/** - * Options for searching. Compatible options - * can be OR'ed together. - */ -enum GNUNET_FS_SearchOptions -{ - /** - * No options (use defaults for everything). - */ - GNUNET_FS_SEARCH_OPTION_NONE = 0, + /** + * Length of the data block. + */ + uint64_t data_len; - /** - * Only search the local host, do not search remote systems (no P2P) - */ - GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY = 1 -}; + /** + * Depth of the given block in the tree; + * 0 would be the lowest level (DBLOCKS). + */ + unsigned int depth; + } progress; + /** + * These values are only valid for + * #GNUNET_FS_STATUS_UNINDEX_RESUME events. + */ + struct GNUNET_FS_UnindexResumeEvent + { + /** + * Error message, NULL if we have not encountered any error yet. + */ + const char *message; + } resume; -/** - * Start search for content. - * - * @param h handle to the file sharing subsystem - * @param uri specifies the search parameters; can be - * a KSK URI or an SKS URI. - * @param anonymity desired level of anonymity - * @param options options for the search - * @param cctx initial value for the client context - * @return context that can be used to control the search - */ -struct GNUNET_FS_SearchContext * -GNUNET_FS_search_start (struct GNUNET_FS_Handle *h, - const struct GNUNET_FS_Uri *uri, uint32_t anonymity, - enum GNUNET_FS_SearchOptions options, void *cctx); + /** + * These values are only valid for + * #GNUNET_FS_STATUS_UNINDEX_ERROR events. + */ + struct GNUNET_FS_UnindexErrorEvent + { + /** + * Error message. + */ + const char *message; + } error; + } specifics; + } unindex; + } value; + /** + * Specific status code (determines the event type). + */ + enum GNUNET_FS_Status status; -/** - * Pause search. - * - * @param sc context for the search that should be paused - */ -void -GNUNET_FS_search_pause (struct GNUNET_FS_SearchContext *sc); + /** + * File-sharing handle that generated the event. + */ + struct GNUNET_FS_Handle *fsh; +}; /** - * Continue paused search. + * Notification of FS to a client about the progress of an + * operation. Callbacks of this type will be used for uploads, + * downloads and searches. Some of the arguments depend a bit + * in their meaning on the context in which the callback is used. * - * @param sc context for the search that should be resumed + * @param cls closure + * @param info details about the event, specifying the event type + * and various bits about the event + * @return client-context (for the next progress call + * for this operation; should be set to NULL for + * SUSPEND and STOPPED events). The value returned + * will be passed to future callbacks in the respective + * field in the `struct GNUNET_FS_ProgressInfo`. */ -void -GNUNET_FS_search_continue (struct GNUNET_FS_SearchContext *sc); +typedef void * +(*GNUNET_FS_ProgressCallback) (void *cls, + const struct GNUNET_FS_ProgressInfo *info); /** - * Stop search for content. - * - * @param sc context for the search that should be stopped + * General (global) option flags for file-sharing. */ -void -GNUNET_FS_search_stop (struct GNUNET_FS_SearchContext *sc); +enum GNUNET_FS_Flags +{ + /** + * No special flags set. + */ + GNUNET_FS_FLAGS_NONE = 0, + /** + * Is persistence of operations desired? + * (will create SUSPEND/RESUME events). + */ + GNUNET_FS_FLAGS_PERSISTENCE = 1, -/** - * Start download probes for the given search result. - * - * @param h file-sharing handle to use for the operation - * @param uri URI to probe - * @param meta meta data associated with the URI - * @param client_info client info pointer to use for associated events - * @param anonymity anonymity level to use for the probes - * @return the search result handle to access the probe activity - */ -struct GNUNET_FS_SearchResult * -GNUNET_FS_probe (struct GNUNET_FS_Handle *h, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_MetaData *meta, - void *client_info, - uint32_t anonymity); + /** + * Should we automatically trigger probes for search results + * to determine availability? + * (will create #GNUNET_FS_STATUS_SEARCH_UPDATE events). + */ + GNUNET_FS_FLAGS_DO_PROBES = 2 +}; /** - * Stop probe activity. Must ONLY be used on values - * returned from #GNUNET_FS_probe. - * - * @param sr search result to stop probing for (freed) - * @return the value of the 'client_info' pointer + * Options specified in the VARARGs portion of GNUNET_FS_start. */ -void * -GNUNET_FS_probe_stop (struct GNUNET_FS_SearchResult *sr); +enum GNUNET_FS_OPTIONS +{ + /** + * Last option in the VARARG list. + */ + GNUNET_FS_OPTIONS_END = 0, + + /** + * Select the desired amount of parallelism (this option should be + * followed by an "unsigned int" giving the desired maximum number + * of parallel downloads). + */ + GNUNET_FS_OPTIONS_DOWNLOAD_PARALLELISM = 1, + + /** + * Maximum number of requests that should be pending at a given + * point in time (individual downloads may go above this, but + * if we are above this threshold, we should not activate any + * additional downloads. + */ + GNUNET_FS_OPTIONS_REQUEST_PARALLELISM = 2 +}; /** - * Options for downloading. Compatible options - * can be OR'ed together. + * Settings for publishing a block (which may of course also + * apply to an entire directory or file). */ -enum GNUNET_FS_DownloadOptions +struct GNUNET_FS_BlockOptions { /** - * No options (use defaults for everything). - */ - GNUNET_FS_DOWNLOAD_OPTION_NONE = 0, - - /** - * Only download from the local host, do not access remote systems (no P2P) + * At what time should the block expire? Data blocks (DBLOCKS and + * IBLOCKS) may still be used even if they are expired (however, + * they'd be removed quickly from the datastore if we are short on + * space), all other types of blocks will no longer be returned + * after they expire. */ - GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY = 1, + struct GNUNET_TIME_Absolute expiration_time; /** - * Do a recursive download (that is, automatically trigger the - * download of files in directories). + * At which anonymity level should the block be shared? + * (0: no anonymity, 1: normal GAP, >1: with cover traffic). */ - GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE = 2, + uint32_t anonymity_level; /** - * Do not append temporary data to - * the target file (for the IBlocks). + * How important is it for us to store the block? If we run + * out of space, the highest-priority, non-expired blocks will + * be kept. */ - GNUNET_FS_DOWNLOAD_NO_TEMPORARIES = 4, + uint32_t content_priority; /** - * Internal option used to flag this download as a 'probe' for a - * search result. Impacts the priority with which the download is - * run and causes signalling callbacks to be done differently. - * Also, probe downloads are not serialized on suspension. Normal - * clients should not use this! + * How often should we try to migrate the block to other peers? + * Only used if "CONTENT_PUSHING" is set to YES, in which case we + * first push each block to other peers according to their + * replication levels. Once each block has been pushed that many + * times to other peers, blocks are chosen for migration at random. + * Naturally, there is no guarantee that the other peers will keep + * these blocks for any period of time (since they won't have any + * priority or might be too busy to even store the block in the + * first place). */ - GNUNET_FS_DOWNLOAD_IS_PROBE = (UINT32_C(1) << 31) + uint32_t replication_level; }; /** - * Download parts of a file. Note that this will store - * the blocks at the respective offset in the given file. Also, the - * download is still using the blocking of the underlying FS - * encoding. As a result, the download may *write* outside of the - * given boundaries (if offset and length do not match the 32k FS - * block boundaries). - * - * The given range can be used to focus a download towards a - * particular portion of the file (optimization), not to strictly - * limit the download to exactly those bytes. - * - * @param h handle to the file sharing subsystem - * @param uri the URI of the file (determines what to download); CHK or LOC URI - * @param meta known metadata for the file (can be NULL) - * @param filename where to store the file, maybe NULL (then no file is - * created on disk and data must be grabbed from the callbacks) - * @param tempname where to store temporary file data, not used if filename is non-NULL; - * can be NULL (in which case we will pick a name if needed); the temporary file - * may already exist, in which case we will try to use the data that is there and - * if it is not what is desired, will overwrite it - * @param offset at what offset should we start the download (typically 0) - * @param length how many bytes should be downloaded starting at offset - * @param anonymity anonymity level to use for the download - * @param options various download options - * @param cctx initial value for the client context for this download - * @param parent parent download to associate this download with (use NULL - * for top-level downloads; useful for manually-triggered recursive downloads) - * @return context that can be used to control this download + * Handle to the file-sharing service. */ -struct GNUNET_FS_DownloadContext * -GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_MetaData *meta, - const char *filename, const char *tempname, - uint64_t offset, uint64_t length, uint32_t anonymity, - enum GNUNET_FS_DownloadOptions options, void *cctx, - struct GNUNET_FS_DownloadContext *parent); +struct GNUNET_FS_Handle; /** - * Download parts of a file based on a search result. The download - * will be associated with the search result (and the association - * will be preserved when serializing/deserializing the state). - * If the search is stopped, the download will not be aborted but - * be 'promoted' to a stand-alone download. + * Setup a connection to the file-sharing service. * - * As with the other download function, this will store - * the blocks at the respective offset in the given file. Also, the - * download is still using the blocking of the underlying FS - * encoding. As a result, the download may *write* outside of the - * given boundaries (if offset and length do not match the 32k FS - * block boundaries). + * @param cfg configuration to use + * @param client_name unique identifier for this client + * @param upcb function to call to notify about FS actions + * @param upcb_cls closure for @a upcb + * @param flags specific attributes for fs-operations + * @param ... list of optional options, terminated with #GNUNET_FS_OPTIONS_END + * @return NULL on error + */ +struct GNUNET_FS_Handle * +GNUNET_FS_start (const struct GNUNET_CONFIGURATION_Handle *cfg, + const char *client_name, + GNUNET_FS_ProgressCallback upcb, + void *upcb_cls, + enum GNUNET_FS_Flags flags, + ...); + + +/** + * Close our connection with the file-sharing service. + * The callback given to #GNUNET_FS_start() will no longer be + * called after this function returns. + * This function MUST NOT be called from within the + * callback itself. * - * The given range can be used to focus a download towards a - * particular portion of the file (optimization), not to strictly - * limit the download to exactly those bytes. + * @param h handle that was returned from #GNUNET_FS_start() + */ +void +GNUNET_FS_stop (struct GNUNET_FS_Handle *h); + + +/** + * Function called on entries in a `struct GNUNET_FS_FileInformation` iteration. * - * @param h handle to the file sharing subsystem - * @param sr the search result to use for the download (determines uri and - * meta data and associations) - * @param filename where to store the file, maybe NULL (then no file is - * created on disk and data must be grabbed from the callbacks) - * @param tempname where to store temporary file data, not used if filename is non-NULL; - * can be NULL (in which case we will pick a name if needed); the temporary file - * may already exist, in which case we will try to use the data that is there and - * if it is not what is desired, will overwrite it - * @param offset at what offset should we start the download (typically 0) - * @param length how many bytes should be downloaded starting at offset - * @param anonymity anonymity level to use for the download - * @param options various download options - * @param cctx initial value for the client context for this download - * @return context that can be used to control this download + * @param cls closure + * @param fi the entry in the publish-structure + * @param length length of the file or directory + * @param meta metadata for the file or directory (can be modified) + * @param uri pointer to the keywords that will be used for this entry (can be modified) + * @param bo block options (can be modified) + * @param do_index should we index (can be modified) + * @param client_info pointer to client context set upon creation (can be modified) + * @return #GNUNET_OK to continue, #GNUNET_NO to remove + * this entry from the directory, #GNUNET_SYSERR + * to abort the iteration */ -struct GNUNET_FS_DownloadContext * -GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h, - struct GNUNET_FS_SearchResult *sr, - const char *filename, - const char *tempname, uint64_t offset, - uint64_t length, uint32_t anonymity, - enum GNUNET_FS_DownloadOptions options, - void *cctx); +typedef int +(*GNUNET_FS_FileInformationProcessor) (void *cls, + struct GNUNET_FS_FileInformation *fi, + uint64_t length, + struct GNUNET_FS_MetaData *meta, + struct GNUNET_FS_Uri **uri, + struct GNUNET_FS_BlockOptions *bo, + int *do_index, + void **client_info); /** - * Stop a download (aborts if download is incomplete). + * Obtain the name under which this file information + * structure is stored on disk. Only works for top-level + * file information structures. * - * @param dc handle for the download - * @param do_delete delete files of incomplete downloads + * @param s structure to get the filename for + * @return NULL on error, otherwise filename that can be used + * to read this fi-struct from disk. */ -void -GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc, int do_delete); +const char * +GNUNET_FS_file_information_get_id (struct GNUNET_FS_FileInformation *s); /** - * Suspend a download. + * Obtain the filename from the file information structure. * - * @param dc handle for the download + * @param s structure to get the filename for + * @return "filename" field of the structure (can be NULL) */ -void -GNUNET_FS_download_suspend (struct GNUNET_FS_DownloadContext *dc); +const char * +GNUNET_FS_file_information_get_filename (const struct + GNUNET_FS_FileInformation *s); /** - * Resume a suspended download. + * Set the filename in the file information structure. + * If filename was already set, frees it before setting the new one. + * Makes a copy of the argument. * - * @param dc handle for the download + * @param s structure to get the filename for + * @param filename filename to set */ void -GNUNET_FS_download_resume (struct GNUNET_FS_DownloadContext *dc); - +GNUNET_FS_file_information_set_filename (struct GNUNET_FS_FileInformation *s, + const char *filename); -/* ******************** Directory API *********************** */ +/** + * Create an entry for a file in a publish-structure. + * + * @param h handle to the file sharing subsystem + * @param client_info initial client-info value for this entry + * @param filename name of the file or directory to publish + * @param keywords under which keywords should this file be available + * directly; can be NULL + * @param meta metadata for the file + * @param do_index #GNUNET_YES for index, #GNUNET_NO for insertion, + * #GNUNET_SYSERR for simulation + * @param bo block options + * @return publish structure entry for the file + */ +struct GNUNET_FS_FileInformation * +GNUNET_FS_file_information_create_from_file (struct GNUNET_FS_Handle *h, + void *client_info, + const char *filename, + const struct + GNUNET_FS_Uri *keywords, + const struct + GNUNET_FS_MetaData *meta, + int do_index, + const struct + GNUNET_FS_BlockOptions *bo); -#define GNUNET_FS_DIRECTORY_MIME "application/gnunet-directory" -#define GNUNET_FS_DIRECTORY_MAGIC "\211GND\r\n\032\n" -#define GNUNET_FS_DIRECTORY_EXT ".gnd" /** - * Does the meta-data claim that this is a directory? - * Checks if the mime-type is that of a GNUnet directory. + * Create an entry for a file in a publish-structure. * - * @return #GNUNET_YES if it is, #GNUNET_NO if it is not, #GNUNET_SYSERR if - * we have no mime-type information (treat as #GNUNET_NO) + * @param h handle to the file sharing subsystem + * @param client_info initial client-info value for this entry + * @param length length of the file + * @param data data for the file (should not be used afterwards by + * the caller; callee will "free") + * @param keywords under which keywords should this file be available + * directly; can be NULL + * @param meta metadata for the file + * @param do_index #GNUNET_YES for index, #GNUNET_NO for insertion, + * #GNUNET_SYSERR for simulation + * @param bo block options + * @return publish structure entry for the file */ -int -GNUNET_FS_meta_data_test_for_directory (const struct GNUNET_FS_MetaData - *md); +struct GNUNET_FS_FileInformation * +GNUNET_FS_file_information_create_from_data (struct GNUNET_FS_Handle *h, + void *client_info, + uint64_t length, + void *data, + const struct + GNUNET_FS_Uri *keywords, + const struct + GNUNET_FS_MetaData *meta, + int do_index, + const struct + GNUNET_FS_BlockOptions *bo); /** - * Set the MIMETYPE information for the given - * metadata to "application/gnunet-directory". + * Function that provides data. * - * @param md metadata to add mimetype to + * @param cls closure + * @param offset offset to read from; it is possible + * that the caller might need to go backwards + * a bit at times; set to UINT64_MAX to tell + * the reader that we won't be reading for a while + * (used to close the file descriptor but NOT fully + * clean up the reader's state); in this case, + * a value of '0' for max should be ignored + * @param max maximum number of bytes that should be + * copied to buf; readers are not allowed + * to provide less data unless there is an error; + * a value of "0" will be used at the end to allow + * the reader to clean up its internal state + * @param buf where the reader should write the data + * @param emsg location for the reader to store an error message + * @return number of bytes written, usually @a max, 0 on error */ -void -GNUNET_FS_meta_data_make_directory (struct GNUNET_FS_MetaData *md); +typedef size_t +(*GNUNET_FS_DataReader) (void *cls, + uint64_t offset, + size_t max, + void *buf, + char **emsg); /** - * Suggest a filename based on given metadata. + * Create an entry for a file in a publish-structure. * - * @param md given meta data - * @return NULL if meta data is useless for suggesting a filename + * @param h handle to the file sharing subsystem + * @param client_info initial client-info value for this entry + * @param length length of the file + * @param reader function that can be used to obtain the data for the file + * @param reader_cls closure for @a reader + * @param keywords under which keywords should this file be available + * directly; can be NULL + * @param meta metadata for the file + * @param do_index #GNUNET_YES for index, #GNUNET_NO for insertion, + * #GNUNET_SYSERR for simulation + * @param bo block options + * @return publish structure entry for the file */ -char * -GNUNET_FS_meta_data_suggest_filename (const struct - GNUNET_FS_MetaData *md); +struct GNUNET_FS_FileInformation * +GNUNET_FS_file_information_create_from_reader (struct GNUNET_FS_Handle *h, + void *client_info, + uint64_t length, + GNUNET_FS_DataReader reader, + void *reader_cls, + const struct GNUNET_FS_Uri + *keywords, + const struct + GNUNET_FS_MetaData *meta, + int do_index, + const struct + GNUNET_FS_BlockOptions *bo); /** - * Function used to process entries in a directory. + * Create an entry for a file in a publish-structure. * - * @param cls closure - * @param filename name of the file in the directory - * @param uri URI of the file - * @param metadata metadata for the file; metadata for - * the directory if everything else is NULL/zero - * @param length length of the available data for the file - * (of type size_t since data must certainly fit - * into memory; if files are larger than size_t - * permits, then they will certainly not be - * embedded with the directory itself). - * @param data data available for the file (length bytes) + * @param h handle to the file sharing subsystem + * @param client_info initial client-info value for this entry + * @param tree_item share tree item for the file or directory to publish + * @param do_index #GNUNET_YES for index, #GNUNET_NO for insertion, + * #GNUNET_SYSERR for simulation + * @param bo block options + * @return publish structure entry for the file */ -typedef void (*GNUNET_FS_DirectoryEntryProcessor) (void *cls, - const char *filename, - const struct GNUNET_FS_Uri * - uri, - const struct - GNUNET_FS_MetaData * - meta, size_t length, - const void *data); +struct GNUNET_FS_FileInformation * +GNUNET_FS_file_information_create_from_share_tree_item ( + struct GNUNET_FS_Handle *h, + void *client_info, + const struct GNUNET_FS_ShareTreeItem *tree_item, + int do_index, + const struct GNUNET_FS_BlockOptions *bo); /** - * Iterate over all entries in a directory. Note that directories - * are structured such that it is possible to iterate over the - * individual blocks as well as over the entire directory. Thus - * a client can call this function on the buffer in the - * GNUNET_FS_ProgressCallback. Also, directories can optionally - * include the contents of (small) files embedded in the directory - * itself; for those files, the processor may be given the - * contents of the file directly by this function. + * Create an entry for an empty directory in a publish-structure. * - * @param size number of bytes in data - * @param data pointer to the beginning of the directory - * @param offset offset of data in the directory - * @param dep function to call on each entry - * @param dep_cls closure for @a dep - * @return #GNUNET_OK if this could be a block in a directory, - * #GNUNET_NO if this could be part of a directory (but not 100% OK) - * #GNUNET_SYSERR if 'data' does not represent a directory + * @param h handle to the file sharing subsystem + * @param client_info initial client-info value for this entry + * @param keywords under which keywords should this directory be available + * directly; can be NULL + * @param meta metadata for the directory + * @param bo block options + * @param filename name of the directory; can be NULL + * @return publish structure entry for the directory , NULL on error */ -int -GNUNET_FS_directory_list_contents (size_t size, const void *data, - uint64_t offset, - GNUNET_FS_DirectoryEntryProcessor dep, - void *dep_cls); +struct GNUNET_FS_FileInformation * +GNUNET_FS_file_information_create_empty_directory (struct GNUNET_FS_Handle *h, + void *client_info, + const struct GNUNET_FS_Uri + *keywords, + const struct + GNUNET_FS_MetaData + *meta, + const struct + GNUNET_FS_BlockOptions *bo, + const char *filename); /** - * Opaque handle to a directory builder. + * Test if a given entry represents a directory. + * + * @param ent check if this FI represents a directory + * @return #GNUNET_YES if so, #GNUNET_NO if not */ -struct GNUNET_FS_DirectoryBuilder; +int +GNUNET_FS_file_information_is_directory (const struct GNUNET_FS_FileInformation + *ent); /** - * Create a directory builder. + * Add an entry to a directory in a publish-structure. Clients + * should never modify publish structures that were passed to + * #GNUNET_FS_publish_start() already. * - * @param mdir metadata for the directory + * @param dir the directory + * @param ent the entry to add; the entry must not have been + * added to any other directory at this point and + * must not include @a dir in its structure + * @return #GNUNET_OK on success, #GNUNET_SYSERR on error */ -struct GNUNET_FS_DirectoryBuilder * -GNUNET_FS_directory_builder_create (const struct GNUNET_FS_MetaData - *mdir); +int +GNUNET_FS_file_information_add (struct GNUNET_FS_FileInformation *dir, + struct GNUNET_FS_FileInformation *ent); /** - * Add an entry to a directory. + * Inspect a file or directory in a publish-structure. Clients + * should never modify publish structures that were passed to + * #GNUNET_FS_publish_start already. When called on a directory, + * this function will FIRST call @a proc with information about + * the directory itself and then for each of the files in the + * directory (but not for files in subdirectories). When called + * on a file, @a proc will be called exactly once (with information + * about the specific file). * - * @param bld directory to extend - * @param uri uri of the entry (must not be a KSK) - * @param md metadata of the entry - * @param data raw data of the entry, can be NULL, otherwise - * data must point to exactly the number of bytes specified - * by the uri + * @param dir the directory + * @param proc function to call on each entry + * @param proc_cls closure for @a proc */ void -GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld, - const struct GNUNET_FS_Uri *uri, - const struct GNUNET_FS_MetaData *md, - const void *data); +GNUNET_FS_file_information_inspect (struct GNUNET_FS_FileInformation *dir, + GNUNET_FS_FileInformationProcessor proc, + void *proc_cls); /** - * Finish building the directory. Frees the - * builder context and returns the directory - * in-memory. + * Destroy publish-structure. Clients should never destroy publish + * structures that were passed to #GNUNET_FS_publish_start already. * - * @param bld directory to finish - * @param rsize set to the number of bytes needed - * @param rdata set to the encoded directory - * @return #GNUNET_OK on success + * @param fi structure to destroy + * @param cleaner function to call on each entry in the structure + * (useful to clean up client_info); can be NULL; return + * values are ignored + * @param cleaner_cls closure for @a cleaner */ -int -GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld, - size_t *rsize, void **rdata); - +void +GNUNET_FS_file_information_destroy (struct GNUNET_FS_FileInformation *fi, + GNUNET_FS_FileInformationProcessor cleaner, + void *cleaner_cls); -/* ******************** DirScanner API *********************** */ /** - * Progress reasons of the directory scanner. + * Options for publishing. Compatible options + * can be OR'ed together. */ -enum GNUNET_FS_DirScannerProgressUpdateReason +enum GNUNET_FS_PublishOptions { /** - * We've started processing a file or directory. - */ - GNUNET_FS_DIRSCANNER_FILE_START = 0, - - /** - * We're having trouble accessing a file (soft-error); it will - * be ignored. - */ - GNUNET_FS_DIRSCANNER_FILE_IGNORED, - - /** - * We've found all files (in the pre-pass). - */ - GNUNET_FS_DIRSCANNER_ALL_COUNTED, - - /** - * We've finished extracting meta data from a file. - */ - GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED, - - /** - * Last call to the progress function: we have finished scanning - * the directory. + * No options (use defaults for everything). */ - GNUNET_FS_DIRSCANNER_FINISHED, + GNUNET_FS_PUBLISH_OPTION_NONE = 0, /** - * There was an internal error. Application should abort the scan. + * Simulate publishing. With this option, no data will be stored + * in the datastore. Useful for computing URIs from files. */ - GNUNET_FS_DIRSCANNER_INTERNAL_ERROR + GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY = 1 }; /** - * Function called over time as the directory scanner makes - * progress on the job at hand. + * Publish a file or directory. * - * @param cls closure - * @param filename which file we are making progress on - * @param is_directory #GNUNET_YES if this is a directory, - * #GNUNET_NO if this is a file - * #GNUNET_SYSERR if it is neither (or unknown) - * @param reason kind of progress we are making + * @param h handle to the file sharing subsystem + * @param fi information about the file or directory structure to publish + * @param ns namespace to publish the file in, NULL for no namespace + * @param nid identifier to use for the published content in the namespace + * (can be NULL, must be NULL if namespace is NULL) + * @param nuid update-identifier that will be used for future updates + * (can be NULL, must be NULL if namespace or nid is NULL) + * @param options options for the publication + * @return context that can be used to control the publish operation */ -typedef void (*GNUNET_FS_DirScannerProgressCallback) (void *cls, - const char *filename, - int is_directory, - enum - GNUNET_FS_DirScannerProgressUpdateReason - reason); +struct GNUNET_FS_PublishContext * +GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h, + struct GNUNET_FS_FileInformation *fi, + const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns, + const char *nid, + const char *nuid, + enum GNUNET_FS_PublishOptions options); /** - * A node of a directory tree (produced by dirscanner) + * Stop a publication. Will abort incomplete publications (but + * not remove blocks that have already been published) or + * simply clean up the state for completed publications. + * Must NOT be called from within the event callback! + * + * @param pc context for the publication to stop */ -struct GNUNET_FS_ShareTreeItem -{ - /** - * This is a doubly-linked list - */ - struct GNUNET_FS_ShareTreeItem *prev; - - /** - * This is a doubly-linked list - */ - struct GNUNET_FS_ShareTreeItem *next; - - /** - * This is a doubly-linked tree - * NULL for top-level entries. - */ - struct GNUNET_FS_ShareTreeItem *parent; - - /** - * This is a doubly-linked tree - * NULL for files and empty directories - */ - struct GNUNET_FS_ShareTreeItem *children_head; - - /** - * This is a doubly-linked tree - * NULL for files and empty directories - */ - struct GNUNET_FS_ShareTreeItem *children_tail; - - /** - * Metadata for this file or directory - */ - struct GNUNET_FS_MetaData *meta; - - /** - * Keywords for this file or directory (derived from metadata). - */ - struct GNUNET_FS_Uri *ksk_uri; - - /** - * Name of the file/directory - */ - char *filename; +void +GNUNET_FS_publish_stop (struct GNUNET_FS_PublishContext *pc); - /** - * Base name of the file/directory. - */ - char *short_filename; - /** - * #GNUNET_YES if this is a directory - */ - int is_directory; -}; +/** + * Signature of a function called as the continuation of a KBlock or + * SBlock publication. + * + * @param cls closure + * @param uri URI under which the block is now available, NULL on error + * @param emsg error message, NULL on success + */ +typedef void +(*GNUNET_FS_PublishContinuation) (void *cls, + const struct GNUNET_FS_Uri *uri, + const char *emsg); /** - * Opaque handle to an asynchronous directory scanning activity. + * Handle to cancel publish KSK operation. */ -struct GNUNET_FS_DirScanner; +struct GNUNET_FS_PublishKskContext; /** - * Start a directory scanner. + * Publish a KBlock on GNUnet. * - * @param filename name of the directory to scan - * @param disable_extractor #GNUNET_YES to not run libextractor on files (only - * build a tree) - * @param ex if not NULL, must be a list of extra plugins for extractor - * @param cb the callback to call when there are scanning progress messages - * @param cb_cls closure for @a cb - * @return directory scanner object to be used for controlling the scanner + * @param h handle to the file sharing subsystem + * @param ksk_uri keywords to use + * @param meta metadata to use + * @param uri URI to refer to in the KBlock + * @param bo block options + * @param options publication options + * @param cont continuation + * @param cont_cls closure for @a cont + * @return NULL on error (@a cont will still be called) */ -struct GNUNET_FS_DirScanner * -GNUNET_FS_directory_scan_start (const char *filename, - int disable_extractor, - const char *ex, - GNUNET_FS_DirScannerProgressCallback cb, - void *cb_cls); +struct GNUNET_FS_PublishKskContext * +GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h, + const struct GNUNET_FS_Uri *ksk_uri, + const struct GNUNET_FS_MetaData *meta, + const struct GNUNET_FS_Uri *uri, + const struct GNUNET_FS_BlockOptions *bo, + enum GNUNET_FS_PublishOptions options, + GNUNET_FS_PublishContinuation cont, void *cont_cls); /** - * Abort the scan. Must not be called from within the progress_callback - * function. + * Abort the KSK publishing operation. * - * @param ds directory scanner structure + * @param pkc context of the operation to abort. */ void -GNUNET_FS_directory_scan_abort (struct GNUNET_FS_DirScanner *ds); +GNUNET_FS_publish_ksk_cancel (struct GNUNET_FS_PublishKskContext *pkc); /** - * Obtain the result of the scan after the scan has signalled - * completion. Must not be called prior to completion. The @a ds is - * freed as part of this call. - * - * @param ds directory scanner structure - * @return the results of the scan (a directory tree) + * Handle to cancel publish SKS operation. */ -struct GNUNET_FS_ShareTreeItem * -GNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds); +struct GNUNET_FS_PublishSksContext; /** - * Process a share item tree, moving frequent keywords up and - * copying frequent metadata up. + * Publish an SBlock on GNUnet. * - * @param toplevel toplevel directory in the tree, returned by the scanner + * @param h handle to the file sharing subsystem + * @param ns namespace to publish in + * @param identifier identifier to use + * @param update update identifier to use + * @param meta metadata to use + * @param uri URI to refer to in the SBlock + * @param bo block options + * @param options publication options + * @param cont continuation + * @param cont_cls closure for @a cont + * @return NULL on error (@a cont will still be called) */ -void -GNUNET_FS_share_tree_trim (struct GNUNET_FS_ShareTreeItem *toplevel); +struct GNUNET_FS_PublishSksContext * +GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h, + const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns, + const char *identifier, + const char *update, + const struct GNUNET_FS_MetaData *meta, + const struct GNUNET_FS_Uri *uri, + const struct GNUNET_FS_BlockOptions *bo, + enum GNUNET_FS_PublishOptions options, + GNUNET_FS_PublishContinuation cont, void *cont_cls); /** - * Release memory of a share item tree. + * Abort the SKS publishing operation. * - * @param toplevel toplevel of the tree to be freed + * @param psc context of the operation to abort. */ void -GNUNET_FS_share_tree_free (struct GNUNET_FS_ShareTreeItem *toplevel); +GNUNET_FS_publish_sks_cancel (struct GNUNET_FS_PublishSksContext *psc); + /** - * Meta data processing - * @defgroup metadata Metadata - * GNU libextractor key-value pairs + * Type of a function called by #GNUNET_FS_get_indexed_files. + * + * @param cls closure + * @param filename the name of the file, NULL for end of list + * @param file_id hash of the contents of the indexed file + * @return #GNUNET_OK to continue iteration, #GNUNET_SYSERR to abort */ +typedef int +(*GNUNET_FS_IndexedFileProcessor) (void *cls, + const char *filename, + const struct GNUNET_HashCode *file_id); -/* ****************** metadata ******************* */ +/** + * Handle to cancel 'GNUNET_FS_get_indexed_files'. + */ +struct GNUNET_FS_GetIndexedContext; /** - * @ingroup metadata - * Create a fresh meta data container. + * Iterate over all indexed files. * - * @return empty meta-data container + * @param h handle to the file sharing subsystem + * @param iterator function to call on each indexed file + * @param iterator_cls closure for @a iterator + * @return NULL on error (@a iterator is not called) */ -struct GNUNET_FS_MetaData * -GNUNET_FS_meta_data_create (void); +struct GNUNET_FS_GetIndexedContext * +GNUNET_FS_get_indexed_files (struct GNUNET_FS_Handle *h, + GNUNET_FS_IndexedFileProcessor iterator, + void *iterator_cls); /** - * @ingroup metadata - * Duplicate a MetaData token. + * Cancel iteration over all indexed files. * - * @param md what to duplicate - * @return duplicate meta-data container + * @param gic operation to cancel */ -struct GNUNET_FS_MetaData * -GNUNET_FS_meta_data_duplicate ( - const struct GNUNET_FS_MetaData *md); +void +GNUNET_FS_get_indexed_files_cancel (struct GNUNET_FS_GetIndexedContext *gic); /** - * @ingroup metadata - * Free meta data. + * Unindex a file. * - * @param md what to free + * @param h handle to the file sharing subsystem + * @param filename file to unindex + * @param cctx initial value for the client context + * @return NULL on error, otherwise handle */ -void -GNUNET_FS_meta_data_destroy (struct GNUNET_FS_MetaData *md); +struct GNUNET_FS_UnindexContext * +GNUNET_FS_unindex_start (struct GNUNET_FS_Handle *h, + const char *filename, + void *cctx); /** - * @ingroup metadata - * Test if two MDs are equal. We consider them equal if - * the meta types, formats and content match (we do not - * include the mime types and plugins names in this - * consideration). + * Clean up after completion of an unindex operation. * - * @param md1 first value to check - * @param md2 other value to check - * @return #GNUNET_YES if they are equal + * @param uc handle */ -int -GNUNET_FS_meta_data_test_equal ( - const struct GNUNET_FS_MetaData *md1, - const struct GNUNET_FS_MetaData *md2); +void +GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc); /** - * @ingroup metadata - * Extend metadata. + * Function called on updateable identifiers. * - * @param md metadata to extend - * @param plugin_name name of the plugin that produced this value; - * special values can be used (e.g. '<zlib>' for zlib being - * used in the main libextractor library and yielding - * meta data). - * @param type libextractor-type describing the meta data - * @param format basic format information about data - * @param data_mime_type mime-type of data (not of the original file); - * can be NULL (if mime-type is not known) - * @param data actual meta-data found - * @param data_size number of bytes in data - * @return #GNUNET_OK on success, #GNUNET_SYSERR if this entry already exists - * data_mime_type and plugin_name are not considered for "exists" checks + * @param cls closure + * @param last_id last identifier + * @param last_uri uri used for the content published under the @a last_id + * @param last_meta metadata associated with @a last_uri + * @param next_id identifier that should be used for updates */ -int -GNUNET_FS_meta_data_insert (struct GNUNET_FS_MetaData *md, - const char *plugin_name, - enum EXTRACTOR_MetaType type, - enum EXTRACTOR_MetaFormat format, - const char *data_mime_type, - const char *data, - size_t data_size); +typedef void (*GNUNET_FS_IdentifierProcessor) (void *cls, + const char *last_id, + const struct + GNUNET_FS_Uri *last_uri, + const struct + GNUNET_FS_MetaData * + last_meta, + const char *next_id); /** - * @ingroup metadata - * Extend metadata. Merges the meta data from the second argument - * into the first, discarding duplicate key-value pairs. + * List all of the identifiers in the namespace for which we could + * produce an update. Namespace updates form a graph where each node + * has a name. Each node can have any number of URI/meta-data entries + * which can each be linked to other nodes. Cycles are possible. * - * @param md metadata to extend - * @param in metadata to merge + * Calling this function with @a next_id NULL will cause the library to + * call @a ip with a root for each strongly connected component of the + * graph (a root being a node from which all other nodes in the Scc + * are reachable). + * + * Calling this function with @a next_id being the name of a node will + * cause the library to call @a ip with all children of the node. Note + * that cycles within an SCC are possible (including self-loops). + * + * @param h fs handle to use + * @param ns namespace to inspect for updateable content + * @param next_id ID to look for; use NULL to look for SCC roots + * @param ip function to call on each updateable identifier + * @param ip_cls closure for @a ip */ void -GNUNET_FS_meta_data_merge (struct GNUNET_FS_MetaData *md, - const struct GNUNET_FS_MetaData *in); +GNUNET_FS_namespace_list_updateable (struct GNUNET_FS_Handle *h, + const struct + GNUNET_CRYPTO_EcdsaPrivateKey *ns, + const char *next_id, + GNUNET_FS_IdentifierProcessor ip, + void *ip_cls); /** - * @ingroup metadata - * Remove an item. - * - * @param md metadata to manipulate - * @param type type of the item to remove - * @param data specific value to remove, NULL to remove all - * entries of the given type - * @param data_size number of bytes in data - * @return #GNUNET_OK on success, #GNUNET_SYSERR if the item does not exist in md + * Options for searching. Compatible options + * can be OR'ed together. */ -int -GNUNET_FS_meta_data_delete (struct GNUNET_FS_MetaData *md, - enum EXTRACTOR_MetaType type, - const char *data, - size_t data_size); +enum GNUNET_FS_SearchOptions +{ + /** + * No options (use defaults for everything). + */ + GNUNET_FS_SEARCH_OPTION_NONE = 0, + + /** + * Only search the local host, do not search remote systems (no P2P) + */ + GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY = 1 +}; /** - * @ingroup metadata - * Remove all items in the container. + * Start search for content. * - * @param md metadata to manipulate + * @param h handle to the file sharing subsystem + * @param uri specifies the search parameters; can be + * a KSK URI or an SKS URI. + * @param anonymity desired level of anonymity + * @param options options for the search + * @param cctx initial value for the client context + * @return context that can be used to control the search */ -void -GNUNET_FS_meta_data_clear (struct GNUNET_FS_MetaData *md); +struct GNUNET_FS_SearchContext * +GNUNET_FS_search_start (struct GNUNET_FS_Handle *h, + const struct GNUNET_FS_Uri *uri, uint32_t anonymity, + enum GNUNET_FS_SearchOptions options, void *cctx); /** - * @ingroup metadata - * Add the current time as the publication date - * to the meta-data. + * Pause search. * - * @param md metadata to modify + * @param sc context for the search that should be paused */ void -GNUNET_FS_meta_data_add_publication_date ( - struct GNUNET_FS_MetaData *md); +GNUNET_FS_search_pause (struct GNUNET_FS_SearchContext *sc); /** - * @ingroup metadata - * Iterate over MD entries. + * Continue paused search. * - * @param md metadata to inspect - * @param iter function to call on each entry, return 0 to continue to iterate - * and 1 to abort iteration in this function (GNU libextractor API!) - * @param iter_cls closure for @a iter - * @return number of entries + * @param sc context for the search that should be resumed */ -int -GNUNET_FS_meta_data_iterate (const struct GNUNET_FS_MetaData *md, - EXTRACTOR_MetaDataProcessor iter, - void *iter_cls); - - -/** - * @ingroup metadata - * Get the first MD entry of the given type. Caller - * is responsible for freeing the return value. - * Also, only meta data items that are strings (0-terminated) - * are returned by this function. +void +GNUNET_FS_search_continue (struct GNUNET_FS_SearchContext *sc); + + +/** + * Stop search for content. * - * @param md metadata to inspect - * @param type type to look for - * @return NULL if no entry was found + * @param sc context for the search that should be stopped */ -char * -GNUNET_FS_meta_data_get_by_type ( - const struct GNUNET_FS_MetaData *md, - enum EXTRACTOR_MetaType type); +void +GNUNET_FS_search_stop (struct GNUNET_FS_SearchContext *sc); /** - * @ingroup metadata - * Get the first matching MD entry of the given types. Caller is - * responsible for freeing the return value. Also, only meta data - * items that are strings (0-terminated) are returned by this - * function. + * Start download probes for the given search result. * - * @param md metadata to inspect - * @param ... -1-terminated list of types - * @return NULL if we do not have any such entry, - * otherwise client is responsible for freeing the value! + * @param h file-sharing handle to use for the operation + * @param uri URI to probe + * @param meta meta data associated with the URI + * @param client_info client info pointer to use for associated events + * @param anonymity anonymity level to use for the probes + * @return the search result handle to access the probe activity */ -char * -GNUNET_FS_meta_data_get_first_by_types ( - const struct GNUNET_FS_MetaData *md, - ...); +struct GNUNET_FS_SearchResult * +GNUNET_FS_probe (struct GNUNET_FS_Handle *h, + const struct GNUNET_FS_Uri *uri, + const struct GNUNET_FS_MetaData *meta, + void *client_info, + uint32_t anonymity); + /** - * @ingroup metadata - * Get a thumbnail from the meta-data (if present). Only matches meta - * data with mime type "image" and binary format. + * Stop probe activity. Must ONLY be used on values + * returned from #GNUNET_FS_probe. * - * @param md metadata to inspect - * @param thumb will be set to the thumbnail data. Must be - * freed by the caller! - * @return number of bytes in thumbnail, 0 if not available + * @param sr search result to stop probing for (freed) + * @return the value of the 'client_info' pointer */ -size_t -GNUNET_FS_meta_data_get_thumbnail ( - const struct GNUNET_FS_MetaData *md, - unsigned char **thumb); +void * +GNUNET_FS_probe_stop (struct GNUNET_FS_SearchResult *sr); /** - * @ingroup metadata - * Options for metadata serialization. + * Options for downloading. Compatible options + * can be OR'ed together. */ -enum GNUNET_FS_MetaDataSerializationOptions +enum GNUNET_FS_DownloadOptions { /** - * @ingroup metadata - * Serialize all of the data. + * No options (use defaults for everything). */ - GNUNET_FS_META_DATA_SERIALIZE_FULL = 0, + GNUNET_FS_DOWNLOAD_OPTION_NONE = 0, /** - * @ingroup metadata - * If not enough space is available, it is acceptable - * to only serialize some of the metadata. + * Only download from the local host, do not access remote systems (no P2P) */ - GNUNET_FS_META_DATA_SERIALIZE_PART = 1, + GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY = 1, /** - * @ingroup metadata - * Speed is of the essence, do not allow compression. + * Do a recursive download (that is, automatically trigger the + * download of files in directories). */ - GNUNET_FS_META_DATA_SERIALIZE_NO_COMPRESS = 2 -}; + GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE = 2, + /** + * Do not append temporary data to + * the target file (for the IBlocks). + */ + GNUNET_FS_DOWNLOAD_NO_TEMPORARIES = 4, -/** - * @ingroup metadata - * Serialize meta-data to target. - * - * @param md metadata to serialize - * @param target where to write the serialized metadata; - * *target can be NULL, in which case memory is allocated - * @param max maximum number of bytes available - * @param opt is it ok to just write SOME of the - * meta-data to match the size constraint, - * possibly discarding some data? - * @return number of bytes written on success, - * -1 on error (typically: not enough - * space) - */ -ssize_t -GNUNET_FS_meta_data_serialize ( - const struct GNUNET_FS_MetaData *md, - char **target, - size_t max, - enum GNUNET_FS_MetaDataSerializationOptions opt); + /** + * Internal option used to flag this download as a 'probe' for a + * search result. Impacts the priority with which the download is + * run and causes signalling callbacks to be done differently. + * Also, probe downloads are not serialized on suspension. Normal + * clients should not use this! + */ + GNUNET_FS_DOWNLOAD_IS_PROBE = (UINT32_C(1) << 31) +}; /** - * @ingroup metadata - * Get the size of the full meta-data in serialized form. + * Download parts of a file. Note that this will store + * the blocks at the respective offset in the given file. Also, the + * download is still using the blocking of the underlying FS + * encoding. As a result, the download may *write* outside of the + * given boundaries (if offset and length do not match the 32k FS + * block boundaries). * - * @param md metadata to inspect - * @return number of bytes needed for serialization, -1 on error + * The given range can be used to focus a download towards a + * particular portion of the file (optimization), not to strictly + * limit the download to exactly those bytes. + * + * @param h handle to the file sharing subsystem + * @param uri the URI of the file (determines what to download); CHK or LOC URI + * @param meta known metadata for the file (can be NULL) + * @param filename where to store the file, maybe NULL (then no file is + * created on disk and data must be grabbed from the callbacks) + * @param tempname where to store temporary file data, not used if filename is non-NULL; + * can be NULL (in which case we will pick a name if needed); the temporary file + * may already exist, in which case we will try to use the data that is there and + * if it is not what is desired, will overwrite it + * @param offset at what offset should we start the download (typically 0) + * @param length how many bytes should be downloaded starting at offset + * @param anonymity anonymity level to use for the download + * @param options various download options + * @param cctx initial value for the client context for this download + * @param parent parent download to associate this download with (use NULL + * for top-level downloads; useful for manually-triggered recursive downloads) + * @return context that can be used to control this download */ -ssize_t -GNUNET_FS_meta_data_get_serialized_size ( - const struct GNUNET_FS_MetaData *md); +struct GNUNET_FS_DownloadContext * +GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, + const struct GNUNET_FS_Uri *uri, + const struct GNUNET_FS_MetaData *meta, + const char *filename, const char *tempname, + uint64_t offset, uint64_t length, uint32_t anonymity, + enum GNUNET_FS_DownloadOptions options, void *cctx, + struct GNUNET_FS_DownloadContext *parent); /** - * @ingroup metadata - * Deserialize meta-data. Initializes md. + * Download parts of a file based on a search result. The download + * will be associated with the search result (and the association + * will be preserved when serializing/deserializing the state). + * If the search is stopped, the download will not be aborted but + * be 'promoted' to a stand-alone download. * - * @param input serialized meta-data. - * @param size number of bytes available - * @return MD on success, NULL on error (e.g. - * bad format) - */ -struct GNUNET_FS_MetaData * -GNUNET_FS_meta_data_deserialize (const char *input, size_t size); - -/** - * Write a metadata container. + * As with the other download function, this will store + * the blocks at the respective offset in the given file. Also, the + * download is still using the blocking of the underlying FS + * encoding. As a result, the download may *write* outside of the + * given boundaries (if offset and length do not match the 32k FS + * block boundaries). * - * @param h the IO handle to write to - * @param what what is being written (for error message creation) - * @param m metadata to write - * @return #GNUNET_OK on success, #GNUNET_SYSERR on error + * The given range can be used to focus a download towards a + * particular portion of the file (optimization), not to strictly + * limit the download to exactly those bytes. + * + * @param h handle to the file sharing subsystem + * @param sr the search result to use for the download (determines uri and + * meta data and associations) + * @param filename where to store the file, maybe NULL (then no file is + * created on disk and data must be grabbed from the callbacks) + * @param tempname where to store temporary file data, not used if filename is non-NULL; + * can be NULL (in which case we will pick a name if needed); the temporary file + * may already exist, in which case we will try to use the data that is there and + * if it is not what is desired, will overwrite it + * @param offset at what offset should we start the download (typically 0) + * @param length how many bytes should be downloaded starting at offset + * @param anonymity anonymity level to use for the download + * @param options various download options + * @param cctx initial value for the client context for this download + * @return context that can be used to control this download */ -enum GNUNET_GenericReturnValue -GNUNET_FS_write_meta_data (struct GNUNET_BIO_WriteHandle *h, - const char *what, - const struct GNUNET_FS_MetaData *m); +struct GNUNET_FS_DownloadContext * +GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h, + struct GNUNET_FS_SearchResult *sr, + const char *filename, + const char *tempname, uint64_t offset, + uint64_t length, uint32_t anonymity, + enum GNUNET_FS_DownloadOptions options, + void *cctx); + /** - * Create the specification to read a metadata container. + * Stop a download (aborts if download is incomplete). * - * @param what describes what is being read (for error message creation) - * @param result the buffer to store a pointer to the (allocated) metadata - * @return the read spec + * @param dc handle for the download + * @param do_delete delete files of incomplete downloads */ -struct GNUNET_BIO_ReadSpec -GNUNET_FS_read_spec_meta_data (const char *what, - struct GNUNET_FS_MetaData **result); - +void +GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc, int do_delete); /** - * Create the specification to write a metadata container. + * Suspend a download. * - * @param what what is being written (for error message creation) - * @param m metadata to write - * @return the write spec + * @param dc handle for the download */ -struct GNUNET_BIO_WriteSpec -GNUNET_FS_write_spec_meta_data (const char *what, - const struct GNUNET_FS_MetaData *m); +void +GNUNET_FS_download_suspend (struct GNUNET_FS_DownloadContext *dc); + /** - * Read a metadata container. + * Resume a suspended download. * - * @param h handle to an open file - * @param what describes what is being read (for error message creation) - * @param result the buffer to store a pointer to the (allocated) metadata - * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure + * @param dc handle for the download */ -enum GNUNET_GenericReturnValue -GNUNET_FS_read_meta_data (struct GNUNET_BIO_ReadHandle *h, - const char *what, - struct GNUNET_FS_MetaData **result); +void +GNUNET_FS_download_resume (struct GNUNET_FS_DownloadContext *dc); #if 0 /* keep Emacsens' auto-indent happy */ diff --git a/src/service/fs/fs_file_information.c b/src/service/fs/fs_file_information.c index f23b9da2a..c776570ff 100644 --- a/src/service/fs/fs_file_information.c +++ b/src/service/fs/fs_file_information.c @@ -208,6 +208,30 @@ GNUNET_FS_file_information_create_from_reader ( } +struct GNUNET_FS_FileInformation * +GNUNET_FS_file_information_create_from_share_tree_item ( + struct GNUNET_FS_Handle *h, + void *client_info, + const struct GNUNET_FS_ShareTreeItem *tree_item, + int do_index, + const struct GNUNET_FS_BlockOptions *bo) +{ + if (NULL == tree_item) + { + GNUNET_break (0); + return NULL; + } + + return GNUNET_FS_file_information_create_from_file (h, + client_info, + tree_item->filename, + tree_item->ksk_uri, + tree_item->meta, + do_index, + bo); +} + + /** * Test if a given entry represents a directory. * diff --git a/src/service/fs/test_fs_file_information.c b/src/service/fs/test_fs_file_information.c index 64ba3627b..bc7e89106 100644 --- a/src/service/fs/test_fs_file_information.c +++ b/src/service/fs/test_fs_file_information.c @@ -47,6 +47,10 @@ #define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) +struct GNUNET_FS_Handle *fs = NULL; +static struct GNUNET_FS_DirScanner *ds = NULL; + + static int mycleaner (void *cls, struct GNUNET_FS_FileInformation *fi, uint64_t length, struct GNUNET_FS_MetaData *meta, struct GNUNET_FS_Uri **uri, @@ -56,6 +60,36 @@ mycleaner (void *cls, struct GNUNET_FS_FileInformation *fi, uint64_t length, } +static void +directory_scan_cb (void *cls, + const char *filename, + int is_directory, + enum GNUNET_FS_DirScannerProgressUpdateReason reason) +{ + struct GNUNET_FS_ShareTreeItem *directory_scan_result = NULL; + + if (GNUNET_FS_DIRSCANNER_FINISHED != reason) + { + return; + } + + directory_scan_result = GNUNET_FS_directory_scan_get_result (ds); + ds = NULL; + GNUNET_FS_share_tree_trim (directory_scan_result); + + struct GNUNET_FS_FileInformation *fi = NULL; + fi = GNUNET_FS_file_information_create_from_share_tree_item ( + fs, + "file_information-context3", + directory_scan_result, + GNUNET_YES, + (struct GNUNET_FS_BlockOptions *)cls); + + GNUNET_assert (NULL != fi); + GNUNET_FS_file_information_destroy (fi, NULL, NULL); +} + + static void run (void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg) @@ -66,13 +100,13 @@ run (void *cls, char *const *args, const char *cfgfile, }; char *fn1; char *fn2; + char *fn3; char *buf; struct GNUNET_FS_MetaData *meta; struct GNUNET_FS_Uri *kuri; struct GNUNET_FS_FileInformation *fi1; struct GNUNET_FS_FileInformation *fi2; struct GNUNET_FS_FileInformation *fidir; - struct GNUNET_FS_Handle *fs; size_t i; struct GNUNET_FS_BlockOptions bo; @@ -100,6 +134,17 @@ run (void *cls, char *const *args, const char *cfgfile, | GNUNET_DISK_PERM_USER_WRITE)); GNUNET_free (buf); + fn3 = GNUNET_DISK_mktemp ("gnunet-file_information-test-dst"); + buf = GNUNET_malloc (FILESIZE); + for (i = 0; i < FILESIZE; i++) + buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); + (void) GNUNET_DISK_directory_remove (fn3); + GNUNET_assert (GNUNET_OK == + GNUNET_DISK_fn_write (fn3, buf, FILESIZE, + GNUNET_DISK_PERM_USER_READ + | GNUNET_DISK_PERM_USER_WRITE)); + GNUNET_free (buf); + meta = GNUNET_FS_meta_data_create (); kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); bo.content_priority = 42; @@ -117,6 +162,13 @@ run (void *cls, char *const *args, const char *cfgfile, "file_information-context2", fn2, kuri, meta, GNUNET_YES, &bo); + + ds = GNUNET_FS_directory_scan_start ("test-fs-file-information", + 1, + NULL, + &directory_scan_cb, + &bo); + GNUNET_assert (fi2 != NULL); fidir = GNUNET_FS_file_information_create_empty_directory (fs, @@ -131,8 +183,10 @@ run (void *cls, char *const *args, const char *cfgfile, GNUNET_FS_file_information_destroy (fidir, &mycleaner, NULL); GNUNET_DISK_directory_remove (fn1); GNUNET_DISK_directory_remove (fn2); + GNUNET_DISK_directory_remove (fn3); GNUNET_free (fn1); GNUNET_free (fn2); + GNUNET_free (fn3); GNUNET_FS_stop (fs); } -- 2.45.3 --------------08048EB8EFF71E62F27DD918--