Re: [AVFS] Caching content of extfs archives

Index: include/Makefile.am
===================================================================
RCS file: /home/cvsroot/avfs/include/Makefile.am,v
retrieving revision 1.1
diff -u -r1.1 Makefile.am
--- include/Makefile.am	5 Jun 2005 21:26:54 -0000	1.1
+++ include/Makefile.am	21 Feb 2006 14:24:43 -0000
@@ -29,6 +29,7 @@
 	serialfile.h \
 	socket.h \
 	state.h \
+	tmpfile.h \
 	ugid.h \
 	version.h \
 	zfile.h \
Index: include/cache.h
===================================================================
RCS file: /home/cvsroot/avfs/include/cache.h,v
retrieving revision 1.1.1.1
diff -u -r1.1.1.1 cache.h
--- include/cache.h	25 May 2005 18:10:46 -0000	1.1.1.1
+++ include/cache.h	16 Feb 2006 19:57:53 -0000
@@ -15,3 +15,7 @@
 void av_cacheobj_setsize(struct cacheobj *cobj, avoff_t diskusage);
 void av_cache_checkspace();
 void av_cache_diskfull();
+
+int av_cache2_set(void *obj, const char *name);
+void *av_cache2_get(const char *name);
+void av_cache2_setsize(const char *name, avoff_t diskusage);
Index: modules/extfs.c
===================================================================
RCS file: /home/cvsroot/avfs/modules/extfs.c,v
retrieving revision 1.2
diff -u -r1.2 extfs.c
--- modules/extfs.c	30 Jan 2006 22:39:38 -0000	1.2
+++ modules/extfs.c	21 Feb 2006 16:52:36 -0000
@@ -1,6 +1,7 @@
 /*
     AVFS: A Virtual File System Library
     Copyright (C) 1998  Miklos Szeredi ([email protected])
+    Copyright (C) 2006  Ralf Hoffmann ([email protected])
     
     This program can be distributed under the terms of the GNU GPL.
     See the file COPYING.
@@ -17,10 +18,15 @@
 #include "parsels.h"
 #include "realfile.h"
 #include "runprog.h"
+#include "filecache.h"
+#include "cache.h"
+#include "exit.h"
 
 #include <unistd.h>
 #include <fcntl.h>
 
+static AV_LOCK_DECL(extfslock);
+
 struct extfsdata {
     int needbase;
     char *progpath;
@@ -30,11 +36,22 @@
     char *fullpath;
 };
 
-struct extfsfile {
+struct extfscacheentry {
     char *tmpfile;
+};
+
+struct extfsfile {
+    struct extfscacheentry *cent;
     int fd;
 };
 
+static void extfscacheentry_delete(struct extfscacheentry *cent)
+{
+    if( cent->tmpfile != NULL ) {
+        av_del_tmpfile(cent->tmpfile);
+    }
+}
+
 static void fill_extfs_link(struct archive *arch, struct entry *ent,
                            char *linkname)
 {
@@ -178,6 +195,25 @@
     return res;
 }
 
+static int get_key_for_node(ventry *ve, struct archfile *fil, char **resp)
+{
+    struct extfsnode *enod = (struct extfsnode *) fil->nod->data;
+    char *key;
+    int res;
+
+    if(enod == NULL) {
+        return -EISDIR;
+    }
+
+    res = av_filecache_getkey(ve, &key);
+    if(res < 0)
+        return res;
+
+    key = av_stradd(key, "/", enod->fullpath, NULL);
+    *resp = key;
+    return 0;
+}
+
 static int get_extfs_file(ventry *ve, struct archfile *fil,
                           const char *tmpfile)
 {
@@ -275,30 +311,63 @@
 {
     int res;
     struct extfsfile *efil;
-    char *tmpfile;
     int fd;
-
-    res = av_get_tmpfile(&tmpfile);
+    char *key;
+    struct extfscacheentry *cent;
+    
+    /* get key for extfscache */
+    res = get_key_for_node(ve, fil, &key);
     if(res < 0)
         return res;
 
-    res = get_extfs_file(ve, fil, tmpfile);
-    if(res < 0) {
-        av_del_tmpfile(tmpfile);
-        return res;
+    AV_LOCK(extfslock);
+    cent = av_cache2_get(key);
+    if (cent == NULL) {
+        char *tmpfile;
+        avoff_t tmpsize;
+
+	/* no entry in cache so create a temporary file... */
+        res = av_get_tmpfile(&tmpfile);
+        if(res < 0) {
+	    av_free(key);
+	    AV_UNLOCK(extfslock);
+            return res;
+	}
+	res = get_extfs_file(ve, fil, tmpfile);
+	if(res < 0) {
+	    av_free(key);
+	    av_del_tmpfile(tmpfile);
+	    AV_UNLOCK(extfslock);
+	    return res;
+	}
+
+	/* ...create an object to store tmpfile */
+	AV_NEW_OBJ(cent, extfscacheentry_delete);
+	cent->tmpfile = tmpfile;
+
+	/* put it in the extfscache */
+	av_cache2_set(cent,key);
+	AV_UNLOCK(extfslock);
+
+        tmpsize = av_tmpfile_blksize(tmpfile);
+        if(tmpsize > 0)
+            av_cache2_setsize(key, tmpsize);
+    } else {
+	AV_UNLOCK(extfslock);
     }
+    av_free(key);
 
-    fd = open(tmpfile, O_RDONLY);
+    fd = open(cent->tmpfile, O_RDONLY);
     if(fd == -1) {
         res = -errno; 
-        av_log(AVLOG_ERROR, "EXTFS: Could not open %s: %s", tmpfile,
+        av_log(AVLOG_ERROR, "EXTFS: Could not open %s: %s", cent->tmpfile,
                strerror(errno));
-        av_del_tmpfile(tmpfile);
+	av_unref_obj(cent);
         return res;
     }
 
     AV_NEW(efil);
-    efil->tmpfile = tmpfile;
+    efil->cent = cent;
     efil->fd = fd;
 
     fil->data = efil;
@@ -311,7 +380,8 @@
     struct extfsfile *efil = (struct extfsfile *) fil->data;
 
     close(efil->fd);
-    av_del_tmpfile(efil->tmpfile);
+
+    av_unref_obj(efil->cent);
     av_free(efil);
     
     return 0;
Index: src/cache.c
===================================================================
RCS file: /home/cvsroot/avfs/src/cache.c,v
retrieving revision 1.4
diff -u -r1.4 cache.c
--- src/cache.c	19 Jun 2005 20:31:41 -0000	1.4
+++ src/cache.c	20 Feb 2006 23:03:54 -0000
@@ -10,10 +10,23 @@
    TODO:
    
    Virtual filesystem where all the cached files can be found.
+
+   There are two interfaces available:
+   The av_cacheobj* functions are built around the cacheobj structure.
+     The user holds the reference to it and can use it to access the
+     stored object. The cache itself creates the cacheobj but doesn't
+     hold a reference to it so it doesn't destroy the cacheobj but
+     will unref the object stored in the cacheobj struct if the space
+     is needed.
+   The av_cache2* functions are built around the name as a key for
+     stored object. The cacheobj is not return and the cache holds the
+     only reference to it and will destroy it at some point.
+
 */
 
 #include "cache.h"
 #include "internal.h"
+#include "exit.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -25,6 +38,8 @@
 
     struct cacheobj *next;
     struct cacheobj *prev;
+
+    int internal_obj;
 };
 
 #define MBYTE (1024 * 1024)
@@ -97,6 +112,27 @@
     return 0;
 }
 
+/**
+ * This is the exit handler to remove all temporary file stored
+ * using the new interface
+ */
+static void destroy_cache()
+{
+    struct cacheobj *cobj;
+
+    AV_LOCK(cachelock);
+    for(cobj = &cachelist; cobj->next != &cachelist; ) {
+        if(cobj->next->internal_obj) {
+            /* unref all internal objects which will remove the */
+            av_unref_obj(cobj->next);
+        } else {
+            /* this shouldn't happen */
+            cobj = cobj->next;
+        }
+    }
+    AV_UNLOCK(cachelock);
+}
+
 void av_init_cache()
 {
     struct statefile statf;
@@ -121,6 +157,8 @@
     statf.get = cache_getfunc;
     statf.data = cache_clear;
     av_avfsstat_register("cache/clear", &statf);
+    
+    av_add_exithandler(destroy_cache);
 }
 
 static void cacheobj_remove(struct cacheobj *cobj)
@@ -155,6 +193,10 @@
     av_free(cobj->name);
 }
 
+/**
+ * This is the destructor for external cacheobj's created
+ * using the old interface
+ */
 static void cacheobj_delete(struct cacheobj *cobj)
 {
     AV_LOCK(cachelock);
@@ -168,6 +210,21 @@
         cacheobj_free(cobj);
 }
 
+/**
+ * This is the destructor for internal cacheobj's created
+ * using the new interface
+ */
+static void cacheobj_internal_delete(struct cacheobj *cobj)
+{
+    if(cobj->obj != NULL) {
+        cacheobj_remove(cobj);
+        disk_usage -= cobj->diskusage;
+    }
+
+    if(cobj->obj != NULL)
+        cacheobj_free(cobj);
+}
+
 struct cacheobj *av_cacheobj_new(void *obj, const char *name)
 {
     struct cacheobj *cobj;
@@ -179,6 +236,7 @@
     cobj->obj = obj;
     cobj->diskusage = 0;
     cobj->name = av_strdup(name);
+    cobj->internal_obj = 0;
     av_ref_obj(obj);
 
     AV_LOCK(cachelock);
@@ -199,13 +257,17 @@
     if(cobj == &cachelist)
         return 0;
 
-    cacheobj_remove(cobj);
-    disk_usage -= cobj->diskusage;
-    tmpcobj = *cobj;
-    cobj->obj = NULL;
-    AV_UNLOCK(cachelock);
-    cacheobj_free(&tmpcobj);
-    AV_LOCK(cachelock);
+    if(cobj->internal_obj) {
+        av_unref_obj(cobj);
+    } else {
+        cacheobj_remove(cobj);
+        disk_usage -= cobj->diskusage;
+        tmpcobj = *cobj;
+        cobj->obj = NULL;
+        AV_UNLOCK(cachelock);
+        cacheobj_free(&tmpcobj);
+        AV_LOCK(cachelock);
+    }
 
     return 1;
 }
@@ -294,3 +356,82 @@
 
     return obj;
 }
+
+static struct cacheobj *cacheobj2_find(const char *name)
+{
+    struct cacheobj *cobj;
+    
+    for(cobj = cachelist.next; cobj != &cachelist; cobj = cobj->next) {
+        if(cobj->internal_obj == 1)
+            if(strcmp(cobj->name, name) == 0)
+                break;
+    }
+
+    if(cobj->obj == NULL)
+        return NULL;
+
+    return cobj;
+}
+
+int av_cache2_set(void *obj, const char *name)
+{
+    struct cacheobj *cobj, *oldcobj;
+
+    if(obj != NULL) {
+        AV_NEW_OBJ(cobj, cacheobj_internal_delete);
+        cobj->obj = obj;
+        cobj->diskusage = 0;
+        cobj->name = av_strdup(name);
+        cobj->internal_obj = 1;
+        av_ref_obj(obj);
+    } else {
+        cobj = NULL;
+    }
+
+    AV_LOCK(cachelock);
+    oldcobj = cacheobj2_find(name);
+
+    if(oldcobj != NULL )
+        av_unref_obj(oldcobj);
+
+    if(cobj != NULL)
+        cacheobj_insert(cobj);
+
+    AV_UNLOCK(cachelock);
+
+    return 0;
+}
+
+void *av_cache2_get(const char *name)
+{
+    struct cacheobj *cobj;
+    void *obj = NULL;
+    
+    AV_LOCK(cachelock);
+    cobj = cacheobj2_find(name);
+    if(cobj != NULL) {
+        cacheobj_remove(cobj);
+        cacheobj_insert(cobj);
+        obj = cobj->obj;
+        av_ref_obj(obj);
+    }
+    AV_UNLOCK(cachelock);
+
+    return obj;
+}
+
+void av_cache2_setsize(const char *name, avoff_t diskusage)
+{
+    struct cacheobj *cobj;
+
+    AV_LOCK(cachelock);
+    cobj = cacheobj2_find(name);
+    if(cobj->obj != NULL && cobj->diskusage != diskusage) {
+        disk_usage -= cobj->diskusage;
+        cobj->diskusage = diskusage;
+        disk_usage += cobj->diskusage;
+        
+        cache_checkspace(0, cobj);
+    }
+    AV_UNLOCK(cachelock);
+}
Index: src/tmpfile.c
===================================================================
RCS file: /home/cvsroot/avfs/src/tmpfile.c,v
retrieving revision 1.1.1.1
diff -u -r1.1.1.1 tmpfile.c
--- src/tmpfile.c	25 May 2005 18:10:47 -0000	1.1.1.1
+++ src/tmpfile.c	21 Feb 2006 12:24:14 -0000
@@ -174,3 +174,23 @@
 
     return freebytes;
 }
+
+avoff_t av_tmpfile_blksize(const char *tmpf)
+{
+    int res;
+    struct stat stbuf;
+    
+    if(tmpf == NULL)
+        return -1;
+
+    res = stat(tmpf, &stbuf);
+    if(res == 0) {
+        /* Ramfs returns 0 diskusage */
+        if(stbuf.st_blocks == 0)
+            return stbuf.st_size;
+        else
+            return stbuf.st_blocks * 512;
+    }
+    else
+        return -1;
+}