propfind invalidates filecache

Before this change, we only serve from the file cache if the content was updated
by a call to the server in the last 2 seconds, or the thread is in saint mode.
The reason is that we do a conditional get using the If-None-Match header and
the Etag from the last response for that file path, hoping to get a relatively
fast 304 most of the time. However, querying total duration of requests broken
down by cluster, method, and status shows that our 304 GETs are using just as
much time in process as normal GETs and PUTs due to sheer volume.

This change invalidates the file cache for a path when a PROPFIND shows that the
path has been updated in Valhalla. The mechanism for invalidation is to set the
timestamp in `filecache_pdata->last_server_update` to 1 for the path in
question, and to serve all paths from file cache if the entry isn't invalidated.
This arrangement affords serving invalidated content while in saint mode, and
aggressive removal of known invalid content by the file cache cleanup mechanism.

An optimization we could consider is to use the etag to locate the file cache
content, in order to provide deduplication and potentially better cache
coherency for the case where the same file content moves between paths. We would
need to add the etag to the propfind data, parse it out, pass it to the
callback, and use that to evaluate the freshness of the file cache data, and
also change the logic for naming a cache file to rename the file after its hash
is known, but before publishing file cache data to leveldb. Since the etag and
the filename are both properties of the file cache data, we would need to either
change the format of the stat cache data, add another entry type, or add a
concept of "unfilled" file cache data.

src/filecache.c

@@ -386,7 +386,7 @@ static void get_fresh_fd(filecache_t *cache,
     // If we're in saint mode, don't go to the server
     if (pdata != NULL &&
             ((flags & O_TRUNC) || use_local_copy ||
-            (pdata->last_server_update == 0) || (time(NULL) - pdata->last_server_update) <= STAT_CACHE_NEGATIVE_TTL)) {
+            (pdata->last_server_update == 0) || pdata->last_server_update > FILECACHE_INVALIDATED)) {
         const float samplerate = 1.0; // always sample stat
         log_print(LOG_DEBUG, SECTION_FILECACHE_OPEN, "%s: file is fresh or being truncated: %s::%s", 
                 funcname, path, pdata->filename);
@@ -403,7 +403,7 @@ static void get_fresh_fd(filecache_t *cache,
             // I believe we can get here on a PUT, and succeed in getting a previous copy from the cache
             // This will eventually fail if we are already in saint mode, but should succeed for now
             if (use_local_copy) {
-                if (rw ) stats_counter("put_saint_mode_failure", 1, samplerate);
+                if (rw) stats_counter("put_saint_mode_failure", 1, samplerate);
                 else stats_counter("get_saint_mode_failure", 1, samplerate);
                 log_print(LOG_WARNING, SECTION_FILECACHE_OPEN, "%s: get_saint_mode_failure on file: %s::%s", funcname, path, pdata->filename);
             } else {
@@ -1047,6 +1047,9 @@ static void put_return_etag(const char *path, int fd, char *etag, GError **gerr)
     // If in saint mode and we count this request as a saint_write in get_fresh_fd(), we would be
     // double counting; but I don't see how we get here at all if we detect saint mode in get_fresh_fd()
     rwp_t rwp = WRITE;
+    GChecksum *sha512_checksum;
+    GChecksum *md5_checksum;
+    FILE *fp;
 
     BUMP(filecache_return_etag);
 
@@ -1070,9 +1073,6 @@ static void put_return_etag(const char *path, int fd, char *etag, GError **gerr)
 
     log_print(LOG_DEBUG, SECTION_FILECACHE_COMM, "%s: file size %d", funcname, st.st_size);
 
-    GChecksum *sha512_checksum;
-    GChecksum *md5_checksum;
-    FILE *fp;
     fp = fdopen(dup(fd), "r");
     if (!fp) {
       g_set_error(gerr, system_quark(), errno, "%s: NULL fp from fdopen on fd %d for path %s", funcname, fd, path);
@@ -1753,6 +1753,28 @@ void filecache_pdata_move(filecache_t *cache, const char *old_path, const char *
     return;
 }
 
+// mark filecache stale for path
+void filecache_invalidate(filecache_t* cache, const char* path, GError** gerr) {
+    struct filecache_pdata *pdata = NULL;
+    GError *subgerr = NULL;
+    const char* funcname = "filecache_invalidate";
+
+    pdata = filecache_pdata_get(cache, path, &subgerr);
+    if (gerr) {
+      g_propagate_prefixed_error(gerr, subgerr, "%s: ", funcname);
+      return;
+    }
+    // We mark the data invalidated by setting the last update very far in the past.
+    // This preserves the etag so we can still try it and get lucky, eg if the file
+    // content was updated, and then updated back to its previous value.
+    pdata->last_server_update = FILECACHE_INVALIDATED;
+    filecache_pdata_set(cache, path, pdata, &subgerr);
+    if (gerr) {
+      g_propagate_prefixed_error(gerr, subgerr, "%s: ", funcname);
+      return;
+    }
+}
+
 // Does *not* allocate a new string.
 static const char *key2path(const char *key) {
     char *prefix;

src/filecache.h

@@ -37,13 +37,14 @@
 #define E_FC_LDBERR EIO
 #define E_FC_CURLERR ENETDOWN
 #define E_FC_FILETOOLARGE EFBIG
+#define FILECACHE_INVALIDATED 1
 
 typedef leveldb_t filecache_t;
 
 void filecache_print_stats(void);
 void filecache_init(char *cache_path, GError **gerr);
 void filecache_delete(filecache_t *cache, const char *path, bool unlink, GError **gerr);
-void filecache_open(char *cache_path, filecache_t *cache, const char *path, struct fuse_file_info *info, 
+void filecache_open(char *cache_path, filecache_t *cache, const char *path, struct fuse_file_info *info,
         bool grace, bool rw, GError **gerr);
 ssize_t filecache_read(struct fuse_file_info *info, char *buf, size_t size, off_t offset, GError **gerr);
 ssize_t filecache_write(struct fuse_file_info *info, const char *buf, size_t size, off_t offset, GError **gerr);
@@ -54,6 +55,7 @@ int filecache_fd(struct fuse_file_info *info);
 void filecache_set_error(struct fuse_file_info *info, int error_code);
 void filecache_forensic_haven(const char *cache_path, filecache_t *cache, const char *path, off_t fsize, GError **gerr);
 void filecache_pdata_move(filecache_t *cache, const char *old_path, const char *new_path, GError **gerr);
+void filecache_invalidate(filecache_t* cache, const char* path, GError** gerr);
 bool filecache_cleanup(filecache_t *cache, const char *cache_path, bool first, GError **gerr);
 struct curl_slist* enhanced_logging(struct curl_slist *slist, int log_level, int section, const char *format, ...);
 

src/fusedav.c

@@ -178,7 +178,7 @@ static void getdir_propfind_callback(__unused void *userdata, const char *path,
     struct stat_cache_value *existing = NULL;
     struct stat_cache_value value;
     rwp_t rwp = PROPFIND;
-    GError *subgerr1 = NULL ;
+    GError *subgerr1 = NULL;
 
     log_print(LOG_INFO, SECTION_FUSEDAV_PROP, "%s: %s (%lu)", funcname, path, status_code);
 
@@ -420,6 +420,10 @@ static void getdir_propfind_callback(__unused void *userdata, const char *path,
             if (subgerr1) {
                 g_propagate_prefixed_error(gerr, subgerr1, "%s: ", funcname);
             }
+            filecache_invalidate(config->cache, path, &subgerr1);
+            if (subgerr1) {
+                g_propagate_prefixed_error(gerr, subgerr1, "%s: ", funcname);
+            }
         }
     }