Using Droplet for Email Servers

Introduction

The open source Droplet library supports both the S3 and CDMI protocols. It provides a convenient way to connect e-mail servers to back-end object storage.

Droplet Rest API commands can be passed using the CDMI protocol to connectors that access object storage servers. They provide the object storage system with filesystem concepts, such as atomic creation of objects, naming by path, and directory listings. Data stored through a Dewpoint CDMI server is available through a FUSE mountpoint on Linux servers, enabling system administrators to mount mailboxes and perform critical managerial tasks, such as backups and maintenance operations, without having to change administrative toolsets.

Typical email use cases

After setting the context for Droplet, you can use Droplet REST API commands for typical e-mail operations, such as sending or reading an e-mail, listing the contents of a mailbox, etc.

Setting the context

To use droplet for e-mail operations, you must first set the context:

#include <droplet.h>
#include <assert.h>
#include <sys/param.h>

int
main(int argc,
     char **argv)
{
  int ret;
  dpl_ctx_t *ctx;
  char *folder = NULL;
  int folder_len;
  dpl_dict_t *metadata = NULL;
  char *data_buf = NULL;
  size_t data_len;
  char *data_buf_returned = NULL;
  u_int data_len_returned;
  dpl_dict_t *metadata_returned = NULL;
  dpl_dict_t *metadata2_returned = NULL;
  dpl_dict_var_t *metadatum = NULL;
  dpl_option_t option;
  dpl_sysmd_t sysmd;
  char *resource_path = NULL;
  char new_path[MAXPATHLEN];
  dpl_vec_t *files = NULL;
  dpl_vec_t *sub_directories = NULL;
  int i;

  if (2 != argc)
    {
      fprintf(stderr, "usage: restrest folder\n");
      ret = 1;
      goto end;
    }

  folder = argv[1];
  folder_len = strlen(folder);
  if (folder_len < 1)
    {
      fprintf(stderr, "bad folder\n");
      ret = 1;
      goto end;
    }
  if (folder[folder_len-1] != '/')
    {
      fprintf(stderr, "folder name must end with a slash\n");
      ret = 1;
      goto end;
    }

  ret = dpl_init();           //init droplet library
  if (DPL_SUCCESS != ret)
    {
      fprintf(stderr, "dpl_init failed\n");
      ret = 1;
      goto end;
    }

  //open default profile
  ctx = dpl_ctx_new(NULL,     //droplet directory, default: "~/.droplet"
                    NULL);    //droplet profile, default:   "default"
  if (NULL == ctx)
    {
      fprintf(stderr, "dpl_ctx_new failed\n");
      ret = 1;
      goto free_dpl;
    }

Creating a folder

Before you can send or read an e-mail, you must create an e-mail folder. You create a folder by calling the ''dpl_put'' command:

  fprintf(stderr, "creating folder\n");

  ret = dpl_put(ctx,           //the context
                NULL,          //no bucket
                folder,        //the folder
                NULL,          //no subresource
                NULL,          //no option
                DPL_FTYPE_DIR, //directory
                NULL,          //no condition
                NULL,          //no range
                NULL,          //no metadata
                NULL,          //no sysmd
                NULL,          //object body
                0);            //object length
  if (DPL_SUCCESS != ret)
    {
      fprintf(stderr, "dpl_put failed: %s (%d)\n", dpl_status_str(ret), ret);
      ret = 1;
      goto free_all;
    }

  /**/

  data_len = 10000;
  data_buf = malloc(data_len);
  if (NULL == data_buf)
    {
      fprintf(stderr, "alloc data failed\n");
      ret = 1;
      goto free_all;
    }

  memset(data_buf, 'z', data_len);

  metadata = dpl_dict_new(13);
  if (NULL == metadata)
    {
      fprintf(stderr, "dpl_dict_new failed\n");
      ret = 1;
      goto free_all;
    }
 
  ret = dpl_dict_add(metadata, "foo", "bar", 0);
  if (DPL_SUCCESS != ret)
    {
      fprintf(stderr, "dpl_dict_add failed\n");
      ret = 1;
      goto free_all;
    }

  ret = dpl_dict_add(metadata, "foo2", "qux", 0);
  if (DPL_SUCCESS != ret)
    {
      fprintf(stderr, "dpl_dict_add failed\n");
      ret = 1;
      goto free_all;
    }

Delivering an email

You can add an e-mail to an e-mail folder with the ''dpl_post'' command. When this command passes the e-mail object to the CDMI server, the e-mail is given a randomized key, such as the following:

resource path /dewpoint/mbox6/347B1515682A22BACE1FFE2F7368C251 (key 999B0C50646334200002E8000000370200000020)

You can then copy and move the e-mail object with the ''dpl_copy'' command to provide it with a more user-friendly name:

mbox6/u.1

This sample code calls both the ''dpl_post'' and ''dpl_copy'' commands:

  fprintf(stderr, "atomic creation of an object+MD\n");

  ret = dpl_post(ctx,           //the context
                 NULL,          //no bucket
                 folder,        //the folder
                 NULL,          //no subresource
                 NULL,          //no option
                 DPL_FTYPE_REG, //regular object
                 metadata,      //the metadata
                 NULL,          //no sysmd
                 data_buf,      //object body
                 data_len,      //object length
                 NULL,          //no query params
                 &sysmd,        //the returned sysmd
                 &resource_path); //the resource location
  if (DPL_SUCCESS != ret)
    {
      fprintf(stderr, "dpl_post failed: %s (%d)\n", dpl_status_str(ret), ret);
      ret = 1;
      goto free_all;
    }

  fprintf(stderr, "resource path %s (key %s)\n", resource_path, sysmd.id);

  snprintf(new_path, sizeof (new_path), "%su.1", folder);

  ret = dpl_copy(ctx,
                 NULL,          //no src bucket
                 resource_path, //the src resource
                 NULL,          //no src sub resource
                 NULL,          //no dst bucket
                 new_path,      //dst resource
                 NULL,          //no dst sub resource
                 NULL,          //no option
                 DPL_FTYPE_REG, //regular file
                 DPL_COPY_DIRECTIVE_MOVE, //rename
                 NULL,          //no metadata
                 NULL,          //no sysmd
                 NULL);         //no server side condition
  if (DPL_SUCCESS != ret)
    {
      fprintf(stderr, "dpl_move %s to %s failed: %s (%d)\n", resource_path, new_path, dpl_status_str(ret), ret);
      ret = 1;
      goto free_all;
    }

Reading an email

You read an e-mail and its header with the ''dpl_get'' command:

fprintf(stderr, "getting object+MD\n");

  ret = dpl_get(ctx,           //the context
                NULL,          //no bucket
                new_path,      //the key
                NULL,          //no subresource
                NULL,          //no opion
                DPL_FTYPE_REG, //object type
                NULL,          //no condition
                NULL,          //no range
                &data_buf_returned,  //data object
                &data_len_returned,  //data object length
                &metadata_returned, //metadata
                NULL);              //sysmd
  if (DPL_SUCCESS != ret)
    {
      fprintf(stderr, "dpl_get_id failed: %s (%d)\n", dpl_status_str(ret), ret);
      ret = 1;
      goto free_all;
    }

  fprintf(stderr, "checking object\n");

  if (data_len != data_len_returned)
    {
      fprintf(stderr, "data lengths mismatch\n");
      ret = 1;
      goto free_all;
    }

  if (0 != memcmp(data_buf, data_buf_returned, data_len))
    {
      fprintf(stderr, "data content mismatch\n");
      ret = 1;
      goto free_all;
    }

  fprintf(stderr, "checking metadata\n");

  metadatum = dpl_dict_get(metadata_returned, "foo");
  if (NULL == metadatum)
    {
      fprintf(stderr, "missing metadatum\n");
      ret = 1;
      goto free_all;
    }

  assert(metadatum->val->type == DPL_VALUE_STRING);
  if (strcmp(metadatum->val->string, "bar"))
    {
      fprintf(stderr, "bad value in metadatum\n");
      ret = 1;
      goto free_all;
    }

  metadatum = dpl_dict_get(metadata_returned, "foo2");
  if (NULL == metadatum)
    {
      fprintf(stderr, "missing metadatum\n");
      ret = 1;
      goto free_all;
    }

  assert(metadatum->val->type == DPL_VALUE_STRING);
  if (strcmp(metadatum->val->string, "qux"))
    {
      fprintf(stderr, "bad value in metadatum\n");
      ret = 1;
      goto free_all;
    }

Setting the metadata only

You can just set the metadata of an e-mail with the ''dpl_dict_add'' command, followed by the ''dpl_copy'' command:

  fprintf(stderr, "setting MD only\n");

  ret = dpl_dict_add(metadata, "foo", "bar2", 0);
  if (DPL_SUCCESS != ret)
    {
      fprintf(stderr, "error updating metadatum: %s (%d)\n", dpl_status_str(ret), ret);
      ret = 1;
      goto free_all;
    }

  ret = dpl_copy(ctx,           //the context
                 NULL,          //no src bucket
                 new_path,      //the key
                 NULL,          //no subresource
                 NULL,          //no dst bucket
                 new_path,      //the same key
                 NULL,          //no subresource
                 NULL,          //no option
                 DPL_FTYPE_REG, //object type
                 DPL_COPY_DIRECTIVE_METADATA_REPLACE,  //tell server to replace metadata
                 metadata,      //the updated metadata
                 NULL,          //no sysmd
                 NULL);         //no condition
  if (DPL_SUCCESS != ret)
    {
      fprintf(stderr, "error updating metadata: %s (%d)\n", dpl_status_str(ret), ret);
      ret = 1;
      goto free_all;
    } 

Reading metadata of an email

To read the metadata of an e-mail, call the ''dpl_head'' command:

 fprintf(stderr, "getting MD only\n");

  ret = dpl_head(ctx,      //the context
                 NULL,     //no bucket,
                 new_path, //the key
                 NULL,     //no subresource
                 NULL,     //no option
                 NULL,     //no condition,
                 &metadata2_returned,
                 NULL);
  if (DPL_SUCCESS != ret)
    {
      fprintf(stderr, "error getting metadata: %s (%d)\n", dpl_status_str(ret), ret);
      ret = 1;
      goto free_all;
    }

  fprintf(stderr, "checking metadata\n");

  metadatum = dpl_dict_get(metadata2_returned, "foo");
  if (NULL == metadatum)
    {
      fprintf(stderr, "missing metadatum\n");
      ret = 1;
      goto free_all;
    }

  assert(metadatum->val->type == DPL_VALUE_STRING);
  if (strcmp(metadatum->val->string, "bar2"))
    {
      fprintf(stderr, "bad value in metadatum\n");
      ret = 1;
      goto free_all;
    }

  metadatum = dpl_dict_get(metadata2_returned, "foo2");
  if (NULL == metadatum)
    {
      fprintf(stderr, "missing metadatum\n");
      ret = 1;
      goto free_all;
    }

  assert(metadatum->val->type == DPL_VALUE_STRING);
  if (strcmp(metadatum->val->string, "qux"))
    {
      fprintf(stderr, "bad value in metadatum\n");
      ret = 1;
      goto free_all;
    }

Listing a mailbox

To list the mail folder contents:

Warning: Listing should only be used for cleanup and recovery operations, not for regular operations like storing or retreiving messages.

 fprintf(stderr, "listing of folder\n");

  ret = dpl_list_bucket(ctx, 
                        NULL,
                        folder,
                        "/",
                        &files,
                        &sub_directories);
  if (DPL_SUCCESS != ret)
    {
      fprintf(stderr, "error listing folder: %s (%d)\n", dpl_status_str(ret), ret);
      ret = 1;
      goto free_all;
    }

  for (i = 0;i < files->n_items;i++)
    {
      dpl_object_t *obj = (dpl_object_t *) dpl_vec_get(files, i);
      dpl_sysmd_t obj_sysmd;
      dpl_dict_t *obj_md = NULL;
      
      ret = dpl_head(ctx, 
                     NULL, //no bucket
                     obj->path, 
                     NULL, //subresource
                     NULL, //option
                     NULL, //condition
                     &obj_md, //user metadata
                     &obj_sysmd); //system metadata
      if (DPL_SUCCESS != ret)
        {
          fprintf(stderr, "getattr error on %s: %s (%d)\n", obj->path, dpl_status_str(ret), ret);
          ret = 1;
          goto free_all;
        }

      fprintf(stderr, "file %s: size=%ld mtime=%lu\nmetadata:\n", obj->path, obj_sysmd.size, obj_sysmd.mtime);
      dpl_dict_print(obj_md, stderr, 5);
      dpl_dict_free(obj_md);
    }

  for (i = 0;i < sub_directories->n_items;i++)
    {
      dpl_common_prefix_t *dir = (dpl_common_prefix_t *) dpl_vec_get(files, i);

      fprintf(stderr, "dir %s\n", dir->prefix);
    }

Deleting an email

Delete an e-mail with the ''dpl_delete'' command:

 fprintf(stderr, "delete object+MD\n");

  ret = dpl_delete(ctx,       //the context
                   NULL,      //no bucket
                   new_path,  //the key
                   NULL,      //no subresource
                   NULL,      //no option
                   NULL);     //no condition
  if (DPL_SUCCESS != ret)
    {
      fprintf(stderr, "error deleting object: %s (%d)\n", dpl_status_str(ret), ret);
      ret = 1;
      goto free_all;
    }
  ret = 0;

Freeing the context

 free_all:

  if (NULL != sub_directories)
    dpl_vec_common_prefixes_free(sub_directories);

  if (NULL != files)
    dpl_vec_objects_free(files);

  if (NULL != resource_path)
    free(resource_path);

  if (NULL != metadata2_returned)
    dpl_dict_free(metadata2_returned);

  if (NULL != metadata_returned)
    dpl_dict_free(metadata_returned);

  if (NULL != data_buf_returned)
    free(data_buf_returned);

  if (NULL != metadata)
    dpl_dict_free(metadata);

  if (NULL != data_buf)
    free(data_buf);

  dpl_ctx_free(ctx); //free context

 free_dpl:
  dpl_free();        //free droplet library

 end:
  return ret;
}