Accessing multiple key/value pairs

Each of the functions presented in the previous tutorial leads to one RPC sent to the provider. Hence, batching can be helpful to optimize performance by accessing multiple key/value pairs in a single RPC.

Yokan provides two ways of accessing batches of key/value pairs: packed, and unpacked, with function signatures varying slightly depending on which way is used.

Unpacked accesses can be used when lists of keys and values are not located in a contiguous memory location. Functions using unpacked accesses will generally have the _multi suffix, and take arrays of pointers.

Packed accesses can be used when keys and values are packed contiguously in memory. Functions using packed accesses will have the _packed suffix, and will take pointers to single buffers instead of arrays of pointers.

Important

In general, it is recommended to use packed accesses as much as possible, even if this means bouncing data through an intermediate buffer. Unpacked accesses should be reserved for cases where (1) such a copy would be costly (e.g. large key/value pairs) and (2) you know the size of the keys and values (or at least a very close upper bound) ahead of time.

Unpacked accesses

The following program gives you an example of usage for all unpacked functions.

client.c (show/hide)

#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <margo.h>
#include <yokan/client.h>
#include <yokan/database.h>

int main(int argc, char** argv)
{
    if(argc != 3) {
        fprintf(stderr, "Usage: %s <address> <provider id>\n", argv[0]);
        exit(-1);
    }
    margo_instance_id mid = margo_init("na+sm", MARGO_CLIENT_MODE, 0, 0);
    assert(mid);

    uint16_t provider_id = atoi(argv[2]);
    hg_addr_t server_addr = HG_ADDR_NULL;
    hg_return_t hret = margo_addr_lookup(mid, argv[1], &server_addr);
    assert(hret == HG_SUCCESS);

    yk_return_t ret;
    yk_client_t client = YOKAN_CLIENT_NULL;

    ret = yk_client_init(mid, &client);
    assert(ret == YOKAN_SUCCESS);

    yk_database_handle_t db_handle = YOKAN_DATABASE_HANDLE_NULL;
    ret = yk_database_handle_create(
        client, server_addr, provider_id, true, &db_handle);
    assert(ret == YOKAN_SUCCESS);

    // ------------------------------------------------------
    printf("Putting the following key/value pairs:\n");
    const char* const keys[] = {
        "matthieu",
        "phil",
        "rob",
        "shane",
        "kevin",
        "zhe",
        "srinivasan"
    };
    const char* const values[] = {
        "dorier",
        "carns",
        "ross",
        "snyder",
        "harms",
        "wang",
        "ramesh"
    };
    size_t ksizes[7];
    size_t vsizes[7];
    for(int i = 0; i < 7; i++) {
        ksizes[i] = strlen(keys[i]);
        vsizes[i] = strlen(values[i]);
        printf("\t%s => %s\n", keys[i], values[i]);
    }

    /* putting multiple key/value pairs at once */
    ret = yk_put_multi(db_handle, YOKAN_MODE_DEFAULT, 7,
                       (const void* const*)keys, ksizes,
                       (const void* const*)values, vsizes);
    assert(ret == YOKAN_SUCCESS);

    // ------------------------------------------------------

    const char* const keys2[4] = {
        "matthieu",
        "marc",
        "anna",
        "shane",
    };
    size_t ksizes2[4];
    for(int i=0; i < 4; i++)
        ksizes2[i] = strlen(keys2[i]);

    /* checking that the keys exist */
    uint8_t flags[1];
    ret = yk_exists_multi(db_handle, YOKAN_MODE_DEFAULT, 4,
                          (const void* const*)keys2, ksizes2, flags);
    assert(ret == YOKAN_SUCCESS);
    printf("Checking if the following keys exist:\n");
    for(int i=0; i < 4; i++) {
        bool flag = yk_unpack_exists_flag(flags, i);
        printf("\t%s => %s\n", keys2[i], flag ? "YES" : "NO");
    }

    // ------------------------------------------------------

    size_t vsizes2[4];

    /* getting the length of values associated with keys */
    ret = yk_length_multi(db_handle, YOKAN_MODE_DEFAULT, 4,
                          (const void* const*)keys2, ksizes2, vsizes2);
    assert(ret == YOKAN_SUCCESS);
    printf("Checking if the length of the values for the following keys:\n");
    for(int i=0; i < 4; i++) {
        if(vsizes2[i] != YOKAN_KEY_NOT_FOUND)
            printf("\t%s => %lu\n", keys2[i], vsizes2[i]);
        else
            printf("\t%s => (not found)\n", keys2[i]);
    }

    // ------------------------------------------------------

    char* values2[4];
    for(int i = 0; i < 4; i++) {
        values2[i] = calloc(1, 16);
        vsizes2[i] = 16;
    }
    // intentionally limiting the size of the last key
    vsizes2[3] = 3;

    /* getting multiple key/value pairs at once */
    ret = yk_get_multi(db_handle, YOKAN_MODE_DEFAULT, 4,
                       (const void* const*)keys2, ksizes2,
                       (void * const*)values2, vsizes2);
    assert(ret == YOKAN_SUCCESS);
    printf("Getting the values for the following keys:\n");
    for(int i=0; i < 4; i++) {
        if(vsizes2[i] == YOKAN_KEY_NOT_FOUND)
            printf("\t%s => (not found)\n", keys2[i]);
        else if(vsizes2[i] == YOKAN_SIZE_TOO_SMALL)
            printf("\t%s => (buffer too small)\n", keys2[i]);
        else
            printf("\t%s => %s\n", keys2[i], values2[i]);
    }

    for(int i = 0; i < 4; i++) {
        free(values2[i]);
    }

    // ------------------------------------------------------

    char* keys3[4];
    size_t ksizes3[4];
    for(int i = 0; i < 4; i++) {
        keys3[i] = calloc(1, 8);
        ksizes3[i] = 8; // 8 won't be enough for "srinivasan"
    }

    ret = yk_list_keys(db_handle, YOKAN_MODE_INCLUSIVE,
            "shane", 5, "", 0, 4, (void* const*)keys3, ksizes3);
    assert(ret == YOKAN_SUCCESS);
    printf("Listed the following keys:\n");
    for(int i=0; i < 4; i++) {
        if(ksizes3[i] == YOKAN_NO_MORE_KEYS)
            break;
        else if(ksizes3[i] == YOKAN_SIZE_TOO_SMALL)
            printf("\t(buffer too small)\n");
        else
            printf("\t%s\n", keys3[i]);
    }

    for(int i=0; i < 4; i++)
        free(keys3[i]);

    // ------------------------------------------------------

    char* values3[4];
    size_t vsizes3[4];
    for(int i = 0; i < 4; i++) {
        values3[i] = calloc(1, 16);
        vsizes3[i] = 16;
        keys3[i] = calloc(1, 16);
        ksizes3[i] = 16;
    }

    ret = yk_list_keyvals(db_handle, YOKAN_MODE_INCLUSIVE,
            "shane", 5, "", 0, 4, (void* const*)keys3, ksizes3, (void* const*)values3, vsizes3);
    assert(ret == YOKAN_SUCCESS);
    printf("Listed the following key/value pairs:\n");
    for(int i=0; i < 4; i++) {
        if(ksizes3[i] == YOKAN_NO_MORE_KEYS)
            break;
        values3[i][vsizes3[i]] = '\0';
        printf("\t%s => %s\n", keys3[i], values3[i]);
    }

    for(int i=0; i < 4; i++) {
        free(keys3[i]);
        free(values3[i]);
    }

    // ------------------------------------------------------

    printf("Erasing all the key/value pairs.\n");
    ret = yk_erase_multi(db_handle, YOKAN_MODE_DEFAULT, 7,
                         (const void * const*)keys, ksizes);
    assert(ret == YOKAN_SUCCESS);

    // ------------------------------------------------------

    ret = yk_database_handle_release(db_handle);
    assert(ret == YOKAN_SUCCESS);

    ret = yk_client_finalize(client);
    assert(ret == YOKAN_SUCCESS);

    margo_finalize(mid);

    return 0;
}

These functions are the following.

  • yk_put_multi: puts multiple key/value pairs into the database.

  • yk_exists_multi: checks if multiple keys exist in the database. The uint8_t flags[] output parameter is a bitfield, hence for N keys, its size should be int((N+8)/8). The bitfield can be interpreted using yk_unpack_exists_flag.

  • yk_length_multi: retrieves the length of the values associated with a set of keys.

  • yk_get_multi: retrieves the values associated with a set of keys.

  • yk_fetch_multi: retrieves the values associated with a set of keys.

  • yk_list_keys: lists a specified number of keys, starting after the specified lower bound.

  • yk_list_keyvals: same as yk_list_keyvals but also returns values.

Once again, more documentation on the semantics of all these functions is available in the yokan/database.h header.

Packed accesses

The following program gives you an example of usage for all packed functions.

client.c (show/hide)

#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <margo.h>
#include <yokan/client.h>
#include <yokan/database.h>

int main(int argc, char** argv)
{
    if(argc != 3) {
        fprintf(stderr, "Usage: %s <address> <provider id>\n", argv[0]);
        exit(-1);
    }
    margo_instance_id mid = margo_init("na+sm", MARGO_CLIENT_MODE, 0, 0);
    assert(mid);

    uint16_t provider_id = atoi(argv[2]);
    hg_addr_t server_addr = HG_ADDR_NULL;
    hg_return_t hret = margo_addr_lookup(mid, argv[1], &server_addr);
    assert(hret == HG_SUCCESS);

    yk_return_t ret;
    yk_client_t client = YOKAN_CLIENT_NULL;

    ret = yk_client_init(mid, &client);
    assert(ret == YOKAN_SUCCESS);

    yk_database_handle_t db_handle = YOKAN_DATABASE_HANDLE_NULL;
    ret = yk_database_handle_create(
        client, server_addr, provider_id, true, &db_handle);
    assert(ret == YOKAN_SUCCESS);

    // ------------------------------------------------------
    printf("Putting the following key/value pairs:\n");
    const char* keys =
        "matthieuphilrobshanekevinzhesrinivasan";
    const char* values =
        "doriercarnsrosssnyderharmswangramesh";
    size_t ksizes[] = { 8, 4, 3, 5, 5, 3, 10 };
    size_t vsizes[] = { 6, 5, 4, 6, 5, 4, 6 };

    size_t koffset = 0;
    size_t voffset = 0;
    for(int i = 0; i < 7; i++) {
        printf("\t%.*s => %.*s\n", (int)ksizes[i], keys+koffset,
                                   (int)vsizes[i], values+voffset);
        koffset += ksizes[i];
        voffset += vsizes[i];
    }

    /* putting multiple key/value pairs at once */
    ret = yk_put_packed(db_handle, YOKAN_MODE_DEFAULT, 7,
                       (const void*)keys, ksizes,
                       (const void*)values, vsizes);
    assert(ret == YOKAN_SUCCESS);

    // ------------------------------------------------------

    const char* keys2 = "matthieumarcannashane";
    size_t ksizes2[] = { 8, 4, 4, 5};

    /* checking that the keys exist */
    uint8_t flags[1];
    ret = yk_exists_packed(db_handle, YOKAN_MODE_DEFAULT, 4,
                          (const void*)keys2, ksizes2, flags);
    assert(ret == YOKAN_SUCCESS);
    printf("Checking if the following keys exist:\n");
    koffset = 0;
    for(int i=0; i < 4; i++) {
        bool flag = yk_unpack_exists_flag(flags, i);
        printf("\t%.*s => %s\n", (int)ksizes2[i], keys2+koffset, flag ? "YES" : "NO");
        koffset += ksizes2[i];
    }

    // ------------------------------------------------------

    size_t vsizes2[4];

    /* getting the length of values associated with keys */
    ret = yk_length_packed(db_handle, YOKAN_MODE_DEFAULT, 4,
                          (const void*)keys2, ksizes2, vsizes2);
    assert(ret == YOKAN_SUCCESS);
    printf("Checking if the length of the values for the following keys:\n");
    koffset = 0;
    for(int i=0; i < 4; i++) {
        if(vsizes2[i] != YOKAN_KEY_NOT_FOUND)
            printf("\t%.*s => %lu\n", (int)ksizes2[i], keys2+koffset, vsizes2[i]);
        else
            printf("\t%.*s => (not found)\n", (int)ksizes2[i], keys2+koffset);
        koffset += ksizes2[i];
    }

    // ------------------------------------------------------

    char values2[64];

    /* getting multiple key/value pairs at once */
    ret = yk_get_packed(db_handle, YOKAN_MODE_DEFAULT, 4,
                       (const void*)keys2, ksizes2,
                       64, (void*)values2, vsizes2);
    assert(ret == YOKAN_SUCCESS);
    printf("Getting the values for the following keys:\n");
    koffset = 0;
    voffset = 0;
    for(int i=0; i < 4; i++) {
        if(vsizes2[i] == YOKAN_KEY_NOT_FOUND)
            printf("\t%.*s => (not found)\n", (int)ksizes2[i], keys2+koffset);
        else if(vsizes2[i] == YOKAN_SIZE_TOO_SMALL)
            printf("\t%.*s => (buffer too small)\n", (int)ksizes2[i], keys2+koffset);
        else {
            printf("\t%.*s => %.*s\n", (int)ksizes2[i], keys2+koffset,
                                   (int)vsizes2[i], values2+voffset);
            voffset += vsizes2[i];
        }
        koffset += ksizes2[i];
    }

    // ------------------------------------------------------

    char keys3[64];
    size_t ksizes3[4];

    ret = yk_list_keys_packed(db_handle, YOKAN_MODE_INCLUSIVE,
            "shane", 5, "", 0, 4, (void*)keys3, 64, ksizes3);
    assert(ret == YOKAN_SUCCESS);
    printf("Listed the following keys:\n");
    koffset = 0;
    for(int i=0; i < 4; i++) {
        if(ksizes3[i] == YOKAN_NO_MORE_KEYS)
            break;
        else if(ksizes3[i] == YOKAN_SIZE_TOO_SMALL)
            printf("\t(buffer too small)\n");
        else {
            printf("\t%.*s\n", (int)ksizes3[i], keys3+koffset);
            koffset += ksizes3[i];
        }
    }

    // ------------------------------------------------------

    char values3[64];
    size_t vsizes3[4];

    ret = yk_list_keyvals_packed(
            db_handle, YOKAN_MODE_INCLUSIVE,
            "shane", 5, "", 0, 4,
            (void*)keys3, 64, ksizes3,
            (void*)values3, 64, vsizes3);

    assert(ret == YOKAN_SUCCESS);
    printf("Listed the following key/value pairs:\n");
    koffset = 0;
    voffset = 0;
    for(int i=0; i < 4; i++) {
        if(ksizes3[i] == YOKAN_NO_MORE_KEYS)
            break;
        printf("\t%.*s => %.*s\n", (int)ksizes3[i], keys3+koffset,
                                   (int)vsizes3[i], values3+voffset);
        koffset += ksizes3[i];
        voffset += vsizes3[i];
    }

    // ------------------------------------------------------

    printf("Erasing all the key/value pairs.\n");
    ret = yk_erase_packed(db_handle, YOKAN_MODE_DEFAULT, 7,
                         (const void*)keys, ksizes);
    assert(ret == YOKAN_SUCCESS);

    // ------------------------------------------------------

    ret = yk_database_handle_release(db_handle);
    assert(ret == YOKAN_SUCCESS);

    ret = yk_client_finalize(client);
    assert(ret == YOKAN_SUCCESS);

    margo_finalize(mid);

    return 0;
}

These functions are the following.

  • yk_put_packed: puts multiple key/value pairs into the database.

  • yk_exists_packed: checks if multiple keys exist in the database.

  • yk_length_packed: retrieves the length of the values associated with a set of keys.

  • yk_get_packed: retrieves the values associated with a set of keys.

  • yk_list_keys_packed: lists a specified number of keys, starting after the specified lower bound.

  • yk_list_keyvals_packed: same as yk_list_keyvals but also returns values.

Note that some of the semantics of these functions differ slighly from that of their _multi counterpart. Functions that retrieve data (yk_get_packed, yk_list_keys_packed, and yk_list_keyvals_packed) do not expect you to provide the size of individual items (apart from the key sizes in yk_get_packed), but require you to provide a total buffer size. The functions will then try to fill this buffer. This is especially useful when keys and values have varying sizes that are not known in advance, as you don’t need to query their sizes first.