Schema Configuration

Overview

To tune TileDB storr performance and storage characteristics, storr.tiledb provides R6 classes that abstracts the complexities when comes to schema configuration.

This is useful for creating storage drivers optimized for different use cases:

Storage-bound workloads: Maximize compression to minimize disk space
Latency-sensitive workloads: Disable compression and tune tile capacity for speed
Balanced scenarios: Apply selective compression strategies to different attributes

What can be customised: compression algorithms, compression levels, tile capacity, cell order, and tile order settings for both the keys and data arrays.

Creating Custom Schemas

Step 1: Initialize Schema Objects

Create a TileDBDriverSchemas object using driver_schemas() constructor:

library(storr.tiledb)

ctx <- new_context()
schemas <- driver_schemas(ctx = ctx)

The schemas object provides access to:

schemas$SchemaKeys: Configuration for the keys/index array
schemas$SchemaData: Configuration for the data/payload array

Each schema has active fields for modifying filter lists and configuration:

dim_namespace, dim_key, attr_hash, attr_expires_at, attr_notes (SchemaKeys)
dim_hash, attr_value (SchemaData)

Step 2: Customize Schemas

Example: High Compression for Storage

For setting up filters, we use tiledb-r client functionality:

# Create a ZSTD filter with high compression level
flt <- tiledb::tiledb_filter("ZSTD", ctx = ctx)
flt <- tiledb::tiledb_filter_set_option(flt, "COMPRESSION_LEVEL", 22)
flt_list <- tiledb::tiledb_filter_list(flt, ctx = ctx)

# Create schemas and apply high compression to data
schemas <- driver_schemas(ctx = ctx)
schemas$SchemaData$attr_value <- flt_list

Example: No Compression for Speed

This is useful only if you wish to apply selectively compression to specific attributes and/or dimensions. Otherwise, prefer compression_level = NULL in storr_tiledb().

# Create schemas without any compression filters
schemas <- driver_schemas(ctx = ctx, none_filter = TRUE)

Example: Mixed Strategy - Fast Keys, Compressed Data

# Start with no compression
schemas <- driver_schemas(ctx = ctx, none_filter = TRUE)

# Apply selective compression to data values only
flt <- tiledb::tiledb_filter("ZSTD", ctx = ctx)
flt <- tiledb::tiledb_filter_set_option(flt, "COMPRESSION_LEVEL", 18)
flt_list <- tiledb::tiledb_filter_list(flt, ctx = ctx)

schemas$SchemaData$attr_value <- flt_list

Note that SchemaKeys remains uncompressed for fast lookups.

Example: Adjust Tile Capacity

Optimize memory usage by adjusting how many cells are stored per tile:

schemas <- driver_schemas(ctx = ctx)

# Lower capacity for memory-constrained environments
schemas$SchemaKeys$capacity <- 5000
schemas$SchemaData$capacity <- 5000

Example: Configure Cell and Tile Order

Tune for specific access patterns:

schemas <- driver_schemas(ctx = ctx)

# Configure for row-major access patterns
schemas$SchemaKeys$cell_order <- "ROW_MAJOR"
schemas$SchemaKeys$tile_order <- "ROW_MAJOR"

schemas$SchemaData$cell_order <- "ROW_MAJOR"
schemas$SchemaData$tile_order <- "ROW_MAJOR"

Step 3: Create Storr with Custom Schemas

Pass the configured schemas to storr_tiledb():

uri <- tempfile()

# Create a storr with custom schemas
sto <- storr_tiledb(uri, init = TRUE, driver_schemas = schemas, context = ctx)

# Use it normally
sto$set("mykey", list(a = 1, b = 2))
sto$get("mykey")
# $a
# [1] 1
# 
# $b
# [1] 2

Alternatively, create a driver first then wrap it with storr:

uri <- tempfile()

# Create driver with custom schemas
dr <- driver_tiledb(uri, init = TRUE, driver_schemas = schemas, context = ctx)

# Create storr from driver
sto <- storr::storr(dr)

# Or instantiate a standard storr that picks the correct serialization (if needed)
sto <- storr_tdb0(uri)

# Use it
sto$set("key1", 123)
sto$get("key1")
# [1] 123

Example: Storage-Optimized Storr

Here’s a complete workflow for a storage-bound use case:

# Set up for maximum compression
ctx <- new_context()
schemas <- driver_schemas(ctx = ctx)

# Create aggressive ZSTD filter (level 22 = high compression)
flt <- tiledb::tiledb_filter("ZSTD", ctx = ctx)
flt <- tiledb::tiledb_filter_set_option(flt, "COMPRESSION_LEVEL", 22)
flt_list <- tiledb::tiledb_filter_list(flt, ctx = ctx)

# Apply to both keys and data for maximum compression
schemas$SchemaKeys$dim_key <- flt_list
schemas$SchemaKeys$dim_namespace <- flt_list
schemas$SchemaKeys$attr_hash <- flt_list
schemas$SchemaData$attr_value <- flt_list

# Create storage-optimized storr
uri_storage <- tempfile()
sto_storage <- storr_tiledb(uri_storage, init = TRUE, 
                            driver_schemas = schemas, context = ctx)

# Store data
sto_storage$set("data1", rep(1:1000, 100))
sto_storage$set("config", list(settings = list(nested = TRUE)))

# Retrieve and verify
sto_storage$list()
# [1] "config" "data1"
sto_storage$get("config")
# $settings
# $settings$nested
# [1] TRUE

Example: Speed-Optimized Storr

Here’s a complete workflow for a latency-sensitive use case:

# Set up for maximum speed (no compression)
ctx <- new_context()
schemas <- driver_schemas(ctx = ctx, none_filter = TRUE)

# Increase tile capacity for fewer I/O operations
schemas$SchemaKeys$capacity <- 20000
schemas$SchemaData$capacity <- 20000

# Create speed-optimized storr
uri_speed <- tempfile()
sto_speed <- storr_tiledb(uri_speed, init = TRUE, 
                          driver_schemas = schemas, context = ctx)

# Store and retrieve rapidly
sto_speed$mset(paste0("key", 1:100), lapply(1:100, function(i) list(id = i)))
keys_to_fetch <- sto_speed$mget(paste0("key", 1:10))

Inspecting Schema Configuration

View the underlying TileDB schema to verify your settings:

# Get TileDB schema object
tdb_schema_keys <- schemas$SchemaKeys$schema()
tdb_schema_data <- schemas$SchemaData$schema()

# Check properties
tiledb::cell_order(tdb_schema_keys)
# [1] "COL_MAJOR"
tiledb::tile_order(tdb_schema_keys)
# [1] "COL_MAJOR"
tiledb::capacity(tdb_schema_keys)
# [1] 20000

Common Configuration Patterns

Default with Data Compression

schemas <- driver_schemas(ctx = ctx)

# Create ZSTD filter
flt <- tiledb::tiledb_filter("ZSTD", ctx = ctx)
flt <- tiledb::tiledb_filter_set_option(flt, "COMPRESSION_LEVEL", 12)
flt_list <- tiledb::tiledb_filter_list(flt, ctx = ctx)

# Apply filter on data
schemas$SchemaData$attr_value <- flt_list

All Compression Disabled

schemas <- driver_schemas(ctx = ctx, none_filter = TRUE)

Asymmetric Optimization (no compression on keys, heavy on data)

# No compression
schemas <- driver_schemas(ctx = ctx, none_filter = TRUE)

# Create ZSTD filter
flt <- tiledb::tiledb_filter("ZSTD", ctx = ctx)
flt <- tiledb::tiledb_filter_set_option(flt, "COMPRESSION_LEVEL", 20)
flt_list <- tiledb::tiledb_filter_list(flt, ctx = ctx)

# Apply filter on data
schemas$SchemaData$attr_value <- flt_list

References

For more information about schemas and compression filter consult the following resources: