library(epwshiftr)
workflow_root <- file.path(tempdir(), "epwshiftr-esg-store")
if (dir.exists(workflow_root)) {
unlink(workflow_root, recursive = TRUE)
}
dir.create(workflow_root, recursive = TRUE, showWarnings = FALSE)
options(
epwshiftr.dir_cache = file.path(workflow_root, "cache")
)
select_cols <- function(x, cols, n = 10L) {
x <- data.table::as.data.table(x)
keep <- intersect(cols, names(x))
if (length(keep)) {
x <- x[, keep, with = FALSE]
}
utils::head(x, n)
}EsgStore is the durable workspace behind the future EPW workflow. The high-level
shift_*() functions carry store paths and manifest IDs for
ordinary workflows, but the lower-level store API is useful when you
need to inspect, update, maintain, or repair a project.
An ESG store has two parts:
Use store methods to change store state. Manual edits to the manifest or moving registered files by hand can break the links that let epwshiftr resume work.
Use a persistent path for real projects. This article uses a temporary path and removes it before each render so the live output is reproducible.
store_path <- file.path(workflow_root, "store")
store <- EsgStore$new(store_path)
store$path
#> [1] "/private/var/folders/8f/t8sk2pps6135xbp47cs8qb2r0000gn/T/RtmptvdKFu/epwshiftr-esg-store/store"
store$manifest
#> [1] "/private/var/folders/8f/t8sk2pps6135xbp47cs8qb2r0000gn/T/RtmptvdKFu/epwshiftr-esg-store/store/manifest.duckdb"
list.files(store$path, all.files = TRUE, no.. = TRUE)
#> [1] "dicts" "downloads" "extracts"
#> [4] "logs" "manifest.duckdb" "manifest.duckdb.wal"
#> [7] "outputs" "queries" "sources"
#> [10] "tmp"EsgStore$new() creates these directories:
queries/: serialized query and result snapshots.dicts/: saved project dictionaries.sources/: raw upstream dictionary or request
sources.downloads/: store-managed NetCDF downloads.extracts/: site-scale extracted Parquet outputs.outputs/: morphing outputs and generated EPW
files.tmp/: temporary working files, including
tmp/downloads/.logs/: downloader background job and daemon logs.The root path and manifest path are not meant to be changed after creation. If you need a new location, create a new store or move the whole store directory as a unit while no R process has it open.
The manifest is a DuckDB database. store$query() is the
safe read path for ad-hoc inspection.
store$query("
SELECT table_name
FROM information_schema.tables
WHERE table_schema = 'main'
ORDER BY table_name
")
#> table_name
#> <char>
#> 1: artifact
#> 2: epw_baseline_summary
#> 3: epw_climate_summary
#> 4: epw_morph_factor
#> 5: epw_morph_plan
#> 6: epw_morph_result
#> 7: epw_output
#> 8: epw_source
#> 9: esg_file
#> 10: esg_query
#> 11: esg_query_dependency
#> 12: esg_query_file
#> 13: esg_query_tag
#> 14: esg_query_update
#> 15: esg_query_update_file
#> 16: extraction_grid_source
#> 17: extraction_plan
#> 18: extraction_result
#> 19: file_catalog
#> 20: query_run
#> 21: shift_run
#> 22: shift_run_case
#> 23: shift_run_event
#> 24: shift_run_job
#> 25: shift_run_step
#> 26: store_meta
#> table_name
#> <char>The core table families are:
store_meta;artifact;esg_query,
esg_query_file, esg_query_update,
esg_query_update_file, esg_query_tag,
esg_query_dependency;esg_file,
file_catalog;extraction_plan,
extraction_result;epw_source,
epw_baseline_summary, epw_climate_summary,
epw_morph_plan, epw_morph_factor,
epw_morph_result, epw_output.Downloader state is stored in the downloader’s own manifest. When you
create it through store$downloader(), that manifest lives
under tmp/downloads/.
store$get_meta("schema_version")
#> [1] "2.8.0"
store$download_layout()
#> $layout
#> [1] "flat"
#>
#> $template
#> NULL
#>
#> $include_version
#> [1] TRUE
#>
#> $collision
#> [1] "error"
#>
#> $missing
#> [1] "fallback"Only the download layout is designed to be changed after store creation. Choose it before large downloads so paths stay stable.
store$set_download_layout(
layout = "drs",
include_version = TRUE,
collision = "error",
missing = "fallback"
)
store$download_layout()
#> $layout
#> [1] "drs"
#>
#> $template
#> NULL
#>
#> $include_version
#> [1] TRUE
#>
#> $collision
#> [1] "error"
#>
#> $missing
#> [1] "fallback"Supported layouts are:
flat: put files directly under
downloads/;dataset: group by ESGF dataset identity;drs: use a CMIP-style DRS path;template: use a custom subdirectory template such as
{source_id}/{experiment_id}/{variable_id}.collision controls what happens when different logical
files map to the same path. missing controls whether
missing layout fields fall back to a safe path or stop the
operation.
add_query() stores a query definition, not just one
collected result. Tracked queries can later be previewed, updated,
tagged, and used to build download plans.
variables <- epw_morph_variables("recommended")
query <- esg_query("https://esgf-data.dkrz.de")$
activity_id("ScenarioMIP")$
experiment_id("ssp585")$
source_id("MPI-ESM1-2-LR")$
variant_label("r1i1p1f1")$
frequency("mon")$
variable_id(variables)$
data_node("esgf.ceda.ac.uk")$
params(table_id = "Amon")$
limit(20L)
query_id <- store$add_query(
query,
label = "singapore-ssp585-amon",
track = TRUE
)
store$tag_query(query_id, c("cmip6", "future-weather", "singapore"))
#> tag_id
#> <char>
#> 1: 9913160dec9459da162a460e13c761b5504a187f3924b51d9ccf9aa0850007a3
#> 2: c18dfc3756029099f2172bfcaf5f881c3e8e90790066bc339ac0f4141e0b1e71
#> 3: 4bf1f26772ef5c8a9d84fbf950e3fa819c6439789a9ced967189215dba758f4b
#> query_id
#> <char>
#> 1: 44182b5adba905c4236489a16fd0e83663c732b4cb31c66ab3ecda5577cf7152
#> 2: 44182b5adba905c4236489a16fd0e83663c732b4cb31c66ab3ecda5577cf7152
#> 3: 44182b5adba905c4236489a16fd0e83663c732b4cb31c66ab3ecda5577cf7152
#> tag created_at
#> <char> <POSc>
#> 1: cmip6 2026-07-25 17:08:31
#> 2: future-weather 2026-07-25 17:08:31
#> 3: singapore 2026-07-25 17:08:31
select_cols(store$queries(), c(
"query_id", "label", "index_node", "tracked", "created_at"
))
#> query_id
#> <char>
#> 1: 44182b5adba905c4236489a16fd0e83663c732b4cb31c66ab3ecda5577cf7152
#> label index_node tracked created_at
#> <char> <char> <lgcl> <POSc>
#> 1: singapore-ssp585-amon https://esgf-data.dkrz.de TRUE 2026-07-25 17:08:31
store$query_tags(query_id)
#> tag_id
#> <char>
#> 1: 9913160dec9459da162a460e13c761b5504a187f3924b51d9ccf9aa0850007a3
#> 2: c18dfc3756029099f2172bfcaf5f881c3e8e90790066bc339ac0f4141e0b1e71
#> 3: 4bf1f26772ef5c8a9d84fbf950e3fa819c6439789a9ced967189215dba758f4b
#> query_id
#> <char>
#> 1: 44182b5adba905c4236489a16fd0e83663c732b4cb31c66ab3ecda5577cf7152
#> 2: 44182b5adba905c4236489a16fd0e83663c732b4cb31c66ab3ecda5577cf7152
#> 3: 44182b5adba905c4236489a16fd0e83663c732b4cb31c66ab3ecda5577cf7152
#> tag created_at
#> <char> <POSc>
#> 1: cmip6 2026-07-25 17:08:31
#> 2: future-weather 2026-07-25 17:08:31
#> 3: singapore 2026-07-25 17:08:31The query snapshot is written under queries/, while the
query identity and tracking state are recorded in the manifest.
list.files(file.path(store$path, "queries"), recursive = TRUE)
#> [1] "query-44182b5adba905c4236489a16fd0e83663c732b4cb31c66ab3ecda5577cf7152.json"
store$query("
SELECT query_id, label, tracked, query_file
FROM esg_query
")
#> query_id
#> <char>
#> 1: 44182b5adba905c4236489a16fd0e83663c732b4cb31c66ab3ecda5577cf7152
#> label tracked
#> <char> <lgcl>
#> 1: singapore-ssp585-amon TRUE
#> query_file
#> <char>
#> 1: queries/query-44182b5adba905c4236489a16fd0e83663c732b4cb31c66ab3ecda5577cf7152.jsonPreviewing asks ESGF for current File records and compares them with the store’s current links. It does not modify the store.
The remaining chunks are live ESGF operations. They are displayed but not executed when the package documentation is precompiled, so rendering remains deterministic and does not depend on the availability of a remote index node. Run them interactively against your store when following the workflow.
preview <- store$preview_update_queries(
query_id = query_id,
detail = TRUE,
fields = "*",
all = FALSE,
limit = TRUE
)
preview$summary
select_cols(preview$changes, c(
"change_type", "file_key", "previous_status", "current_status",
"current_version", "data_node_changed", "url_changed"
), n = 12L)Updating writes the current File identities, file metadata, query-file links, and an update record.
updated <- store$update_queries(
query_id = query_id,
fields = "*",
all = FALSE,
limit = TRUE
)
select_cols(updated, c(
"query_id", "update_id", "status", "file_total",
"new_count", "changed_count", "stale_count"
))
select_cols(store$query_files(query_id), c(
"query_id", "file_key", "status", "filename", "variable_id",
"data_node", "datetime_start", "datetime_end"
), n = 12L)query_updates() and query_changes() make
update history reviewable.
updates <- store$query_updates(query_id)
select_cols(updates, c(
"update_id", "query_id", "status", "file_total",
"new_count", "changed_count", "completed_at"
))
latest_update <- store$query_updates(query_id, latest = TRUE)
select_cols(
store$query_changes(update_id = latest_update$update_id),
c("change_type", "file_key", "current_status", "current_version"),
n = 12L
)The store owns downloader paths for store-managed downloads. Prefer
store$downloader() so completed tasks can be synchronized
back into esg_file, file_catalog, and
artifact.
downloader <- store$downloader(
n_workers = 2L,
retries = 1L,
timeout = 120L,
resource_policy = list(min_free_space = 0)
)
download_preview <- store$download_preflight(
query_id = query_id,
downloader = downloader,
replica = "current",
service = "HTTPServer",
probe = FALSE,
fields = "*",
all = FALSE,
limit = TRUE
)
download_preview$summary
select_cols(download_preview$files, c(
"file_key", "filename", "variable_id", "size",
"local_path", "status"
), n = 12L)
select_cols(download_preview$candidates, c(
"filename", "service", "data_node", "size", "url"
), n = 12L)To enqueue or run downloads, use download_query() or
download_files(). This article stops at preflight because
CMIP NetCDF files are large.
session_id <- store$download_query(
query_id = query_id,
downloader = downloader,
replica = "current",
service = "HTTPServer",
run = TRUE,
progress = TRUE
)
store$sync_downloads(downloader)Extraction plans connect query files to a site, time range,
variables, and a nearest-grid rule. Planning is metadata-only;
extraction reads OPeNDAP or local NetCDF data and writes Parquet outputs
under extracts/.
region <- store$plan_region(
query_id = query_id,
lon = 103.98,
lat = 1.37,
time = c("2060-01-01T00:00:00Z", "2060-12-31T23:59:59Z"),
site_id = "SIN",
variable_id = variables,
method = "nearest"
)
select_cols(region, c(
"plan_id", "query_id", "file_key", "site_id", "variable_id",
"lon", "lat", "time_start", "time_stop", "status"
), n = 12L)
select_cols(store$coverage(plan_id = region$plan_id), c(
"plan_id", "site_id", "variable_id", "status",
"complete", "output_rows", "output_time_count"
), n = 12L)Use store$extract() when you want to execute the plan.
With fallback = "auto", extraction tries OPeNDAP first and
may fall back to local or HTTP downloads when needed.
Artifacts connect files on disk to manifest records. Query snapshots, dictionaries, NetCDF downloads, extracted Parquet files, morphing intermediates, and EPW outputs all use the same artifact table.
store$validate()
store$query("
SELECT kind, role, project, relative_path, status, query_id, file_key
FROM artifact
ORDER BY kind, relative_path
")When you have an artifact_id, use
artifact_path() to resolve the absolute path instead of
assembling paths by hand.
Maintenance methods are intentionally preview-first. Use validation and dry-run cleanup before changing anything.
report <- store$storage_report(detail = TRUE)
report$summary
select_cols(report$downloads, c("relative_path", "size", "mtime"), n = 8L)
select_cols(report$registered, c("source", "file_key", "relative_path", "size"), n = 8L)
validation <- store$validate_files(
query_id = query_id,
checksum = FALSE,
layout = TRUE
)
validation$summary
select_cols(validation$actions, c(
"action", "file_key", "artifact_id", "from_path", "to_path", "reason"
), n = 10L)
store$repair_files(validation$actions, dry_run = TRUE)
store$cleanup_downloads(
scope = "tmp",
older_than = 7 * 24 * 3600,
dry_run = TRUE
)Checksum validation is useful, but it must read files and can be expensive on a large store. Layout validation is cheaper and catches many broken local-path or artifact-record problems.
Store schemas are not migrated between incompatible development versions. If a package update rejects an older manifest, recreate the default store with:
The previous store is moved to a timestamped sibling, so its queries, downloads, extracts, generated EPWs, and run history remain available for manual recovery. Permanent removal requires both safeguards explicitly:
store_reset(backup = FALSE, force = TRUE)Close open EsgStore objects and stop active jobs before
resetting. To retain the old store in place, pass a new directory
through the store argument or change
epwshiftr.dir_store instead.
shift_*() for the normal end-to-end future EPW
workflow.EsgQuery when you need to understand live
Dataset/File/Aggregation results before they enter a store.EsgStore when you need durable state, query
tracking, file catalog inspection, extraction planning, artifact
validation, or maintenance.Downloader when you need transfer execution
details: sessions, tasks, retries, background jobs, daemon mode, and
node history.EpwMorpher when you need lower-level control over
climate summaries, morphing factors, diagnostics, and EPW writing.