vignettes/articles/esgf-query-results.Rmd
esgf-query-results.Rmd
library(epwshiftr)
workflow_root <- file.path(tempdir(), "epwshiftr-esgf-query-results")
if (dir.exists(workflow_root)) {
unlink(workflow_root, recursive = TRUE)
}
dir.create(workflow_root, recursive = TRUE, showWarnings = FALSE)
options(
epwshiftr.dir_cache = file.path(workflow_root, "cache")
)
select_cols <- function(x, cols, n = 10L) {
x <- data.table::as.data.table(x)
keep <- intersect(cols, names(x))
if (length(keep)) {
x <- x[, keep, with = FALSE]
}
utils::head(x, n)
}
result_table <- function(x, cols, n = 10L, formatted = TRUE) {
if (!x$count()) {
return(data.frame(note = "No records returned by this live query."))
}
select_cols(x$to_data_table(formatted = formatted), cols, n = n)
}This article explains the live ESGF query layer used by Create Future EPW Files. The main
workflow starts with shift_request(), but that request is
translated into lower-level EsgQuery,
EsgResultDataset, EsgResultFile, and
EsgResultAggregation objects.
Use this layer when you need to audit the exact ESGF URL, inspect an
index node, compare Dataset matches, decide whether File or Aggregation
records are available, or debug why a shift_*() request is
too broad or empty.
esg_query() creates an EsgQuery object. It
stores the index node and query parameters; it does not contact ESGF
until you call a method such as $count(),
$collect(), or one of the listing helpers.
The query below uses the same light CMIP6 selection as the main
future EPW article: one ScenarioMIP source, one scenario, one member,
monthly Amon data, the CEDA data node, and the current
recommended morphing variables.
index_node <- "https://esgf-data.dkrz.de"
variables <- epw_morph_variables("recommended")
dataset_fields <- c(
"id", "source_id", "experiment_id", "variant_label",
"frequency", "table_id", "variable_id", "data_node",
"number_of_files", "number_of_aggregations", "access"
)
new_dataset_query <- function() {
esg_query(index_node)$
activity_id("ScenarioMIP")$
experiment_id("ssp585")$
source_id("MPI-ESM1-2-LR")$
variant_label("r1i1p1f1")$
frequency("mon")$
variable_id(variables)$
data_node("esgf.ceda.ac.uk")$
params(table_id = "Amon")$
fields(dataset_fields)$
limit(20L)
}
query <- new_dataset_query()
query$state(null = FALSE)
#> $index_node
#> [1] "https://esgf-data.dkrz.de"
#>
#> $parameter
#> $parameter$project
#> =CMIP6
#>
#> $parameter$activity_id
#> =ScenarioMIP
#>
#> $parameter$experiment_id
#> =ssp585
#>
#> $parameter$source_id
#> =MPI-ESM1-2-LR
#>
#> $parameter$variable_id
#> =tas,hurs,psl,rlds,rsds,sfcWind,clt
#>
#> $parameter$frequency
#> =mon
#>
#> $parameter$variant_label
#> =r1i1p1f1
#>
#> $parameter$data_node
#> =esgf.ceda.ac.uk
#>
#> $parameter$fields
#> =id,source_id,experiment_id,variant_label,frequency,table_id,variable_id,data_node,number_of_files,number_of_aggregations,access
#>
#> $parameter$type
#> =Dataset
#>
#> $parameter$offset
#> =0
#>
#> $parameter$distrib
#> =true
#>
#> $parameter$limit
#> =20
#>
#> $parameter$format
#> =application%2Fsolr%2Bjson
#>
#> $parameter$table_id
#> =Amon
query$url()
#> [1] "https://esgf-data.dkrz.de/esg-search/search?project=CMIP6&activity_id=ScenarioMIP&experiment_id=ssp585&source_id=MPI-ESM1-2-LR&variable_id=tas,hurs,psl,rlds,rsds,sfcWind,clt&frequency=mon&variant_label=r1i1p1f1&data_node=esgf.ceda.ac.uk&fields=id,source_id,experiment_id,variant_label,frequency,table_id,variable_id,data_node,number_of_files,number_of_aggregations,access&type=Dataset&offset=0&distrib=true&limit=20&format=application%2Fsolr%2Bjson&table_id=Amon"The small new_dataset_query() factory is deliberate.
EsgQuery methods mutate the query object, so the range
examples below each start from a fresh Dataset query. That keeps
datetime_range(), timestamp_range(), and
version_range() from leaking into the later
Dataset/File/Aggregation examples, while keeping the shared base request
in one place.
Use state() to inspect the request as epwshiftr
understands it. Use url() when you need the exact Solr
request sent to the index node.
count() is the first live request in this article. It
asks the index node how many Dataset records match the query.
query$count()
#> [1] 7EsgQuery has three range helpers that add Solr
query= constraints to the Dataset search.
datetime_range(start, stop) filters by dataset temporal
coverage overlap. It maps to ESGF datetime_start and
datetime_stop constraints.timestamp_range(from, to) filters by the Solr index
timestamp, which is useful when auditing recently indexed or recently
changed records.version_range(min, max) filters the numeric ESGF
version field. CMIP-style versions usually look like
YYYYMMDD, such as 20190710.For the future EPW workflow, use datetime_range() only
to narrow Dataset matches. After collecting File or Aggregation records,
use filter_time() for file-level coverage. CMIP files often
cover multi-year ranges, and File-layer filtering is easier to
audit.
All three helpers use epwshiftr’s Solr date parser internally. You
can inspect the parser directly with solr_date() when you
want to see exactly how a value will be normalized before it is placed
into a query.
solr_inputs <- c(
"2060",
"2060-06",
"20600615",
"2060-06-15T12:30:45Z",
"2060-06-15T20:30:45+08:00",
"NOW/DAY-1YEAR+6MONTHS",
"2060-01-01T00:00:00Z+1MONTH",
"[2055 TO 2075]",
"{2055 TO 2075]",
"[* TO 2060]"
)
solr_parsed <- lapply(solr_inputs, solr_date)
data.frame(
input = solr_inputs,
iso = vapply(solr_parsed, format, character(1)),
num = suppressWarnings(vapply(
solr_parsed,
function(x) format(x, as = "num"),
character(1)
)),
is_solr_date = vapply(solr_parsed, is.solr_date, logical(1))
)
#> input iso
#> 1 2060 2060-01-01T00:00:00Z
#> 2 2060-06 2060-06-01T00:00:00Z
#> 3 20600615 2060-06-15T00:00:00Z
#> 4 2060-06-15T12:30:45Z 2060-06-15T12:30:45Z
#> 5 2060-06-15T20:30:45+08:00 2060-06-15T12:30:45Z
#> 6 NOW/DAY-1YEAR+6MONTHS NOW/DAY-1YEAR+6MONTHS
#> 7 2060-01-01T00:00:00Z+1MONTH 2060-01-01T00:00:00Z+1MONTH
#> 8 [2055 TO 2075] [2055-01-01T00:00:00Z TO 2075-01-01T00:00:00Z]
#> 9 {2055 TO 2075] {2055-01-01T00:00:00Z TO 2075-01-01T00:00:00Z]
#> 10 [* TO 2060] [* TO 2060-01-01T00:00:00Z]
#> num is_solr_date
#> 1 20600101 TRUE
#> 2 20600601 TRUE
#> 3 20600615 TRUE
#> 4 20600615 TRUE
#> 5 20600615 TRUE
#> 6 NOW/DAY-1YEAR+6MONTHS TRUE
#> 7 20600101+1MONTH TRUE
#> 8 [20550101 TO 20750101] TRUE
#> 9 {20550101 TO 20750101] TRUE
#> 10 [* TO 20600101] TRUEThe parser accepts Date and UTC POSIXct
inputs too. POSIXct values must already be UTC so that the
query does not silently depend on the local timezone.
typed_inputs <- list(
date = as.Date("2060-06-15"),
posixct_utc = as.POSIXct("2060-06-15 12:30:45", tz = "UTC")
)
data.frame(
input_type = names(typed_inputs),
iso = vapply(
typed_inputs,
function(x) format(solr_date(x)),
character(1)
)
)
#> input_type iso
#> date date 2060-06-15T00:00:00Z
#> posixct_utc posixct_utc 2060-06-15T12:30:45Z
coverage_query <- new_dataset_query()$
datetime_range(
start = "2060",
stop = "2060-12-31T23:59:59Z"
)
coverage_query$datetime_range()
#> $start
#> [* TO 2060-01-01T00:00:00Z]
#>
#> $stop
#> [2060-12-31T23:59:59Z TO *]
coverage_query$url()
#> [1] "https://esgf-data.dkrz.de/esg-search/search?project=CMIP6&activity_id=ScenarioMIP&experiment_id=ssp585&source_id=MPI-ESM1-2-LR&variable_id=tas,hurs,psl,rlds,rsds,sfcWind,clt&frequency=mon&variant_label=r1i1p1f1&data_node=esgf.ceda.ac.uk&fields=id,source_id,experiment_id,variant_label,frequency,table_id,variable_id,data_node,number_of_files,number_of_aggregations,access&type=Dataset&offset=0&distrib=true&limit=20&format=application%2Fsolr%2Bjson&table_id=Amon&query=datetime_start%3A%5B%2A%20TO%202060-01-01T00%3A00%3A00Z%5D%20AND%20datetime_stop%3A%5B2060-12-31T23%3A59%3A59Z%20TO%20%2A%5D"
coverage_query$count()
#> [1] 7datetime_range() accepts complete ISO datetimes,
simplified dates such as "2060" or "2060-06",
Solr Date Math such as "NOW-10YEARS", and complete Solr
range expressions. The start boundary keeps datasets whose
coverage starts no later than the requested boundary; the
stop boundary keeps datasets whose coverage ends no earlier
than the requested boundary.
custom_coverage_query <- new_dataset_query()$
datetime_range(
start = "[2055 TO 2075]",
stop = "NOW+100YEARS"
)
custom_coverage_query$datetime_range()
#> $start
#> [2055-01-01T00:00:00Z TO 2075-01-01T00:00:00Z]
#>
#> $stop
#> [NOW+100YEARS TO *]
custom_coverage_query$url()
#> [1] "https://esgf-data.dkrz.de/esg-search/search?project=CMIP6&activity_id=ScenarioMIP&experiment_id=ssp585&source_id=MPI-ESM1-2-LR&variable_id=tas,hurs,psl,rlds,rsds,sfcWind,clt&frequency=mon&variant_label=r1i1p1f1&data_node=esgf.ceda.ac.uk&fields=id,source_id,experiment_id,variant_label,frequency,table_id,variable_id,data_node,number_of_files,number_of_aggregations,access&type=Dataset&offset=0&distrib=true&limit=20&format=application%2Fsolr%2Bjson&table_id=Amon&query=datetime_start%3A%5B2055-01-01T00%3A00%3A00Z%20TO%202075-01-01T00%3A00%3A00Z%5D%20AND%20datetime_stop%3A%5BNOW%2B100YEARS%20TO%20%2A%5D"Use full range expressions only when you intentionally need raw Solr
range control. For ordinary workflow windows, separate
start and stop boundaries are easier to
read.
timestamp_query <- new_dataset_query()$
timestamp_range(
from = "NOW/YEAR-10YEARS",
to = "NOW"
)
timestamp_query$timestamp_range()
#> $from
#> NOW/YEAR-10YEARS
#>
#> $to
#> NOW
timestamp_query$url()
#> [1] "https://esgf-data.dkrz.de/esg-search/search?project=CMIP6&activity_id=ScenarioMIP&experiment_id=ssp585&source_id=MPI-ESM1-2-LR&variable_id=tas,hurs,psl,rlds,rsds,sfcWind,clt&frequency=mon&variant_label=r1i1p1f1&data_node=esgf.ceda.ac.uk&fields=id,source_id,experiment_id,variant_label,frequency,table_id,variable_id,data_node,number_of_files,number_of_aggregations,access&type=Dataset&offset=0&distrib=true&limit=20&format=application%2Fsolr%2Bjson&table_id=Amon&query=_timestamp%3A%5BNOW%2FYEAR-10YEARS%20TO%20NOW%5D"
timestamp_query$count()
#> [1] 7Use timestamp_range() when the question is about index
freshness, not climate time coverage. It accepts point-like dates and
Solr Date Math boundaries. Unlike datetime_range(), it does
not accept a full [... TO ...] range expression as one
argument; pass from and to separately.
version_query <- new_dataset_query()$
version_range(
min = "2019-07",
max = "20190831"
)
version_query$version_range()
#> $min
#> [2019-07-01T00:00:00Z TO *]
#>
#> $max
#> [* TO 2019-08-31T00:00:00Z]
version_query$url()
#> [1] "https://esgf-data.dkrz.de/esg-search/search?project=CMIP6&activity_id=ScenarioMIP&experiment_id=ssp585&source_id=MPI-ESM1-2-LR&variable_id=tas,hurs,psl,rlds,rsds,sfcWind,clt&frequency=mon&variant_label=r1i1p1f1&data_node=esgf.ceda.ac.uk&fields=id,source_id,experiment_id,variant_label,frequency,table_id,variable_id,data_node,number_of_files,number_of_aggregations,access&type=Dataset&offset=0&distrib=true&limit=20&format=application%2Fsolr%2Bjson&table_id=Amon&query=version%3A%5B20190701%20TO%20%2A%5D%20AND%20version%3A%5B%2A%20TO%2020190831%5D"
version_query$count()
#> [1] 7Use version_range() when you need to select ESGF
publication versions. It is a numeric comparison, so use version-like
boundaries such as "20190701" or simplified dates such as
"2019-07". It does not support Solr Date Math.
Pass NULL to clear an existing range boundary, for
example
query$datetime_range(start = NULL, stop = NULL).
Before collecting results from an unfamiliar index node, inspect what the node can answer.
facets <- query$list_facets()
fields <- query$list_fields()
shards <- query$list_shards()
values <- query$list_values(c("activity_id", "experiment_id", "table_id"))
utils::head(facets, 20)
#> [1] "Conventions" "access" "activity"
#> [4] "activity_drs" "activity_id" "amodell"
#> [7] "branch_method" "cera_acronym" "cf_standard_name"
#> [10] "cmor_table" "contact" "creation_date"
#> [13] "data_node" "data_specs_version" "data_structure"
#> [16] "data_type" "dataset_category" "dataset_status"
#> [19] "dataset_version_number" "datetime_end"
utils::head(fields, 20)
#> [1] "id" "version"
#> [3] "access" "activity_drs"
#> [5] "activity_id" "cf_standard_name"
#> [7] "citation_url" "data_node"
#> [9] "data_specs_version" "dataset_id_template_"
#> [11] "datetime_start" "datetime_stop"
#> [13] "directory_format_template_" "east_degrees"
#> [15] "experiment_id" "experiment_title"
#> [17] "frequency" "further_info_url"
#> [19] "geo" "geo_units"
utils::head(shards, 8)
#> [1] "solr-slave:8983/solr/datasets"
#> [2] "solr-replica-ipsl:8983/solr/datasets"
#> [3] "solr-replica-ceda:8983/solr/datasets"
#> [4] "solr-replica-nci:8983/solr/datasets"
#> [5] "solr-replica-gfdl:8983/solr/datasets"
lapply(values, utils::head, 8)
#> $activity_id
#> AerChemMIP C4MIP CDRMIP CFMIP CMIP DAMIP DCPP
#> 205565 41323 14686 34092 404195 181406 1240354
#> FAFMIP
#> 14424
#>
#> $experiment_id
#> 1pctCO2 1pctCO2-4xext 1pctCO2-bgc 1pctCO2-cdr
#> 33014 61 6933 1224
#> 1pctCO2-rad 1pctCO2Ndep 1pctCO2Ndep-bgc 1pctCO2to4x-withism
#> 4760 845 925 90
#>
#> $table_id
#> 3hr 6hrLev 6hrPlev 6hrPlevPt AERday AERhr AERmon AERmonZ
#> 27603 4999 44394 26243 18283 463 139840 24448list_shards() is about distributed search indexes. It is
not the same thing as data_node: a shard answers search
requests, while a data node hosts files and service URLs.
The dictionary article explains local value validation before a query is sent. The listing helpers here explain what the live index node currently exposes. Use both when you need to understand whether an empty result is a local request problem or an index-node/data-node problem.
Dataset records answer “which logical ESGF datasets match this request?” They are not NetCDF files yet. A Dataset record is the parent identity used to collect File or Aggregation child records.
datasets <- query$collect(type = "Dataset", all = FALSE, limit = TRUE)
datasets$count()
#> [1] 7
result_table(datasets, c(
"id", "source_id", "experiment_id", "variant_label",
"table_id", "variable_id", "data_node", "number_of_files",
"number_of_aggregations", "access"
))
#> id
#> <char>
#> 1: CMIP6.ScenarioMIP.MPI-M.MPI-ESM1-2-LR.ssp585.r1i1p1f1.Amon.clt.gn.v20190710|esgf.ceda.ac.uk
#> 2: CMIP6.ScenarioMIP.MPI-M.MPI-ESM1-2-LR.ssp585.r1i1p1f1.Amon.hurs.gn.v20190815|esgf.ceda.ac.uk
#> 3: CMIP6.ScenarioMIP.MPI-M.MPI-ESM1-2-LR.ssp585.r1i1p1f1.Amon.psl.gn.v20190710|esgf.ceda.ac.uk
#> 4: CMIP6.ScenarioMIP.MPI-M.MPI-ESM1-2-LR.ssp585.r1i1p1f1.Amon.rlds.gn.v20190710|esgf.ceda.ac.uk
#> 5: CMIP6.ScenarioMIP.MPI-M.MPI-ESM1-2-LR.ssp585.r1i1p1f1.Amon.rsds.gn.v20190710|esgf.ceda.ac.uk
#> 6: CMIP6.ScenarioMIP.MPI-M.MPI-ESM1-2-LR.ssp585.r1i1p1f1.Amon.sfcWind.gn.v20190710|esgf.ceda.ac.uk
#> 7: CMIP6.ScenarioMIP.MPI-M.MPI-ESM1-2-LR.ssp585.r1i1p1f1.Amon.tas.gn.v20190710|esgf.ceda.ac.uk
#> source_id experiment_id variant_label table_id variable_id
#> <char> <char> <char> <char> <char>
#> 1: MPI-ESM1-2-LR ssp585 r1i1p1f1 Amon clt
#> 2: MPI-ESM1-2-LR ssp585 r1i1p1f1 Amon hurs
#> 3: MPI-ESM1-2-LR ssp585 r1i1p1f1 Amon psl
#> 4: MPI-ESM1-2-LR ssp585 r1i1p1f1 Amon rlds
#> 5: MPI-ESM1-2-LR ssp585 r1i1p1f1 Amon rsds
#> 6: MPI-ESM1-2-LR ssp585 r1i1p1f1 Amon sfcWind
#> 7: MPI-ESM1-2-LR ssp585 r1i1p1f1 Amon tas
#> data_node number_of_files access
#> <char> <int> <list>
#> 1: esgf.ceda.ac.uk 5 HTTPServer,OPENDAP
#> 2: esgf.ceda.ac.uk 5 HTTPServer,OPENDAP
#> 3: esgf.ceda.ac.uk 5 HTTPServer,OPENDAP
#> 4: esgf.ceda.ac.uk 5 HTTPServer,OPENDAP
#> 5: esgf.ceda.ac.uk 5 HTTPServer,OPENDAP
#> 6: esgf.ceda.ac.uk 5 HTTPServer,OPENDAP
#> 7: esgf.ceda.ac.uk 5 HTTPServer,OPENDAPUse Dataset inspection before asking for all child files. If the
table is too broad, tighten request facets such as
source_id, variant_label,
table_id, or data_node. If the table is empty,
use the metadata helpers above and the troubleshooting article.
Result objects are local containers after collection.
$slice() and $filter() operate on the records
already in memory and keep selection provenance.
first_datasets <- datasets$slice(seq_len(min(3L, datasets$count())))
result_table(first_datasets, c("id", "variable_id", "data_node", "access"))
#> id
#> <char>
#> 1: CMIP6.ScenarioMIP.MPI-M.MPI-ESM1-2-LR.ssp585.r1i1p1f1.Amon.clt.gn.v20190710|esgf.ceda.ac.uk
#> 2: CMIP6.ScenarioMIP.MPI-M.MPI-ESM1-2-LR.ssp585.r1i1p1f1.Amon.hurs.gn.v20190815|esgf.ceda.ac.uk
#> 3: CMIP6.ScenarioMIP.MPI-M.MPI-ESM1-2-LR.ssp585.r1i1p1f1.Amon.psl.gn.v20190710|esgf.ceda.ac.uk
#> variable_id data_node access
#> <char> <char> <list>
#> 1: clt esgf.ceda.ac.uk HTTPServer,OPENDAP
#> 2: hurs esgf.ceda.ac.uk HTTPServer,OPENDAP
#> 3: psl esgf.ceda.ac.uk HTTPServer,OPENDAP
tas_datasets <- datasets$filter(function(x) x$variable_id == "tas")
result_table(tas_datasets, c("id", "variable_id", "number_of_files", "access"))
#> id
#> <char>
#> 1: CMIP6.ScenarioMIP.MPI-M.MPI-ESM1-2-LR.ssp585.r1i1p1f1.Amon.tas.gn.v20190710|esgf.ceda.ac.uk
#> variable_id number_of_files access
#> <char> <int> <list>
#> 1: tas 5 HTTPServer,OPENDAP
tas_datasets$selection()
#> $source_count
#> [1] 7
#>
#> $source_num_found
#> [1] 7
#>
#> $source_indices
#> [1] 7File records identify concrete NetCDF files and service URLs. They are the dependable route for store-backed downloads and OPeNDAP-first extraction.
files <- datasets$collect(
type = "File",
fields = "*",
all = TRUE,
limit = NULL
)
files$count()
#> [1] 35
result_table(files, c(
"filename", "variable_id", "data_node", "datetime_start",
"datetime_end", "size", "url_opendap", "url_download"
), n = 12L)
#> filename variable_id
#> <char> <char>
#> 1: clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_201501-203412.nc clt
#> 2: clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_203501-205412.nc clt
#> 3: clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc clt
#> 4: clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_207501-209412.nc clt
#> 5: clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_209501-210012.nc clt
#> 6: hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_201501-203412.nc hurs
#> 7: hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_203501-205412.nc hurs
#> 8: hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc hurs
#> 9: hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_207501-209412.nc hurs
#> 10: hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_209501-210012.nc hurs
#> 11: psl_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_201501-203412.nc psl
#> 12: psl_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_203501-205412.nc psl
#> data_node size
#> <char> <units>
#> 1: esgf.ceda.ac.uk 11.655127 [MiB]
#> 2: esgf.ceda.ac.uk 11.661527 [MiB]
#> 3: esgf.ceda.ac.uk 11.652859 [MiB]
#> 4: esgf.ceda.ac.uk 11.648442 [MiB]
#> 5: esgf.ceda.ac.uk 3.531520 [MiB]
#> 6: esgf.ceda.ac.uk 10.593455 [MiB]
#> 7: esgf.ceda.ac.uk 10.588588 [MiB]
#> 8: esgf.ceda.ac.uk 10.582312 [MiB]
#> 9: esgf.ceda.ac.uk 10.572694 [MiB]
#> 10: esgf.ceda.ac.uk 3.215569 [MiB]
#> 11: esgf.ceda.ac.uk 7.863791 [MiB]
#> 12: esgf.ceda.ac.uk 7.866961 [MiB]
#> url_opendap
#> <char>
#> 1: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/clt/gn/v20190710/clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_201501-203412.nc
#> 2: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/clt/gn/v20190710/clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_203501-205412.nc
#> 3: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/clt/gn/v20190710/clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc
#> 4: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/clt/gn/v20190710/clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_207501-209412.nc
#> 5: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/clt/gn/v20190710/clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_209501-210012.nc
#> 6: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/hurs/gn/v20190815/hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_201501-203412.nc
#> 7: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/hurs/gn/v20190815/hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_203501-205412.nc
#> 8: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/hurs/gn/v20190815/hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc
#> 9: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/hurs/gn/v20190815/hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_207501-209412.nc
#> 10: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/hurs/gn/v20190815/hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_209501-210012.nc
#> 11: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/psl/gn/v20190710/psl_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_201501-203412.nc
#> 12: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/psl/gn/v20190710/psl_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_203501-205412.nc
#> url_download
#> <char>
#> 1: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/clt/gn/v20190710/clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_201501-203412.nc
#> 2: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/clt/gn/v20190710/clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_203501-205412.nc
#> 3: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/clt/gn/v20190710/clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc
#> 4: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/clt/gn/v20190710/clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_207501-209412.nc
#> 5: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/clt/gn/v20190710/clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_209501-210012.nc
#> 6: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/hurs/gn/v20190815/hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_201501-203412.nc
#> 7: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/hurs/gn/v20190815/hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_203501-205412.nc
#> 8: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/hurs/gn/v20190815/hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc
#> 9: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/hurs/gn/v20190815/hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_207501-209412.nc
#> 10: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/hurs/gn/v20190815/hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_209501-210012.nc
#> 11: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/psl/gn/v20190710/psl_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_201501-203412.nc
#> 12: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/psl/gn/v20190710/psl_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_203501-205412.ncfilter_time(method = "drs") parses CMIP-style filename
and DRS time ranges. It is a fast planning filter and is the right
default for raw articles and store workflows.
files_2060 <- files$filter_time(
"2060-01-01T00:00:00Z",
"2060-12-31T23:59:59Z",
method = "drs"
)
files_2060$count()
#> [1] 7
result_table(files_2060, c(
"filename", "variable_id", "datetime_start", "datetime_end",
"data_node", "url_opendap", "url_download"
), n = 12L)
#> filename variable_id
#> <char> <char>
#> 1: clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc clt
#> 2: hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc hurs
#> 3: psl_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc psl
#> 4: rlds_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc rlds
#> 5: rsds_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc rsds
#> 6: sfcWind_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc sfcWind
#> 7: tas_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc tas
#> datetime_start datetime_end data_node
#> <char> <char> <char>
#> 1: 2055-01-01T00:00:00Z 2074-12-31T23:59:59Z esgf.ceda.ac.uk
#> 2: 2055-01-01T00:00:00Z 2074-12-31T23:59:59Z esgf.ceda.ac.uk
#> 3: 2055-01-01T00:00:00Z 2074-12-31T23:59:59Z esgf.ceda.ac.uk
#> 4: 2055-01-01T00:00:00Z 2074-12-31T23:59:59Z esgf.ceda.ac.uk
#> 5: 2055-01-01T00:00:00Z 2074-12-31T23:59:59Z esgf.ceda.ac.uk
#> 6: 2055-01-01T00:00:00Z 2074-12-31T23:59:59Z esgf.ceda.ac.uk
#> 7: 2055-01-01T00:00:00Z 2074-12-31T23:59:59Z esgf.ceda.ac.uk
#> url_opendap
#> <char>
#> 1: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/clt/gn/v20190710/clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc
#> 2: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/hurs/gn/v20190815/hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc
#> 3: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/psl/gn/v20190710/psl_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc
#> 4: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/rlds/gn/v20190710/rlds_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc
#> 5: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/rsds/gn/v20190710/rsds_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc
#> 6: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/sfcWind/gn/v20190710/sfcWind_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc
#> 7: https://esgf.ceda.ac.uk/thredds/dodsC/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/tas/gn/v20190710/tas_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc
#> url_download
#> <char>
#> 1: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/clt/gn/v20190710/clt_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc
#> 2: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/hurs/gn/v20190815/hurs_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc
#> 3: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/psl/gn/v20190710/psl_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc
#> 4: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/rlds/gn/v20190710/rlds_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc
#> 5: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/rsds/gn/v20190710/rsds_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc
#> 6: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/sfcWind/gn/v20190710/sfcWind_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.nc
#> 7: https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/ScenarioMIP/MPI-M/MPI-ESM1-2-LR/ssp585/r1i1p1f1/Amon/tas/gn/v20190710/tas_Amon_MPI-ESM1-2-LR_ssp585_r1i1p1f1_gn_205501-207412.ncUse filter_time(method = "opendap") only when
filename-derived time coverage is not enough and you are willing to open
remote datasets to read their time axes. That is more precise for
irregular data, but it is slower and more sensitive to remote OPeNDAP
availability.
Aggregation records are optional. Some Dataset records and index nodes expose them, and some do not. They can be useful for service access, but File records remain the robust path for the main future EPW workflow.
aggregations <- datasets$collect(
type = "Aggregation",
fields = "*",
all = TRUE,
limit = NULL
)
aggregations$count()
#> [1] 0
if (aggregations$count()) {
result_table(aggregations, c(
"title", "variable_id", "data_node", "datetime_start",
"datetime_end", "url_opendap", "url_download"
), n = 12L)
} else {
data.frame(
note = "This live query did not return Aggregation records; continue with File records."
)
}
#> note
#> 1 This live query did not return Aggregation records; continue with File records.When Aggregation records are present, the same time-filtering and table inspection pattern applies.
if (aggregations$count()) {
aggregations_2060 <- aggregations$filter_time(
"2060-01-01T00:00:00Z",
"2060-12-31T23:59:59Z",
method = "drs"
)
aggregations_2060$count()
result_table(aggregations_2060, c(
"title", "variable_id", "datetime_start", "datetime_end",
"url_opendap", "url_download"
), n = 12L)
} else {
data.frame(
note = "This live query did not return Aggregation records; continue with File records."
)
}
#> note
#> 1 This live query did not return Aggregation records; continue with File records.Query and result objects can be saved as JSON. This is useful for
debugging a selection or sharing a small reproducible query state. For
durable project work, use an EsgStore instead of managing
JSON paths manually.
query_path <- file.path(workflow_root, "query.json")
files_path <- file.path(workflow_root, "files-2060.json")
query$save(query_path)
#> [1] "/private/var/folders/8f/t8sk2pps6135xbp47cs8qb2r0000gn/T/Rtmp3Hed7X/epwshiftr-esgf-query-results/query.json"
files_2060$save(files_path)
#> [1] "/private/var/folders/8f/t8sk2pps6135xbp47cs8qb2r0000gn/T/Rtmp3Hed7X/epwshiftr-esgf-query-results/files-2060.json"
query_copy <- esg_query()$load(query_path)
files_copy <- esg_result("file")$load(files_path)
query_copy$state(null = FALSE)
#> $index_node
#> [1] "https://esgf-data.dkrz.de"
#>
#> $parameter
#> $parameter$project
#> =CMIP6
#>
#> $parameter$activity_id
#> =ScenarioMIP
#>
#> $parameter$experiment_id
#> =ssp585
#>
#> $parameter$source_id
#> =MPI-ESM1-2-LR
#>
#> $parameter$variable_id
#> =tas,hurs,psl,rlds,rsds,sfcWind,clt
#>
#> $parameter$frequency
#> =mon
#>
#> $parameter$variant_label
#> =r1i1p1f1
#>
#> $parameter$data_node
#> =esgf.ceda.ac.uk
#>
#> $parameter$fields
#> =id,source_id,experiment_id,variant_label,frequency,table_id,variable_id,data_node,number_of_files,number_of_aggregations,access
#>
#> $parameter$type
#> =Dataset
#>
#> $parameter$offset
#> =0
#>
#> $parameter$distrib
#> =true
#>
#> $parameter$limit
#> =20
#>
#> $parameter$format
#> =application%2Fsolr%2Bjson
#>
#> $parameter$table_id
#> =Amon
files_copy$count()
#> [1] 7The next layer is ESG stores, where these query and file objects become durable manifest records connected to downloads, extraction plans, and generated EPW artifacts.