From 4a7bcc0765555b9656903a939b5868643e989037 Mon Sep 17 00:00:00 2001 From: Tom Hodson Date: Thu, 21 Nov 2024 13:54:23 +0000 Subject: [PATCH] Add test config --- config/destinE/schema | 130 ++++ config/language.yaml | 1232 ++++++++++++++++++++++++++++++++++++ config/standard_fdb_schema | 590 +++++++++++++++++ config/test_schema | 11 + 4 files changed, 1963 insertions(+) create mode 100644 config/destinE/schema create mode 100644 config/language.yaml create mode 100644 config/standard_fdb_schema create mode 100644 config/test_schema diff --git a/config/destinE/schema b/config/destinE/schema new file mode 100644 index 0000000..faa8402 --- /dev/null +++ b/config/destinE/schema @@ -0,0 +1,130 @@ +# * Format of the rules is: + +# [a1, a2, a3 ...[b1, b2, b3... [c1, c2, c3...]]] + +# - The first level (a) defines which attributes are used to name the top level directory +# - The second level (b) defines which attributes are used to name the data files +# - The third level (c) defines which attributes are used as index keys + +# * Rules can be grouped + +# [a1, a2, a3 ... +# [b1, b2, b3... [c1, c2, c3...]] +# [B1, B2, B3... [C1, C2, C3...]] +# ] + +# * A list of values can be given for an attribute +# [ ..., stream=enfo/efov, ... ] +# This will be used when matching rules. + +# * Attributes can be typed +# Globally, at the begining of this file: + +# refdate: Date; + +# or in the context of a rule: +# [type=cl, ... [date:ClimateMonth, ...]] + +# Typing attributes is done when the user's requests or the GRIB values need to be modified before directories, files and indexes are created. For example, ClimateMonth will transform 2010-04-01 to 'may' internally. + +# * Attributes can be optional +# [ step, levelist?, param ] +# They will be replaced internally by an empty value. It is also posiible to provide a default subtitution value: e.g. [domain?g] will consider the domain to be 'g' if missing. + +# * Attributes can be removed: +# [grid-] +# This is useful to remove attributes present in the GRIB that should not be ignored + +# * Rules are matched: + +# - If the attributes are present in the GRIB/Request, or marked optional or ignored +# - If a list of possible value is provided, one of them must match, for example +# [ class, expver, stream=enfo/efov, date, time, domain ] +# will match either stream=enfo or stream=efov, all other attributes will be matched if they exist in the GRIB or user's request + +# * On archive: +# - Attributes are extracted from the GRIB (namespace 'mars'), possibly modified by the attribute type +# - Only the first rule is used, so order is important +# - All GRIB attributes must be used by the rules, otherwise an error is raised + +# * On retrieve: +# - Attributes are extracted from the user's request, possibly modified by the attribute type (e.g. for handling of U/V) +# - All the matching rules are considered +# - Only attributes listed in the rules are used to extract values from the user's request + + +# Default types + +param: Param; +step: Step; +date: Date; +hdate: Date; +refdate: Date; +latitude: Double; +longitude: Double; +levelist: Double; +grid: Grid; +expver: Expver; + +time: Time; +fcmonth: Integer; + +number: Integer; +frequency: Integer; +direction: Integer; +channel: Integer; + +instrument: Integer; +ident: Integer; + +diagnostic: Integer; +iteration: Integer; +system: Integer; +method: Integer; + + +######################################################## +# These are the rules for the Climate DT + +# clte/wave +[ class=d1, dataset=climate-dt, activity, experiment, generation, model, realization, expver, stream=clte/wave, date + [ resolution, type, levtype + [ time, levelist?, param, frequency?, direction? ]] +] + +# clmn +[ class=d1, dataset=climate-dt, activity, experiment, generation, model, realization, expver, stream=clmn, year + [ month, resolution, type, levtype + [ levelist?, param ]] +] + +######################################################## +# These are the rules for the Extremes DT +# oper/wave/lwda/lwwv +[ class=d1, dataset=extremes-dt, expver, stream=oper/wave/lwda/lwwv, date, time + [ resolution?, type, levtype + [ step, levelist?, param, frequency?, direction? ]] +] +######################################################## +# These are the rules for the On-Demand Extremes DT +# oper/wave +[ class=d1, dataset=on-demand-extremes-dt, expver, stream=oper/wave, date, time + [ type, levtype + [ step, levelist?, param, frequency?, direction? ]] +] +######################################################## + + +######################################################## +# +# These are the rules for rd +# oper/wave/lwda/lwwv +[ class=rd, expver, stream=oper/wave/lwda/lwwv/dcda/enfo, date, time, domain? + [ type, levtype + [ number?, step, levelist?, param, frequency?, direction? ]] +] + +[ class=rd, expver, stream=mnth, domain? + [ type, levtype + [ date , time, step?, levelist?, param ]] +] \ No newline at end of file diff --git a/config/language.yaml b/config/language.yaml new file mode 100644 index 0000000..50d793f --- /dev/null +++ b/config/language.yaml @@ -0,0 +1,1232 @@ +--- +_field: &_field + + foo: + description: A foo field + values: + - bar + - baz + + class: + description: Class selects the main category of data to be retrieved such as operational, research or AIFS + category: data + default: od + flatten: false + type: enum + values: + - [ai, operational aifs] + # - [at, austria] + # - [be, belgium] + # - [c3, c3s] + # - [ce, cems] + # - [ch, switzerland] + # - [ci, cerise] + # - [co, cosmo] + # - [cr, cams research] + # - [cs, ecsn] + # - [d1, destine] + # - [de, germany] + # - [dk, denmark] + # - [dm, demeter] + # - [dt, dts] + # - [e2, e20c] + # - [e4, reanalyse40] + # - [e6, era6] + # - [ea, era5, esat] + # - [ed, eerie] + # - [ef, efas] + # - [ei, era interim] + # - [el, eldas] + # - [em, e20cm] + # - [en, ensembles] + # - [ep, cera-20c, cera20c] + # - [er, reanalyse] + # - [es, spain] + # - [et, cera-sat, cerasat] + # - [fi, finland] + # - [fr, france] + # - [gf, glofas] + # - [gg, greenhouse gases] + # - [gr, greece] + # - [gw, global wildfire information system] + # - [hr, croatia] + # - [hu, hungary] + # - [ie, ireland] + # - [is, iceland] + # - [it, italy] + # - [j5, jra55] + # - [l5, era5l] + # - [l6, era6l] + # - [la, aladin-laef, laef, lace] + # - [lw, WMO lead centre wave forecast verification] + # - [ma, metaps] + # - [mc, macc] + # - [me, mersea] + # - [ml, machine learning] + # - [ms, member states] + # - [nl, netherlands] + # - ['no', norway] + # - [nr, ncep 20cr, 20cr] + # - [o6, 'ocean 6 reanalysis'] + - [od, operations] + # - [pe, permanent experiment] + # - [pt, portugal] + # - [pv, provost] + - [rd, research] + # - [rm, euro4m] + # - [rr, regional reanalysis] + # - [s2, s2s] + # - [se, sweden] + # - [si, slovenia] + # - [sr, sreps] + # - [te, test] + # - [to, tost] + # - [tr, turkey] + # - [uk, united kingdom] + # - [ul, ulysses] + # - [ur, uerra] + # - [yp, yopp] + # - [yt, yotc] + + type: + category: data + default: an + flatten: false + type: enum + multiple: true + values: + # - [3g, 3d variational gradients] + # - [3v, 3d variational analysis] + # - [4g, 4d variational gradients] + # - [4i, 4d variational increments] + # - [4v, 4d variational analysis] + # - [ab, analysis bias] + # - [af, analysis feedback] + # - [ai, analysis input] + # - [an, analysis] + # - [as, adjoint singular vector] + # - [bf, bias-corrected forecast] + # - [cd, climate distribution] + # - [cf, control forecast] + # - [ci, clustering information] + # - [cl, climatology] + # - [cm, cluster means] + # - [cr, cluster representative] + # - [cs, cluster std deviations] + # - [cv, calibration validation forecast] + # - [ea, errors in analysis] + # - [ed, empirical distribution] + # - [ef, errors in first guess] + # - [efi, extreme forecast index] + # - [efic, extreme forecast index control] + # - [em, ensemble mean] + # - [eme, ensemble data assimilation model errors] + # - [emtm, ensemble mean of temporal mean] + # - [ep, event probability] + # - [es, ensemble standard deviation] + # - [est, ensemble statistics] + # - [estdtm, ensemble standard deviation of temporal mean] + # - [fa, forecast accumulation] + # - [fb, feedback] + - [fc, forecast] + # - [fcdfb, forecast departures feedback] + # - [fcmax, forecast maximum] + # - [fcmean, forecast mean] + # - [fcmin, forecast minimum] + # - [fcstdev, forecast standard deviation] + # - [ff, flux forcing realtime] + # - [fg, first guess] + # - [fp, forecast probability] + # - [fsoifb, forecast sensitivity to observations impact feedback] + # - [fu, fill-up] + # - [fx, flux forcing] + # - [ga, gfas analysis] + # - [gbf, bias-corrected gridbox] + # - [gai, gridded analysis input] + # - [go, gridded observations] + # - [gsd, gridded satellite data] + # - [gwt, weather type gridbox] + # - [hcmean, hindcast mean] + # - [ia, init. analysis] + # - [icp, initial condition perturbation] + # - [mpp, model physics perturbation] + # - [if, interim forecast] + # - [im, images] + # - [me, model errors] + # - [mfb, mondb feedback] + # - [oai, odb analysis input] + # - [ob, observations] + # - [of, ocean forward] + # - [ofb, odb feedback] + # - [oi, oi analysis] + # - [oldim, old format images] + # - [or, ocean reanalysis] + # - [pa, perturbed analysis] + # - [pb, probability boundary] + # - [pd, probability distribution] + - [pf, perturbed forecast] + # - [pfc, point values] + # - [ppm, point value metrics] + # - [s3, climate 30 days simulation] + # - [ses, scaled ensemble standard deviation] + # - [sf, sensitivity forecast] + # - [sfb, summary feedback] + # - [sfo, simulations with forcing] + # - [sg, sensitivity gradient] + # - [si, climate simulation] + # - [sim, simulated images] + # - [sot, shift of tails] + # - [ssd, simulated satellite data] + # - [sv, singular vector] + # - [svar, signal variance] + # - [taem, time average ensemble mean] + # - [taes, time average ensemble standard deviation] + # - [tpa, time processed analysis] + # - [tf, trajectory forecast] + # - [tu, tube] + # - [wem, weighted ensemble mean] + # - [wes, weighted ensemble standard deviation] + # - [wp, weather parameters] + + stream: + description: Stream selects the kind of data to be retrieved, for example the forecast model or the ensemble model. + category: data + default: oper + flatten: false + type: enum + values: + # - [amap, analysis for multianalysis project] + # - [ammc, melbourne] + # - [cher, ch, chernobyl] + # - [clte, climate, Climate run output] + # - [clmn, climate-monthly, Climate run monthly means output] + # - [cnrm, meteo france climate centre] + # - [cwao, montreal] + # - [dacl, daily climatology] + # - [dacw, daily climatology wave] + # - [dahc, daily archive hindcast] + # - [dcda, atmospheric model (delayed cutoff)] + # - [dcwv, wave model (delayed cutoff)] + # - [edmm, ensemble data assimilation monthly means] + # - [edmo, ensemble data assimilation monthly means of daily means] + # - [edzw, offenbach] + # - [eefh, extended ensemble forecast hindcast] + # - [eefo, extended ensemble prediction system] + # - [eehs, extended ensemble forecast hindcast statistics] + # - [efas, european flood awareness system (efas)] + # - [efcl, european flood awareness system (efas) climatology] + # - [efhc, ensemble forecast hindcasts (obsolete)] + # - [efho, ensemble forecast hindcast overlap] + # - [efhs, ensemble forecast hindcast statistics] + # - [efov, ensemble forecast overlap] + # - [efrf, european flood awareness system (efas) reforecasts] + # - [efse, european flood awareness system (efas) seasonal forecasts] + # - [efsr, european flood awareness system (efas) seasonal reforecasts] + # - [egrr, exeter, bracknell] + # - [ehmm, combined multi-model hindcast monthly means] + # - [elda, ensemble long window data assimilation] + # - [enda, ensemble data assimilation] + # - [enfh, ensemble forecast hindcasts] + - [enfo, ef, ensemble prediction system] + # - [enwh, ensemble forecast wave hindcasts] + # - [esmm, combined multi-model monthly means] + # - [espd, ensemble supplementary data] + # - [ewda, ensemble wave data assimilation] + # - [ewhc, wave ensemble forecast hindcast (obsolete)] + # - [ewho, ensemble forecast wave hindcast overlap] + # - [ewla, ensemble wave long window data assimilation] + # - [ewmm, ensemble wave data assimilation monthly means] + # - [ewmo, ensemble wave data assimilation monthly means of daily means] + # - [fgge, fg] + # - [fsob, forecast sensitivity to observations] + # - [fsow, forecast sensitivity to observations wave] + # - [gfas, global fire assimilation system] + # - [gfra, global fire assimilation system reanalysis] + # - [kwbc, washington] + # - [lfpw, paris, toulouse] + # - [lwda, long window daily archive] + # - [lwwv, long window wave] + # - [ma, means archive] + # - [maed, multianalysis ensemble data] + # - [mawm, wave anomaly means] + # - [mawv, multianalysis wave data] + # - [mdfa, monthly means of daily forecast accumulations] + # - [mfam, anomaly means] + # - [mfaw, wave anomalies] + # - [mfhm, hindcast means] + # - [mfhw, monthly forecast hindcasts wave] + # - [mfwm, wave real-time means] + # - [mhwm, wave hindcast means] + # - [mmaf, multi-model multi-annual forecast] + # - [mmam, multi-model multi-annual forecast means] + # - [mmaw, multi-model multi-annual forecast wave] + # - [mmsa, multi-model seasonal forecast monthly anomalies] + # - [mmsf, multi-model seasonal forecast] + # - [mmwm, multi-model multi-annual forecast wave means] + # - [mnfa, anomalies] + # - [mnfc, real-time] + # - [mnfh, hindcasts] + # - [mnfm, real-time means] + # - [mnfw, wave real-time] + # - [mnth, mo, monthly, monthly means] + # - [mnvr, monthly variance and covariance data using g. boer's step function] + # - [moda, monthly means of daily means] + # - [mofc, monthly forecast] + # - [mofm, monthly forecast means] + # - [monr, monthly means using g. boer's step function] + # - [mpic, max plank institute] + # - [msda, monthly standard deviation and covariance of daily means] + # - [msdc, mv, monthly standard deviation and covariance] + # - [msmm, multi-model seasonal forecast atmospheric monthly means] + # - [mswm, multi-model seasonal forecast wave monthly means] + # - [ocda, ocean data assimilation] + # - [ocea, ocean] + # - [olda, ocean Long window data assimilation] + - [oper, da, daily archive, atmospheric model] + # - [rjtd, tokyo] + # - [scda, atmospheric model (short cutoff)] + # - [scwv, wave model (short cutoff)] + # - [seap, sensitive area prediction] + # - [seas, seasonal forecast] + # - [sens, sf, sensitivity forecast] + # - [sfmm, seasonal forecast atmospheric monthly means] + # - [smma, seasonal monthly means anomalies] + # - [supd, sd, deterministic supplementary data] + # - [swmm, seasonal forecast wave monthly means] + # - [toga, tg] + # - [ukmo, ukmo climate centre] + # - [waef, we, wave ensemble forecast] + # - [wamd, wave monthly means of daily means] + # - [wamf, wave monthly forecast] + # - [wamo, wave monthly means] + # - [wams, multi-model seasonal forecast wave] + # - [wasf, wave seasonal forecast] + # - [wave, wv, wave model] + # - [wavm, wave model (standalone)] + # - [weef, wave extended ensemble forecast] + # - [weeh, wave extended ensemble forecast hindcast] + # - [wees, wave extended ensemble forecast hindcast statistics] + # - [wehs, wave ensemble forecast hindcast statistics] + # - [weov, wave ensemble forecast overlap] + # - [wfas, global flood awareness system (glofas)] + # - [wfcl, global flood awareness system (glofas) climatology] + # - [wfrf, global flood awareness system (glofas) reforecasts] + # - [wfse, global flood awareness system (glofas) seasonal forecasts] + # - [wfsr, global flood awareness system (glofas) seasonal reforecasts] + # - [wmfm, wave monthly forecast means] + # - [wvhc, wave hindcast] + expver: + description: Experiment number, 0001 is operational data. + category: data + default: '0001' + flatten: false + type: enum + values: + - ['0001', 'Operational Data'] + - ['xxxx', 'Experimental Data'] + + dataset: + # flatten: false + multiple: true + type: any + + model: + category: data + type: lowercase + + repres: + flatten: false + multiple: true + type: enum + values: + - gg + - sh + - ll + - np + - rl + + obsgroup: + category: data + multiple: true + type: enum + values: + # - [conventional] + - [sat, satellite] + - [ers1] + - [trmm] + - [qscat] + - [reo3] # reo3 needs to stay for compatibility + # previously in "obsgroups.def" + - [hirs, 1, HIRS ] + - [amsua, 2, AMSUA ] + - [amsub, 3, AMSUB ] + - [mhs, 4, MHS ] + - [geos, 5, GEOS ] + - [resat, 6, RESAT ] + - [meris, 7, MERIS ] + - [gpsro, 8, GPSRO ] + - [satob, 9, SATOB ] + - [scatt, 10, SCATT ] + - [ssmi_as, 11, SSMI ALL-SKY ] + - [iasi, 12, IASI ] + - [airs, 13, AIRS ] + - [ssmis_as, 14, SSMIS ALL-SKY ] + - [tmi_as, 15, TMI ALL-SKY ] + - [amsre_as, 16, AMSRE ALL-SKY ] + - [conv, 17, CONV ] + - [smos, 19, SMOS ] + - [windsat_as, 20, WINDSAT ALL-SKY ] + - [ssmi, 21, SSMI ] + - [amsua_as, 22, AMSUA ALL-SKY ] + - [amsre, 23, AMSRE ] + - [tmi, 24, TMI ] + - [ssmis, 25, SSMIS ] + - [gbrad, 26, GBRAD ] + - [mwhs, 27, MWHS ] + - [mwts, 28, MWTS ] + - [mwri_as, 29, MWRI ALL-SKY ] + - [iras, 30, IRAS ] + - [msu, 31, MSU ] + - [ssu, 32, SSU ] + - [vtpr1, 33, VTPR1 ] + - [vtpr2, 34, VTPR2 ] + - [atms, 35, ATMS ] + - [resat_ak, 36, RESAT AVERAGING KERNELS ] + - [cris, 37, CRIS ] + - [wave_ip, 38, WAVE INTEGRATED PARAMETERS ] + - [wave_sp, 39, WAVE SPECTRA ] + - [raingg, 40, RAINGG ] + - [sfc_ms, 41, SURFACE MULTISENSOR ] + - [amsr2_as, 42, AMSR-2 ALL-SKY ] + - [saphir_as, 43, SAPHIR ALL-SKY ] + - [amsub_as, 44, AMSUB ALL-SKY ] + - [mhs_as, 45, MHS ALL-SKY ] + - [dwl, 46, DOPPLER WIND LIDAR ] + - [iris, 47, IRIS ] + - [aatsr, 49, AATSR ] + - [atms_as, 50, ATMS ALL-SKY ] + - [gmi_as, 51, GMI ALL-SKY ] + - [godae_sst, 52, GODAE SEA SURFACE TEMPERATURES ] + - [atovs_ms, 53, ATOVS MULTISENSOR ] + - [atmospheric_composition, 54, ATMOSPHERIC COMPOSITION ] + - [non_sfc_ms, 55, NON-SURFACE MULTISENSOR ] + - [mwts2, 56, MWTS2 ] + - [ssmi_1d, 57, SSMI 1DVAR TCWV CLOUDY-SKY ] + - [mwhs2_as, 58, MWHS2 ALL-SKY ] + - [ssmt2, 59, SSMT2 ] + - [smap, 60, SMAP ] + - [tovs_ms, 61, TOVS MULTISENSOR ] + - [cloud_r, 62, CLOUD REFLECTIVITY ] + - [cloud_l, 63, CLOUD LIDAR ] + - [satellite_lightning, 64, SATELLITE LIGHTNING ] + - [geos_vis, 65, GEOS VIS ] + - [oconv, 66, OCONV ] + - [mwts3_as, 67, MWTS3 All-sky ] + - [giirs, 68, GIIRS ] + - [test, 99, TEST ] + + reportype: + category: data + type: any + multiple: true + + # rdbprefix + + levtype: + description: The Level Type, can be pressure levels, the surface, model levels etc. + category: data + default: pl + flatten: false + type: enum + values: + # - [cat, category] + # - [dp, depth] + # - [layer] + # - [ml, model levels] + - [pl, pressure levels] + # - [hl, height levels] + # - [pt, potential temperature] + # - [pv, potential vorticity] + - [sfc, surface] + # - [sol, surface other (multi)levels] + # - [wv, ocean wave] + # - [o2d, ocean surface] + # - [o3d, ocean model levels] + never: + - type: ssd + + levelist: + category: data + multiple: true + by: 1 + default: + - 1000 + - 850 + - 700 + - 500 + - 400 + - 300 + never: + - levtype: [sfc, o2d] + - type: ssd + type: to-by-list-float + + param: + category: data + default: 129 + multiple: true + type: param + never: + - type: [tf, ob] + +################################################################# + + # year + # decade + # month + + date: + category: data + default: 0 + type: date + multiple: true + + year: + category: data + type: to-by-list + multiple: true + by: 1 + + month: + category: data + flatten: true + type: enum + multiple: true + values: + - [1, jan, January] + - [2, feb, February] + - [3, mar, March] + - [4, apr, April] + - [5, may, May] + - [6, jun, June] + - [7, jul, July] + - [8, aug, August] + - [9, sep, September] + - [10, oct, October] + - [11, nov, November] + - [12, dec, December] + + # verify + # refdate + + hdate: + category: data + multiple: true + only: + - stream: + - enfh + - enwh + - efho + - ehmm + - ewho + - eefh + - weeh + type: integer + + offsetdate: + category: data + multiple: true + type: date + + fcmonth: + category: data + multiple: true + by: 1 + type: to-by-list + + fcperiod: + category: data + multiple: true + type: integer + + time: + category: data + default: '1200' + multiple: true + type: time + + offsettime: + category: data + multiple: true + type: time + + # leadtime + # opttime + # range + + step: + description: Specify which forecast we want in hours past the date/time. + category: data + multiple: true + by: 12 + default: 0 + type: range + never: + - dataset: + - climate-dt + - stream: + - msmm + - mmsa + - swmm + + anoffset: + category: data + multiple: true + type: integer + + reference: + category: data + multiple: true + type: integer + +################################################################# + + # cluster + # probability + + number: + description: Selects a subset of ensemble members + category: data + multiple: true + aliases: + - ensemble + by: 1 + only: + - type: [pf, cr, cm, fcmean, fcmin, fcmax, fcstdev, sot, fc, wp, 4i, 4v] + never: + # This is to prevent number with type=fc and stream=oper + - stream: [oper, wave] + type: to-by-list + + quantile: + category: data + multiple: true + only: + - type: + - pd + - pb + - taem + - cd + # - sot + type: to-by-list-quantile + denominators: [2,3,4,5,10,100,1000] + by: 1 + domain: + description: The large scale geographic region. + category: data + default: g + flatten: false + type: enum + never: + - dataset: + - climate-dt + values: + # - [a, north west europe] + # - [b, north east europe, baltic and black sea] + - [c, south west europe] + - [d, south east europe] + - [e, europe] + # - [f, fastex] + - [g, globe, general european area] + # - [h] + # - [i] + # - [j] + # - [k] + # - [l] + # - [m, mediterranean] + # - ['n', northern hemisphere] + # - [o] + # - [p] + # - [q] + # - [r] + # - [s, southern hemisphere] + # - [t, tropics] + # - [u, tropics 2] + # - [v] + # - [w, western atlantic] + # - [x] + # - ['y'] + # - [z] + + frequency: + category: data + multiple: true + by: 1 + only: + - param: + - '140251' + type: to-by-list + + direction: + category: data + multiple: true + by: 1 + only: + - param: + - '140251' + type: to-by-list + + diagnostic: + category: data + type: integer + multiple: true + + iteration: + category: data + type: integer + multiple: true + + channel: + category: data + only: + - type: ssd + type: integer + multiple: true + + ident: + category: data + only: + - type: ssd + type: integer + multiple: true + + instrument: + category: data + only: + - type: ssd + type: integer + multiple: true + + method: + category: data + type: integer + + origin: + category: data + multiple: true + type: enum + values: + - [ammc, 1, melbourne] + - [babj, 38, beijing] + - [cmcc] + - [cnmc, 80] + - [consensus, 255] + - [crfc, 239, cerfacs] + - [cwao, 54, montreal] + - [ecmf, 98, ecmwf] + - [edzw, dwd, 78, offenbach] + - [egrr, 74, exeter, bracknell] + - [enmi, 88, oslo] + - [fnmo, fnmoc, 58, fleet numerical] + - [hadc, 247, hadley centre] + - [ifmk, 246] + - [ingv, 235] + - [knmi, 245] + - [kwbc, 7, washington] + - [lemm, 214, madrid] + - [lfpw, 84, 85, paris, toulouse] + - [rjtd, 34, tokyo] + - [rksl, 40, seoul] + - [sbsj, 46, cptec] + - [vuwien, 244, university of vienna] + + system: + category: data + type: integer + +####################################################################### +# DestinE ClimateDT related keywords + + activity: + category: data + type: lowercase + + experiment: + category: data + type: lowercase + + generation: + category: data + type: integer + + realization: + category: data + type: integer + + resolution: + category: data + type: lowercase + +####################################################################### + +_observation: &_observation + + obstype: + category: data + type: any + multiple: true + + obsgroup: + category: data + type: any + multiple: true + +####################################################################### + +_postproc: &_postproc + + accuracy: + category: postproc + flatten: false + type: [enum, integer] + values: + - [av] + - ['off', normal, auto] + + bitmap: + category: postproc + flatten: false + type: any + + format: + category: postproc + flatten: false + type: enum + values: + - - grib + - grib1 + - gb + - - grib2 + - - bufr + - bf + - - grid + - gd + - odb + - ascii + + frame: + category: postproc + type: integer + + gaussian: + category: postproc + type: enum + values: + - reduced + - regular + + area: + category: postproc + flatten: false + multiple: true + type: [float, enum] + values: + - ['off', g, global] + - [e, europe] + + grid: + category: postproc + flatten: false + multiple: true + type: [enum, float, regex] + values: + - auto + - N16 + - N24 + - N32 + - N48 + - N64 + - N80 + - N96 + - N128 + - N160 + - N200 + - N256 + - N320 + - N400 + - N512 + - N576 + - N640 + - N800 + - N1024 + - N1280 + - N1600 + - N2000 + - N4000 + - N8000 + regex: + - '^[oOfF][1-9][0-9]+$' + uppercase: true + + interpolation: + category: postproc + flatten: false + type: enum + values: + - - linear + - - nearest-lsm + - nearest lsm + - - 'off' + - default + - any + + packing: + category: postproc + flatten: false + type: enum + values: + - - so + - second order + - ['off', av] + - [co, complex] + - simple + - ccsds + + resol: + category: postproc + flatten: false + aliases: + - tra + type: [enum, integer] + values: + - - 'off' + - av + - reduced gaussian 160 + + rotation: + category: postproc + flatten: false + multiple: true + type: float + + intgrid: + category: postproc + flatten: false + type: [enum, regex] + values: + - 'off' + - auto + - N32 + - N48 + - N64 + - N80 + - N96 + - N128 + - N160 + - N192 + - N200 + - N256 + - N320 + - N400 + - N512 + - N640 + - N800 + - N912 + - N1024 + - N1280 + regex: + - '^[oOfF][1-9][0-9]+$' + + truncation: + category: postproc + flatten: false + type: [enum, integer] + values: + - auto + - 'off' +####################################################################### + +_obspproc: &_obspproc + + filter: + type: any + category: postproc + + ident: + type: any + category: postproc + + +####################################################################### + +disseminate: + <<: *_field + <<: *_postproc + + requirements: + type: any + + use: + category: data + flatten: false + multiple: true + type: enum + values: + - bc + - monday + - tuesday + - wednesday + - thursday + - friday + - saturday + - sunday + + option: + default: normal + flatten: false + multiple: true + type: enum + values: + - normal + - delay + - asap + - gts + - opendata + + compatibility: + category: postproc + flatten: false + multiple: true + type: enum + values: + - 'off' + - 'no-local-extension' + + priority: + flatten: false + type: integer + + target: + flatten: false + type: any + +############################################################## + +archive: + <<: *_field + <<: *_observation + + + database: + flatten: false + multiple: true + type: any + + source: + flatten: false + multiple: true + type: any + + expect: + flatten: false + multiple: false + type: integer + +############################################################## + +retrieve: + + <<: *_field + <<: *_observation + <<: *_postproc + <<: *_obspproc + + target: + flatten: false + multiple: true + type: any + + expect: + flatten: false + multiple: false + type: integer + + fieldset: + flatten: false + multiple: false + type: any + + database: + flatten: false + multiple: true + type: any + + optimise: + type: enum + values: + - true + - false + default: + "off" + + padding: + flatten: false + type: enum + values: + - none + - auto + +############################################################## + +read: + source: + flatten: false + multiple: true + type: any + + <<: *_field + <<: *_observation + <<: *_postproc + <<: *_obspproc + + target: + flatten: false + multiple: true + type: any + + fieldset: + flatten: false + multiple: false + type: any + + _defaults: + class: null + date: null + domain: null + expver: null + levelist: null + levtype: null + param: null + step: null + stream: null + time: null + type: null + + _options: + param: + # expand_with: # In case not type/stream/levtype is provided + # type: an + # stream: oper + # levtype: pl + first_rule: true +############################################################## + +get: + + tape: + flatten: false + multiple: false + type: any + + database: + flatten: false + multiple: true + type: any + + target: + flatten: false + multiple: true + type: any + +############################################################## + +list: + + <<: *_field + <<: *_observation + + database: + flatten: false + multiple: true + type: any + + target: + flatten: false + multiple: true + type: any + + _defaults: + # class: null + date: null + domain: null + expver: null + levelist: null + levtype: null + param: null + step: null + stream: null + time: null + type: null + +############################################################## + +compute: + formula: + flatten: false + multiple: false + type: any + + fieldset: + flatten: false + multiple: false + type: any + +############################################################## + +write: + + fieldset: + flatten: false + multiple: false + type: any + + target: + flatten: false + multiple: true + type: any + +############################################################## + +pointdb: + lat: + multiple: false + type: float + + lon: + multiple: false + type: float + + <<: *_field + + _defaults: + class: null + date: null + domain: null + expver: null + levelist: null + levtype: null + param: null + step: null + stream: null + time: null + type: null + + _options: + param: + # expand_with: # In case not type/stream/levtype is provided + # type: an + # stream: oper + # levtype: pl + first_rule: true + +end: {} diff --git a/config/standard_fdb_schema b/config/standard_fdb_schema new file mode 100644 index 0000000..394223f --- /dev/null +++ b/config/standard_fdb_schema @@ -0,0 +1,590 @@ + +# * Format of the rules is: + +# [a1, a2, a3 ...[b1, b2, b3... [c1, c2, c3...]]] + +# - The first level (a) defines which attributes are used to name the top level directory +# - The second level (b) defines which attributes are used to name the data files +# - The third level (c) defines which attributes are used as index keys + +# * Rules can be grouped + +# [a1, a2, a3 ... +# [b1, b2, b3... [c1, c2, c3...]] +# [B1, B2, B3... [C1, C2, C3...]] +# ] + +# * A list of values can be given for an attribute +# [ ..., stream=enfo/efov, ... ] +# This will be used when matching rules. + +# * Attributes can be typed +# Globally, at the begining of this file: + +# refdate: Date; + +# or in the context of a rule: +# [type=cl, ... [date:ClimateMonth, ...]] + +# Typing attributes is done when the user's requests or the GRIB values need to be modified before directories, files and indexes are created. For example, ClimateMonth will transform 2010-04-01 to 'may' internally. + +# * Attributes can be optional +# [ step, levelist?, param ] +# They will be replaced internally by an empty value. It is also posiible to provide a default subtitution value: e.g. [domain?g] will consider the domain to be 'g' if missing. + +# * Attributes can be removed: +# [grid-] +# This is useful to remove attributes present in the GRIB that should not be ignored + +# * Rules are matched: + +# - If the attributes are present in the GRIB/Request, or marked optional or ignored +# - If a list of possible value is provided, one of them must match, for example +# [ class, expver, stream=enfo/efov, date, time, domain ] +# will match either stream=enfo or stream=efov, all other attributes will be matched if they exist in the GRIB or user's request + +# * On archive: +# - Attributes are extracted from the GRIB (namespace 'mars'), possibly modified by the attribute type +# - Only the first rule is used, so order is important +# - All GRIB attributes must be used by the rules, otherwise an error is raised + +# * On retrieve: +# - Attributes are extracted from the user's request, possibly modified by the attribute type (e.g. for handling of U/V) +# - All the matching rules are considered +# - Only attributes listed in the rules are used to extract values from the user's request + + +# Default types + +param: Param; +step: Step; +date: Date; +hdate: Date; +refdate: Date; +latitude: Double; +longitude: Double; +levelist: Double; +grid: Grid; +expver: Expver; + +time: Time; +fcmonth: Integer; + +number: Integer; +frequency: Integer; +direction: Integer; +channel: Integer; + +instrument: Integer; +ident: Integer; + +diagnostic: Integer; +iteration: Integer; +system: Integer; +method: Integer; + +# ??????? + +# reference: Integer; +# fcperiod: Integer; + +# opttime: Integer; +# leadtime: Integer; + +# quantile: ?????? +# range: ?????? + +# band: Integer; + + +######################################################## +# These rules must be first, otherwise fields of These +# classes will be index with the default rule for oper +[ class=ti/s2, expver, stream, date, time, model + [ origin, type, levtype, hdate? + [ step, number?, levelist?, param ]] +] + +[ class=ms, expver, stream, date, time, country=de + [ domain, type, levtype, dbase, rki, rty, ty + [ step, levelist?, param ]] +] + +[ class=ms, expver, stream, date, time, country=it + [ domain, type, levtype, model, bcmodel, icmodel:First3 + [ step, levelist?, param ] + ] +] + +[ class=el, expver, stream, date, time, domain + [ origin, type, levtype + [ step, levelist?, param ]] +] + +######################################################## +# The are the rules matching most of the fields +# oper/dcda +[ class, expver, stream=oper/dcda/scda, date, time, domain? + + [ type=im/sim + [ step?, ident, instrument, channel ]] + + [ type=ssd + [ step, param, ident, instrument, channel ]] + + [ type=4i, levtype + [ step, iteration, levelist, param ]] + + [ type=me, levtype + [ step, number, levelist?, param ]] + + [ type=ef, levtype + [ step, levelist?, param, channel? ]] + + [ type=ofb/mfb + [ obsgroup, reportype ]] + + [ type, levtype + [ step, levelist?, param ]] + +] + +# dcwv/scwv/wave +[ class, expver, stream=dcwv/scwv/wave, date, time, domain + [ type, levtype + [ step, param, frequency?, direction? ]]] + +# enfo +[ class, expver, stream=enfo/efov, date, time, domain + + [ type, levtype=dp, product?, section? + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + + [ type=tu, levtype, reference + [ step, number, levelist?, param ]] + + [ type, levtype + [ step, quantile?, number?, levelist?, param ]] + +] + +# waef/weov +[ class, expver, stream=waef/weov, date, time, domain + [ type, levtype + [ step, number?, param, frequency?, direction? ]] +] + +######################################################## +# enda +[ class, expver, stream=enda, date, time, domain + + [ type=ef/em/es/ses, levtype + [ step, number?, levelist?, param, channel? ]] + + [ type=ssd + [ step, number, param, ident, instrument, channel ]] + + + [ type, levtype + [ step, number?, levelist?, param ]] +] + +# ewda +[ class, expver, stream=ewda, date, time, domain + [ type, levtype + [ step, number?, param, frequency?, direction? ]] +] + + +######################################################## +# elda +[ class, expver, stream=elda, date, time, domain? + + [ type=ofb/mfb + [ obsgroup, reportype ]] + + [ type, levtype, anoffset + [ step, number?, levelist?, iteration?, param, channel? ]] +] + +# ewda +[ class, expver, stream=ewla, date, time, domain + [ type, levtype, anoffset + [ step, number?, param, frequency?, direction? ]] +] + +######################################################## +# elda +[ class, expver, stream=lwda, date, time, domain? + + [ type=ssd, anoffset + [ step, param, ident, instrument, channel ]] + + [type=me, levtype, anoffset + [ number, step, levelist?, param]] + + [ type=4i, levtype, anoffset + [ step, iteration, levelist, param ]] + + [ type=ofb/mfb + [ obsgroup, reportype ]] + + [ type, levtype, anoffset + [ step, levelist?, param]] +] + +# ewda +[ class, expver, stream=lwwv, date, time, domain + [ type, levtype, anoffset + [ step, param, frequency?, direction? ]] +] +######################################################## +# amap +[ class, expver, stream=amap, date, time, domain + [ type, levtype, origin + [ step, levelist?, param ]]] + +# maed +[ class, expver, stream=maed, date, time, domain + [ type, levtype, origin + [ step, levelist?, param ]]] + +# mawv +[ class, expver, stream=mawv, date, time, domain + [ type, levtype, origin + [ step, param, frequency?, direction? ]]] + +# cher +[ class, expver, stream=cher, date, time, domain + [ type, levtype + [ step, levelist, param ]]] + + +# efhc +[ class, expver, stream=efhc, refdate, time, domain + [ type, levtype, date + [ step, number?, levelist?, param ]]] + +# efho +[ class, expver, stream=efho, date, time, domain + [ type, levtype, hdate + [ step, number?, levelist?, param ]]] + + +# efhs +[ class, expver, stream=efhs, date, time, domain + [ type, levtype + [ step, quantile?, number?, levelist?, param ]]] + +# wehs +[ class, expver, stream=wehs, date, time, domain + [ type, levtype + [ step, quantile?, number?, levelist?, param ]]] + +# kwbc +[ class, expver, stream=kwbc, date, time, domain + [ type, levtype + [ step, number?, levelist?, param ]]] + +# ehmm +[ class, expver, stream=ehmm, date, time, domain + [ type, levtype, hdate + [ fcmonth, levelist?, param ]]] + + +# ammc/cwao/edzw/egrr/lfpw/rjtd/toga +[ class, expver, stream=ammc/cwao/edzw/egrr/lfpw/rjtd/toga/fgge, date, time, domain + [ type, levtype + [ step, levelist?, param ]]] + +######################################################################## + +# enfh +[ class, expver, stream=enfh, date, time, domain + + [ type, levtype=dp, hdate, product?, section? + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + + [ type, levtype, hdate + [ step, number?, levelist?, param ]] +] + +# enwh +[ class, expver, stream=enwh, date, time, domain + [ type, levtype, hdate + [ step, number?, param, frequency?, direction? ]] +] + +######################################################################## +# sens +[ class, expver, stream=sens, date, time, domain + [ type, levtype + [ step, diagnostic, iteration, levelist?, param ]]] + +######################################################################## +# esmm +[ class, expver, stream=esmm, date, time, domain + [ type, levtype + [ fcmonth, levelist?, param ]]] +# ewhc +[ class, expver, stream=ewhc, refdate, time, domain + [ type, levtype, date + [ step, number?, param, frequency?, direction? ]]] + +######################################################################## +# ewho +[ class, expver, stream=ewho, date, time, domain + [ type, levtype, hdate + [ step, number?, param, frequency?, direction? ]]] + +# mfam +[ class, expver, stream=mfam, date, time, domain + + [ type=pb/pd, levtype, origin, system?, method + [ fcperiod, quantile, levelist?, param ]] + + [ type, levtype, origin, system?, method + [ fcperiod, number?, levelist?, param ]] + +] + +# mfhm +[ class, expver, stream=mfhm, refdate, time, domain + [ type, levtype, origin, system?, method, date? + [ fcperiod, number?, levelist?, param ]]] +# mfhw +[ class, expver, stream=mfhw, refdate, time, domain + [ type, levtype, origin, system?, method, date + [ step, number?, param ]]] +# mfwm +[ class, expver, stream=mfwm, date, time, domain + [ type, levtype, origin, system?, method + [ fcperiod, number, param ]]] +# mhwm +[ class, expver, stream=mhwm, refdate, time, domain + [ type, levtype, origin, system?, method, date + [ fcperiod, number, param ]]] + +# mmsf +[ class, expver, stream=mmsf, date, time, domain + + [ type, levtype=dp, origin, product, section, system?, method + [ step, number, levelist?, latitude?, longitude?, range?, param ]] + + [ type, levtype, origin, system?, method + [ step, number, levelist?, param ]] +] + +# mnfc +[ class, expver, stream=mnfc, date, time, domain + + [ type, levtype=dp, origin, product, section, system?, method + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + + [ type, levtype, origin, system?, method + [ step, number?, levelist?, param ]] +] + +# mnfh +[ class, expver, stream=mnfh, refdate, time, domain + [ type, levtype=dp, origin, product, section, system?, method, date + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + [ type, levtype, origin, system?, method, date? + [ step, number?, levelist?, param ]] +] + +# mnfm +[ class, expver, stream=mnfm, date, time, domain + [ type, levtype, origin, system?, method + [ fcperiod, number?, levelist?, param ]]] + +# mnfw +[ class, expver, stream=mnfw, date, time, domain + [ type, levtype, origin, system?, method + [ step, number?, param ]]] + +# ea/mnth +[ class=ea, expver, stream=mnth, date, domain + [ type, levtype + [ time, step?, levelist?, param ]]] + +# mnth +[ class, expver, stream=mnth, domain + [ type=cl, levtype + [ date: ClimateMonthly, time, levelist?, param ]] + [ type, levtype + [ date , time, step?, levelist?, param ]]] + +# mofc +[ class, expver, stream=mofc, date, time, domain + [ type, levtype=dp, product, section, system?, method + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + [ type, levtype, system?, method + [ step, number?, levelist?, param ]] +] + +# mofm +[ class, expver, stream=mofm, date, time, domain + [ type, levtype, system?, method + [ fcperiod, number, levelist?, param ]]] + +# mmsa/msmm +[ class, expver, stream=mmsa, date, time, domain + [ type, levtype, origin, system?, method + [ fcmonth, number?, levelist?, param ]]] + +[ class, expver, stream=msmm, date, time, domain + [ type, levtype, origin, system?, method + [ fcmonth, number?, levelist?, param ]]] + +# ocea +[ class, expver, stream=ocea, date, time, domain + [ type, levtype, product, section, system?, method + [ step, number, levelist?, latitude?, longitude?, range?, param ]] +] + +#=# seas +[ class, expver, stream=seas, date, time, domain + + [ type, levtype=dp, product, section, system?, method + [ step, number, levelist?, latitude?, longitude?, range?, param ]] + + [ type, levtype, system?, method + [ step, number, levelist?, param ]] +] + +# sfmm/smma +[ class, expver, stream=sfmm/smma, date, time, domain + [ type, levtype, system?, method + [ fcmonth, number?, levelist?, param ]]] + +# supd +[ class=od, expver, stream=supd, date, time, domain + [ type, levtype, origin?, grid + [ step, levelist?, param ]]] + +# For era +[ class, expver, stream=supd, date, time, domain + [ type, levtype, grid- # The minus sign is here to consume 'grid', but don't index it + [ step, levelist?, param ]]] + +# swmm +[ class, expver, stream=swmm, date, time, domain + [ type, levtype, system?, method + [ fcmonth, number, param ]]] + +# wamf +[ class, expver, stream=wamf, date, time, domain + [ type, levtype, system?, method + [ step, number?, param ]]] + +# ea/wamo +[ class=ea, expver, stream=wamo, date, domain + [ type, levtype + [ time, step?, param ]]] + +# wamo +[ class, expver, stream=wamo, domain + [ type=cl, levtype + [ date: ClimateMonthly, time, param ]] + [ type, levtype + [ date, time, step?, param ]]] + +# wamd +[ class, expver, stream=wamd, date, domain + [ type, levtype + [ param ]]] + +# wasf +[ class, expver, stream=wasf, date, time, domain + [ type, levtype, system?, method + [ step, number, param ]]] +# wmfm +[ class, expver, stream=wmfm, date, time, domain + [ type, levtype, system?, method + [ fcperiod, number, param ]]] + +# moda +[ class, expver, stream=moda, date, domain + [ type, levtype + [ levelist?, param ]]] + +# msdc/mdfa/msda +[ class, expver, stream=msdc/mdfa/msda, domain + [ type, levtype + [ date, time?, step?, levelist?, param ]]] + + + +# seap +[ class, expver, stream=seap, date, time, domain + [ type=sv/svar, levtype, origin, method? + [ step, leadtime, opttime, number, levelist?, param ]] + + [ type=ef, levtype, origin + [ step, levelist?, param, channel? ]] + + [ type, levtype, origin + [ step, levelist?, param ]] + + ] + +[ class, expver, stream=mmaf, date, time, domain + [ type, levtype, origin, system?, method + [ step, number, levelist?, param ]] +] + +[ class, expver, stream=mmam, date, time, domain + [ type, levtype, origin, system?, method + [ fcmonth, number, levelist?, param ]] +] + + +[ class, expver, stream=dacl, domain + [ type=pb, levtype + [ date: ClimateDaily, time, step, quantile, levelist?, param ]] + [ type, levtype + [ date: ClimateDaily, time, step, levelist?, param ]] + +] + +[ class, expver, stream=dacw, domain + [ type=pb, levtype + [ date: ClimateDaily, time, step, quantile, param ]] + [ type, levtype + [ date: ClimateDaily, time, step, param ]] + +] + +[ class, expver, stream=edmm/ewmm, date, time, domain + [ type=ssd + [ step, number, param, ident, instrument, channel ]] + [ type, levtype + [ step, number, levelist?, param ]] +] + +[ class, expver, stream=edmo/ewmo, date, domain + [ type, levtype + [ number, levelist?, param ]] +] + +# stream gfas +[ class=mc/rd, expver, stream=gfas, date, time, domain + [ type=ga, levtype + [ step, param ]] + + [ type=gsd + [ param, ident, instrument ]] + +] + +# class is e2 +[ class, expver, stream=espd, date, time, domain + [ type, levtype, origin, grid + [ step, number, levelist?, param ]]] + +[ class=cs, expver, stream, date:Default, time, domain + [ type, levtype + [ step, levelist?, param ]]] + + diff --git a/config/test_schema b/config/test_schema new file mode 100644 index 0000000..a0a684d --- /dev/null +++ b/config/test_schema @@ -0,0 +1,11 @@ +[ class=od, stream, date, time + [ domain, type, levtype, dbase, rki, rty, ty + [ step, levelist?, param ]] +] + +[ class=ensemble, number, stream, date, time, + [ domain, type, levtype, dbase, rki, rty, ty + [ step, levelist?, param ]] +] + +[ class, foo] \ No newline at end of file