diff --git a/.gitignore b/.gitignore
index c34d28b..52c6e04 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,6 @@
__pycache__
-.DS_Store
\ No newline at end of file
+.DS_Store
+config.yaml
+.venv
+*.json
+raw_list
\ No newline at end of file
diff --git a/backend/destinE_schema b/backend/destinE_schema
new file mode 100644
index 0000000..faa8402
--- /dev/null
+++ b/backend/destinE_schema
@@ -0,0 +1,130 @@
+# * Format of the rules is:
+
+# [a1, a2, a3 ...[b1, b2, b3... [c1, c2, c3...]]]
+
+# - The first level (a) defines which attributes are used to name the top level directory
+# - The second level (b) defines which attributes are used to name the data files
+# - The third level (c) defines which attributes are used as index keys
+
+# * Rules can be grouped
+
+# [a1, a2, a3 ...
+# [b1, b2, b3... [c1, c2, c3...]]
+# [B1, B2, B3... [C1, C2, C3...]]
+# ]
+
+# * A list of values can be given for an attribute
+# [ ..., stream=enfo/efov, ... ]
+# This will be used when matching rules.
+
+# * Attributes can be typed
+# Globally, at the begining of this file:
+
+# refdate: Date;
+
+# or in the context of a rule:
+# [type=cl, ... [date:ClimateMonth, ...]]
+
+# Typing attributes is done when the user's requests or the GRIB values need to be modified before directories, files and indexes are created. For example, ClimateMonth will transform 2010-04-01 to 'may' internally.
+
+# * Attributes can be optional
+# [ step, levelist?, param ]
+# They will be replaced internally by an empty value. It is also posiible to provide a default subtitution value: e.g. [domain?g] will consider the domain to be 'g' if missing.
+
+# * Attributes can be removed:
+# [grid-]
+# This is useful to remove attributes present in the GRIB that should not be ignored
+
+# * Rules are matched:
+
+# - If the attributes are present in the GRIB/Request, or marked optional or ignored
+# - If a list of possible value is provided, one of them must match, for example
+# [ class, expver, stream=enfo/efov, date, time, domain ]
+# will match either stream=enfo or stream=efov, all other attributes will be matched if they exist in the GRIB or user's request
+
+# * On archive:
+# - Attributes are extracted from the GRIB (namespace 'mars'), possibly modified by the attribute type
+# - Only the first rule is used, so order is important
+# - All GRIB attributes must be used by the rules, otherwise an error is raised
+
+# * On retrieve:
+# - Attributes are extracted from the user's request, possibly modified by the attribute type (e.g. for handling of U/V)
+# - All the matching rules are considered
+# - Only attributes listed in the rules are used to extract values from the user's request
+
+
+# Default types
+
+param: Param;
+step: Step;
+date: Date;
+hdate: Date;
+refdate: Date;
+latitude: Double;
+longitude: Double;
+levelist: Double;
+grid: Grid;
+expver: Expver;
+
+time: Time;
+fcmonth: Integer;
+
+number: Integer;
+frequency: Integer;
+direction: Integer;
+channel: Integer;
+
+instrument: Integer;
+ident: Integer;
+
+diagnostic: Integer;
+iteration: Integer;
+system: Integer;
+method: Integer;
+
+
+########################################################
+# These are the rules for the Climate DT
+
+# clte/wave
+[ class=d1, dataset=climate-dt, activity, experiment, generation, model, realization, expver, stream=clte/wave, date
+ [ resolution, type, levtype
+ [ time, levelist?, param, frequency?, direction? ]]
+]
+
+# clmn
+[ class=d1, dataset=climate-dt, activity, experiment, generation, model, realization, expver, stream=clmn, year
+ [ month, resolution, type, levtype
+ [ levelist?, param ]]
+]
+
+########################################################
+# These are the rules for the Extremes DT
+# oper/wave/lwda/lwwv
+[ class=d1, dataset=extremes-dt, expver, stream=oper/wave/lwda/lwwv, date, time
+ [ resolution?, type, levtype
+ [ step, levelist?, param, frequency?, direction? ]]
+]
+########################################################
+# These are the rules for the On-Demand Extremes DT
+# oper/wave
+[ class=d1, dataset=on-demand-extremes-dt, expver, stream=oper/wave, date, time
+ [ type, levtype
+ [ step, levelist?, param, frequency?, direction? ]]
+]
+########################################################
+
+
+########################################################
+#
+# These are the rules for rd
+# oper/wave/lwda/lwwv
+[ class=rd, expver, stream=oper/wave/lwda/lwwv/dcda/enfo, date, time, domain?
+ [ type, levtype
+ [ number?, step, levelist?, param, frequency?, direction? ]]
+]
+
+[ class=rd, expver, stream=mnth, domain?
+ [ type, levtype
+ [ date , time, step?, levelist?, param ]]
+]
\ No newline at end of file
diff --git a/backend/destine_language.yaml b/backend/destine_language.yaml
new file mode 100644
index 0000000..d947a44
--- /dev/null
+++ b/backend/destine_language.yaml
@@ -0,0 +1,1253 @@
+---
+_field: &_field
+
+
+
+# type:
+# values:
+# - [fc, ]
+
+
+
+
+ levtype:
+ values:
+ - [pl, ]
+
+ class:
+ description: Class selects the main category of data to be retrieved such as operational, research or AIFS
+ category: data
+ default: od
+ flatten: false
+ type: enum
+ values:
+ # - [ai, operational aifs]
+ - [d1, Destine data]
+ # - [od, operations]
+ # - [rd, research]
+
+
+ type:
+ category: data
+ default: an
+ flatten: false
+ type: enum
+ multiple: true
+ values:
+ # - [3g, 3d variational gradients]
+ # - [3v, 3d variational analysis]
+ # - [4g, 4d variational gradients]
+ # - [4i, 4d variational increments]
+ # - [4v, 4d variational analysis]
+ # - [ab, analysis bias]
+ # - [af, analysis feedback]
+ # - [ai, analysis input]
+ # - [an, analysis]
+ # - [as, adjoint singular vector]
+ # - [bf, bias-corrected forecast]
+ # - [cd, climate distribution]
+ # - [cf, control forecast]
+ # - [ci, clustering information]
+ # - [cl, climatology]
+ # - [cm, cluster means]
+ # - [cr, cluster representative]
+ # - [cs, cluster std deviations]
+ # - [cv, calibration validation forecast]
+ # - [ea, errors in analysis]
+ # - [ed, empirical distribution]
+ # - [ef, errors in first guess]
+ # - [efi, extreme forecast index]
+ # - [efic, extreme forecast index control]
+ # - [em, ensemble mean]
+ # - [eme, ensemble data assimilation model errors]
+ # - [emtm, ensemble mean of temporal mean]
+ # - [ep, event probability]
+ # - [es, ensemble standard deviation]
+ # - [est, ensemble statistics]
+ # - [estdtm, ensemble standard deviation of temporal mean]
+ # - [fa, forecast accumulation]
+ # - [fb, feedback]
+ - [fc, forecast]
+ # - [fcdfb, forecast departures feedback]
+ # - [fcmax, forecast maximum]
+ # - [fcmean, forecast mean]
+ # - [fcmin, forecast minimum]
+ # - [fcstdev, forecast standard deviation]
+ # - [ff, flux forcing realtime]
+ # - [fg, first guess]
+ # - [fp, forecast probability]
+ # - [fsoifb, forecast sensitivity to observations impact feedback]
+ # - [fu, fill-up]
+ # - [fx, flux forcing]
+ # - [ga, gfas analysis]
+ # - [gbf, bias-corrected gridbox]
+ # - [gai, gridded analysis input]
+ # - [go, gridded observations]
+ # - [gsd, gridded satellite data]
+ # - [gwt, weather type gridbox]
+ # - [hcmean, hindcast mean]
+ # - [ia, init. analysis]
+ # - [icp, initial condition perturbation]
+ # - [mpp, model physics perturbation]
+ # - [if, interim forecast]
+ # - [im, images]
+ # - [me, model errors]
+ # - [mfb, mondb feedback]
+ # - [oai, odb analysis input]
+ # - [ob, observations]
+ # - [of, ocean forward]
+ # - [ofb, odb feedback]
+ # - [oi, oi analysis]
+ # - [oldim, old format images]
+ # - [or, ocean reanalysis]
+ # - [pa, perturbed analysis]
+ # - [pb, probability boundary]
+ # - [pd, probability distribution]
+ - [pf, perturbed forecast]
+ # - [pfc, point values]
+ # - [ppm, point value metrics]
+ # - [s3, climate 30 days simulation]
+ # - [ses, scaled ensemble standard deviation]
+ # - [sf, sensitivity forecast]
+ # - [sfb, summary feedback]
+ # - [sfo, simulations with forcing]
+ # - [sg, sensitivity gradient]
+ # - [si, climate simulation]
+ # - [sim, simulated images]
+ # - [sot, shift of tails]
+ # - [ssd, simulated satellite data]
+ # - [sv, singular vector]
+ # - [svar, signal variance]
+ # - [taem, time average ensemble mean]
+ # - [taes, time average ensemble standard deviation]
+ # - [tpa, time processed analysis]
+ # - [tf, trajectory forecast]
+ # - [tu, tube]
+ # - [wem, weighted ensemble mean]
+ # - [wes, weighted ensemble standard deviation]
+ # - [wp, weather parameters]
+
+ stream:
+ description: Stream selects the kind of data to be retrieved, for example the forecast model or the ensemble model.
+ category: data
+ default: oper
+ flatten: false
+ type: enum
+ values:
+ - [clte, climate, Climate run output] # climate-dt
+ - [oper, da, daily archive, atmospheric model] # climate-dt / extremes-dt / on-demand-extremes-dt
+ - [wave, wv, wave model] # climate-dt / extremes-dt
+ - [lwda, long window daily archive] # extremes-dt
+ - [lwwv, long window wave] # extremes-dt
+ - [clmn, climate-monthly, Climate run monthly means output] # climate-dt
+
+ # - [amap, analysis for multianalysis project]
+ # - [ammc, melbourne]
+ # - [cher, ch, chernobyl]
+
+ # - [cnrm, meteo france climate centre]
+ # - [cwao, montreal]
+ # - [dacl, daily climatology]
+ # - [dacw, daily climatology wave]
+ # - [dahc, daily archive hindcast]
+ # - [dcda, atmospheric model (delayed cutoff)]
+ # - [dcwv, wave model (delayed cutoff)]
+ # - [edmm, ensemble data assimilation monthly means]
+ # - [edmo, ensemble data assimilation monthly means of daily means]
+ # - [edzw, offenbach]
+ # - [eefh, extended ensemble forecast hindcast]
+ # - [eefo, extended ensemble prediction system]
+ # - [eehs, extended ensemble forecast hindcast statistics]
+ # - [efas, european flood awareness system (efas)]
+ # - [efcl, european flood awareness system (efas) climatology]
+ # - [efhc, ensemble forecast hindcasts (obsolete)]
+ # - [efho, ensemble forecast hindcast overlap]
+ # - [efhs, ensemble forecast hindcast statistics]
+ # - [efov, ensemble forecast overlap]
+ # - [efrf, european flood awareness system (efas) reforecasts]
+ # - [efse, european flood awareness system (efas) seasonal forecasts]
+ # - [efsr, european flood awareness system (efas) seasonal reforecasts]
+ # - [egrr, exeter, bracknell]
+ # - [ehmm, combined multi-model hindcast monthly means]
+ # - [elda, ensemble long window data assimilation]
+ # - [enda, ensemble data assimilation]
+ # - [enfh, ensemble forecast hindcasts]
+ # - [enfo, ef, ensemble prediction system]
+ # - [enwh, ensemble forecast wave hindcasts]
+ # - [esmm, combined multi-model monthly means]
+ # - [espd, ensemble supplementary data]
+ # - [ewda, ensemble wave data assimilation]
+ # - [ewhc, wave ensemble forecast hindcast (obsolete)]
+ # - [ewho, ensemble forecast wave hindcast overlap]
+ # - [ewla, ensemble wave long window data assimilation]
+ # - [ewmm, ensemble wave data assimilation monthly means]
+ # - [ewmo, ensemble wave data assimilation monthly means of daily means]
+ # - [fgge, fg]
+ # - [fsob, forecast sensitivity to observations]
+ # - [fsow, forecast sensitivity to observations wave]
+ # - [gfas, global fire assimilation system]
+ # - [gfra, global fire assimilation system reanalysis]
+ # - [kwbc, washington]
+ # - [lfpw, paris, toulouse]
+ # - [ma, means archive]
+ # - [maed, multianalysis ensemble data]
+ # - [mawm, wave anomaly means]
+ # - [mawv, multianalysis wave data]
+ # - [mdfa, monthly means of daily forecast accumulations]
+ # - [mfam, anomaly means]
+ # - [mfaw, wave anomalies]
+ # - [mfhm, hindcast means]
+ # - [mfhw, monthly forecast hindcasts wave]
+ # - [mfwm, wave real-time means]
+ # - [mhwm, wave hindcast means]
+ # - [mmaf, multi-model multi-annual forecast]
+ # - [mmam, multi-model multi-annual forecast means]
+ # - [mmaw, multi-model multi-annual forecast wave]
+ # - [mmsa, multi-model seasonal forecast monthly anomalies]
+ # - [mmsf, multi-model seasonal forecast]
+ # - [mmwm, multi-model multi-annual forecast wave means]
+ # - [mnfa, anomalies]
+ # - [mnfc, real-time]
+ # - [mnfh, hindcasts]
+ # - [mnfm, real-time means]
+ # - [mnfw, wave real-time]
+ # - [mnth, mo, monthly, monthly means]
+ # - [mnvr, monthly variance and covariance data using g. boer's step function]
+ # - [moda, monthly means of daily means]
+ # - [mofc, monthly forecast]
+ # - [mofm, monthly forecast means]
+ # - [monr, monthly means using g. boer's step function]
+ # - [mpic, max plank institute]
+ # - [msda, monthly standard deviation and covariance of daily means]
+ # - [msdc, mv, monthly standard deviation and covariance]
+ # - [msmm, multi-model seasonal forecast atmospheric monthly means]
+ # - [mswm, multi-model seasonal forecast wave monthly means]
+ # - [ocda, ocean data assimilation]
+ # - [ocea, ocean]
+ # - [olda, ocean Long window data assimilation]
+ # - [rjtd, tokyo]
+ # - [scda, atmospheric model (short cutoff)]
+ # - [scwv, wave model (short cutoff)]
+ # - [seap, sensitive area prediction]
+ # - [seas, seasonal forecast]
+ # - [sens, sf, sensitivity forecast]
+ # - [sfmm, seasonal forecast atmospheric monthly means]
+ # - [smma, seasonal monthly means anomalies]
+ # - [supd, sd, deterministic supplementary data]
+ # - [swmm, seasonal forecast wave monthly means]
+ # - [toga, tg]
+ # - [ukmo, ukmo climate centre]
+ # - [waef, we, wave ensemble forecast]
+ # - [wamd, wave monthly means of daily means]
+ # - [wamf, wave monthly forecast]
+ # - [wamo, wave monthly means]
+ # - [wams, multi-model seasonal forecast wave]
+ # - [wasf, wave seasonal forecast]
+ # - [wavm, wave model (standalone)]
+ # - [weef, wave extended ensemble forecast]
+ # - [weeh, wave extended ensemble forecast hindcast]
+ # - [wees, wave extended ensemble forecast hindcast statistics]
+ # - [wehs, wave ensemble forecast hindcast statistics]
+ # - [weov, wave ensemble forecast overlap]
+ # - [wfas, global flood awareness system (glofas)]
+ # - [wfcl, global flood awareness system (glofas) climatology]
+ # - [wfrf, global flood awareness system (glofas) reforecasts]
+ # - [wfse, global flood awareness system (glofas) seasonal forecasts]
+ # - [wfsr, global flood awareness system (glofas) seasonal reforecasts]
+ # - [wmfm, wave monthly forecast means]
+ # - [wvhc, wave hindcast]
+ expver:
+ description: Experiment number, 0001 is operational data.
+ category: data
+ default: '0001'
+ flatten: false
+ type: enum
+ values:
+ - ['0001', 'Operational Data']
+ - ['xxxx', 'Experimental Data']
+ - ['xxxy', 'Experimental Data']
+
+ dataset:
+ # flatten: false
+ multiple: true
+ type: enum
+ values:
+ - [climate-dt, Climate Data]
+ - [extremes-dt, Extremes Data]
+ - [on-demand-extremes-dt, On-Demand Extremes Data]
+
+ model:
+ category: data
+ type: lowercase
+
+ repres:
+ flatten: false
+ multiple: true
+ type: enum
+ values:
+ - gg
+ - sh
+ - ll
+ - np
+ - rl
+
+ obsgroup:
+ category: data
+ multiple: true
+ type: enum
+ values:
+ # - [conventional]
+ - [sat, satellite]
+ - [ers1]
+ - [trmm]
+ - [qscat]
+ - [reo3] # reo3 needs to stay for compatibility
+ # previously in "obsgroups.def"
+ - [hirs, 1, HIRS ]
+ - [amsua, 2, AMSUA ]
+ - [amsub, 3, AMSUB ]
+ - [mhs, 4, MHS ]
+ - [geos, 5, GEOS ]
+ - [resat, 6, RESAT ]
+ - [meris, 7, MERIS ]
+ - [gpsro, 8, GPSRO ]
+ - [satob, 9, SATOB ]
+ - [scatt, 10, SCATT ]
+ - [ssmi_as, 11, SSMI ALL-SKY ]
+ - [iasi, 12, IASI ]
+ - [airs, 13, AIRS ]
+ - [ssmis_as, 14, SSMIS ALL-SKY ]
+ - [tmi_as, 15, TMI ALL-SKY ]
+ - [amsre_as, 16, AMSRE ALL-SKY ]
+ - [conv, 17, CONV ]
+ - [smos, 19, SMOS ]
+ - [windsat_as, 20, WINDSAT ALL-SKY ]
+ - [ssmi, 21, SSMI ]
+ - [amsua_as, 22, AMSUA ALL-SKY ]
+ - [amsre, 23, AMSRE ]
+ - [tmi, 24, TMI ]
+ - [ssmis, 25, SSMIS ]
+ - [gbrad, 26, GBRAD ]
+ - [mwhs, 27, MWHS ]
+ - [mwts, 28, MWTS ]
+ - [mwri_as, 29, MWRI ALL-SKY ]
+ - [iras, 30, IRAS ]
+ - [msu, 31, MSU ]
+ - [ssu, 32, SSU ]
+ - [vtpr1, 33, VTPR1 ]
+ - [vtpr2, 34, VTPR2 ]
+ - [atms, 35, ATMS ]
+ - [resat_ak, 36, RESAT AVERAGING KERNELS ]
+ - [cris, 37, CRIS ]
+ - [wave_ip, 38, WAVE INTEGRATED PARAMETERS ]
+ - [wave_sp, 39, WAVE SPECTRA ]
+ - [raingg, 40, RAINGG ]
+ - [sfc_ms, 41, SURFACE MULTISENSOR ]
+ - [amsr2_as, 42, AMSR-2 ALL-SKY ]
+ - [saphir_as, 43, SAPHIR ALL-SKY ]
+ - [amsub_as, 44, AMSUB ALL-SKY ]
+ - [mhs_as, 45, MHS ALL-SKY ]
+ - [dwl, 46, DOPPLER WIND LIDAR ]
+ - [iris, 47, IRIS ]
+ - [aatsr, 49, AATSR ]
+ - [atms_as, 50, ATMS ALL-SKY ]
+ - [gmi_as, 51, GMI ALL-SKY ]
+ - [godae_sst, 52, GODAE SEA SURFACE TEMPERATURES ]
+ - [atovs_ms, 53, ATOVS MULTISENSOR ]
+ - [atmospheric_composition, 54, ATMOSPHERIC COMPOSITION ]
+ - [non_sfc_ms, 55, NON-SURFACE MULTISENSOR ]
+ - [mwts2, 56, MWTS2 ]
+ - [ssmi_1d, 57, SSMI 1DVAR TCWV CLOUDY-SKY ]
+ - [mwhs2_as, 58, MWHS2 ALL-SKY ]
+ - [ssmt2, 59, SSMT2 ]
+ - [smap, 60, SMAP ]
+ - [tovs_ms, 61, TOVS MULTISENSOR ]
+ - [cloud_r, 62, CLOUD REFLECTIVITY ]
+ - [cloud_l, 63, CLOUD LIDAR ]
+ - [satellite_lightning, 64, SATELLITE LIGHTNING ]
+ - [geos_vis, 65, GEOS VIS ]
+ - [oconv, 66, OCONV ]
+ - [mwts3_as, 67, MWTS3 All-sky ]
+ - [giirs, 68, GIIRS ]
+ - [test, 99, TEST ]
+
+ reportype:
+ category: data
+ type: any
+ multiple: true
+
+ # rdbprefix
+
+ levtype:
+ description: The Level Type, can be pressure levels, the surface, model levels etc.
+ category: data
+ default: pl
+ flatten: false
+ type: enum
+ values:
+ # - [cat, category]
+ # - [dp, depth]
+ # - [layer]
+ # - [ml, model levels]
+ - [pl, pressure levels]
+ # - [hl, height levels]
+ # - [pt, potential temperature]
+ # - [pv, potential vorticity]
+ - [sfc, surface]
+ # - [sol, surface other (multi)levels]
+ # - [wv, ocean wave]
+ # - [o2d, ocean surface]
+ # - [o3d, ocean model levels]
+ never:
+ - type: ssd
+
+ levelist:
+ category: data
+ multiple: true
+ by: 1
+ default:
+ - 1000
+ - 850
+ - 700
+ - 500
+ - 400
+ - 300
+ never:
+ - levtype: [sfc, o2d]
+ - type: ssd
+ type: enum
+ values:
+ - [1, ]
+ - [5, ]
+ - [10, ]
+ - [20, ]
+ - [30, ]
+ - [50, ]
+ - [70, ]
+ - [100, ]
+ - [150, ]
+ - [200, ]
+ - [250, ]
+ - [300, ]
+ - [400, ]
+ - [500, ]
+ - [600, ]
+ - [700, ]
+ - [850, ]
+ - [925, ]
+ - [1000, ]
+
+ param:
+ category: data
+ default: 129
+ multiple: true
+ type: param
+ never:
+ - type: [tf, ob]
+ values:
+ - [60, "Potential vorticity"]
+ - [129, "Geopotential"]
+ - [130, "Temperature"]
+ - [131, "U component of wind"]
+ - [132, "V component of wind"]
+ - [133, "Specific humidity"]
+ - [135, "Vertical velocity"]
+ - [157, "Relative humidity"]
+ - [246, "Specific cloud liquid water content"]
+
+
+#################################################################
+
+ # year
+ # decade
+ # month
+
+ date:
+ category: data
+ default: 0
+ type: enum
+ multiple: true
+ values:
+ - [20211021, ]
+
+ year:
+ category: data
+ type: to-by-list
+ multiple: true
+ by: 1
+
+ month:
+ category: data
+ flatten: true
+ type: enum
+ multiple: true
+ values:
+ - [1, jan, January]
+ - [2, feb, February]
+ - [3, mar, March]
+ - [4, apr, April]
+ - [5, may, May]
+ - [6, jun, June]
+ - [7, jul, July]
+ - [8, aug, August]
+ - [9, sep, September]
+ - [10, oct, October]
+ - [11, nov, November]
+ - [12, dec, December]
+
+ # verify
+ # refdate
+
+ hdate:
+ category: data
+ multiple: true
+ only:
+ - stream:
+ - enfh
+ - enwh
+ - efho
+ - ehmm
+ - ewho
+ - eefh
+ - weeh
+ type: integer
+
+ offsetdate:
+ category: data
+ multiple: true
+ type: date
+
+ fcmonth:
+ category: data
+ multiple: true
+ by: 1
+ type: to-by-list
+
+ fcperiod:
+ category: data
+ multiple: true
+ type: integer
+
+ time:
+ category: data
+ default: '1200'
+ multiple: true
+ type: enum
+ values:
+ - ["0000", ]
+ - ["0100", ]
+ - ["0200", ]
+ - ["0300", ]
+ - ["0400", ]
+ - ["0500", ]
+ - ["0600", ]
+ - ["0700", ]
+ - ["0800", ]
+ - ["0900", ]
+ - ["1000", ]
+ - ["1100", ]
+ - ["1200", ]
+ - ["1300", ]
+ - ["1400", ]
+ - ["1500", ]
+ - ["1600", ]
+ - ["1700", ]
+ - ["1800", ]
+ - ["1900", ]
+ - ["2000", ]
+ - ["2100", ]
+ - ["2200", ]
+ - ["2300", ]
+
+ offsettime:
+ category: data
+ multiple: true
+ type: time
+
+ # leadtime
+ # opttime
+ # range
+
+ step:
+ description: Specify which forecast we want in hours past the date/time.
+ category: data
+ multiple: true
+ by: 12
+ default: 0
+ type: range
+ never:
+ - dataset:
+ - climate-dt
+ - stream:
+ - msmm
+ - mmsa
+ - swmm
+
+ anoffset:
+ category: data
+ multiple: true
+ type: integer
+
+ reference:
+ category: data
+ multiple: true
+ type: integer
+
+#################################################################
+
+ # cluster
+ # probability
+
+ number:
+ description: Selects a subset of ensemble members
+ category: data
+ multiple: true
+ aliases:
+ - ensemble
+ by: 1
+ only:
+ - type: [pf, cr, cm, fcmean, fcmin, fcmax, fcstdev, sot, fc, wp, 4i, 4v]
+ never:
+ # This is to prevent number with type=fc and stream=oper
+ - stream: [oper, wave]
+ type: to-by-list
+
+ quantile:
+ category: data
+ multiple: true
+ only:
+ - type:
+ - pd
+ - pb
+ - taem
+ - cd
+ # - sot
+ type: to-by-list-quantile
+ denominators: [2,3,4,5,10,100,1000]
+ by: 1
+ domain:
+ description: The large scale geographic region.
+ category: data
+ default: g
+ flatten: false
+ type: enum
+ never:
+ - dataset:
+ - climate-dt
+ values:
+ # - [a, north west europe]
+ # - [b, north east europe, baltic and black sea]
+ - [c, south west europe]
+ - [d, south east europe]
+ - [e, europe]
+ # - [f, fastex]
+ - [g, globe, general european area]
+ # - [h]
+ # - [i]
+ # - [j]
+ # - [k]
+ # - [l]
+ # - [m, mediterranean]
+ # - ['n', northern hemisphere]
+ # - [o]
+ # - [p]
+ # - [q]
+ # - [r]
+ # - [s, southern hemisphere]
+ # - [t, tropics]
+ # - [u, tropics 2]
+ # - [v]
+ # - [w, western atlantic]
+ # - [x]
+ # - ['y']
+ # - [z]
+
+ frequency:
+ category: data
+ multiple: true
+ by: 1
+ only:
+ - param:
+ - '140251'
+ type: to-by-list
+
+ direction:
+ category: data
+ multiple: true
+ by: 1
+ only:
+ - param:
+ - '140251'
+ type: to-by-list
+
+ diagnostic:
+ category: data
+ type: integer
+ multiple: true
+
+ iteration:
+ category: data
+ type: integer
+ multiple: true
+
+ channel:
+ category: data
+ only:
+ - type: ssd
+ type: integer
+ multiple: true
+
+ ident:
+ category: data
+ only:
+ - type: ssd
+ type: integer
+ multiple: true
+
+ instrument:
+ category: data
+ only:
+ - type: ssd
+ type: integer
+ multiple: true
+
+ method:
+ category: data
+ type: integer
+
+ origin:
+ category: data
+ multiple: true
+ type: enum
+ values:
+ - [ammc, 1, melbourne]
+ - [babj, 38, beijing]
+ - [cmcc]
+ - [cnmc, 80]
+ - [consensus, 255]
+ - [crfc, 239, cerfacs]
+ - [cwao, 54, montreal]
+ - [ecmf, 98, ecmwf]
+ - [edzw, dwd, 78, offenbach]
+ - [egrr, 74, exeter, bracknell]
+ - [enmi, 88, oslo]
+ - [fnmo, fnmoc, 58, fleet numerical]
+ - [hadc, 247, hadley centre]
+ - [ifmk, 246]
+ - [ingv, 235]
+ - [knmi, 245]
+ - [kwbc, 7, washington]
+ - [lemm, 214, madrid]
+ - [lfpw, 84, 85, paris, toulouse]
+ - [rjtd, 34, tokyo]
+ - [rksl, 40, seoul]
+ - [sbsj, 46, cptec]
+ - [vuwien, 244, university of vienna]
+
+ system:
+ category: data
+ type: integer
+
+#######################################################################
+# DestinE ClimateDT related keywords
+
+
+ model:
+ type: enum
+ description: Which climate model to use.
+ values:
+ - [ifs-fesom, Integrated Forecast System - FESOM]
+
+ activity:
+ category: data
+ type: enum
+ values:
+ - [story-nudging, ]
+
+ experiment:
+ category: data
+ type: enum
+ values:
+ - [tplus2.0k, ]
+
+ generation:
+ category: data
+ type: enum
+ values:
+ - [1, ]
+
+ realization:
+ category: data
+ type: integer
+ values:
+ - [1, ]
+
+ resolution:
+ category: data
+ type: enum
+ values:
+ - [standard, ]
+
+#######################################################################
+
+_observation: &_observation
+
+ obstype:
+ category: data
+ type: any
+ multiple: true
+
+ obsgroup:
+ category: data
+ type: any
+ multiple: true
+
+#######################################################################
+
+_postproc: &_postproc
+
+ accuracy:
+ category: postproc
+ flatten: false
+ type: [enum, integer]
+ values:
+ - [av]
+ - ['off', normal, auto]
+
+ bitmap:
+ category: postproc
+ flatten: false
+ type: any
+
+ format:
+ category: postproc
+ flatten: false
+ type: enum
+ values:
+ - - grib
+ - grib1
+ - gb
+ - - grib2
+ - - bufr
+ - bf
+ - - grid
+ - gd
+ - odb
+ - ascii
+
+ frame:
+ category: postproc
+ type: integer
+
+ gaussian:
+ category: postproc
+ type: enum
+ values:
+ - reduced
+ - regular
+
+ area:
+ category: postproc
+ flatten: false
+ multiple: true
+ type: [float, enum]
+ values:
+ - ['off', g, global]
+ - [e, europe]
+
+ grid:
+ category: postproc
+ flatten: false
+ multiple: true
+ type: [enum, float, regex]
+ values:
+ - auto
+ - N16
+ - N24
+ - N32
+ - N48
+ - N64
+ - N80
+ - N96
+ - N128
+ - N160
+ - N200
+ - N256
+ - N320
+ - N400
+ - N512
+ - N576
+ - N640
+ - N800
+ - N1024
+ - N1280
+ - N1600
+ - N2000
+ - N4000
+ - N8000
+ regex:
+ - '^[oOfF][1-9][0-9]+$'
+ uppercase: true
+
+ interpolation:
+ category: postproc
+ flatten: false
+ type: enum
+ values:
+ - - linear
+ - - nearest-lsm
+ - nearest lsm
+ - - 'off'
+ - default
+ - any
+
+ packing:
+ category: postproc
+ flatten: false
+ type: enum
+ values:
+ - - so
+ - second order
+ - ['off', av]
+ - [co, complex]
+ - simple
+ - ccsds
+
+ resol:
+ category: postproc
+ flatten: false
+ aliases:
+ - tra
+ type: [enum, integer]
+ values:
+ - - 'off'
+ - av
+ - reduced gaussian 160
+
+ rotation:
+ category: postproc
+ flatten: false
+ multiple: true
+ type: float
+
+ intgrid:
+ category: postproc
+ flatten: false
+ type: [enum, regex]
+ values:
+ - 'off'
+ - auto
+ - N32
+ - N48
+ - N64
+ - N80
+ - N96
+ - N128
+ - N160
+ - N192
+ - N200
+ - N256
+ - N320
+ - N400
+ - N512
+ - N640
+ - N800
+ - N912
+ - N1024
+ - N1280
+ regex:
+ - '^[oOfF][1-9][0-9]+$'
+
+ truncation:
+ category: postproc
+ flatten: false
+ type: [enum, integer]
+ values:
+ - auto
+ - 'off'
+#######################################################################
+
+_obspproc: &_obspproc
+
+ filter:
+ type: any
+ category: postproc
+
+ ident:
+ type: any
+ category: postproc
+
+
+#######################################################################
+
+disseminate:
+ <<: *_field
+ <<: *_postproc
+
+ requirements:
+ type: any
+
+ use:
+ category: data
+ flatten: false
+ multiple: true
+ type: enum
+ values:
+ - bc
+ - monday
+ - tuesday
+ - wednesday
+ - thursday
+ - friday
+ - saturday
+ - sunday
+
+ option:
+ default: normal
+ flatten: false
+ multiple: true
+ type: enum
+ values:
+ - normal
+ - delay
+ - asap
+ - gts
+ - opendata
+
+ compatibility:
+ category: postproc
+ flatten: false
+ multiple: true
+ type: enum
+ values:
+ - 'off'
+ - 'no-local-extension'
+
+ priority:
+ flatten: false
+ type: integer
+
+ target:
+ flatten: false
+ type: any
+
+##############################################################
+
+archive:
+ <<: *_field
+ <<: *_observation
+
+
+ database:
+ flatten: false
+ multiple: true
+ type: any
+
+ source:
+ flatten: false
+ multiple: true
+ type: any
+
+ expect:
+ flatten: false
+ multiple: false
+ type: integer
+
+##############################################################
+
+retrieve:
+
+ <<: *_field
+ <<: *_observation
+ <<: *_postproc
+ <<: *_obspproc
+
+ target:
+ flatten: false
+ multiple: true
+ type: any
+
+ expect:
+ flatten: false
+ multiple: false
+ type: integer
+
+ fieldset:
+ flatten: false
+ multiple: false
+ type: any
+
+ database:
+ flatten: false
+ multiple: true
+ type: any
+
+ optimise:
+ type: enum
+ values:
+ - true
+ - false
+ default:
+ "off"
+
+ padding:
+ flatten: false
+ type: enum
+ values:
+ - none
+ - auto
+
+##############################################################
+
+read:
+ source:
+ flatten: false
+ multiple: true
+ type: any
+
+ <<: *_field
+ <<: *_observation
+ <<: *_postproc
+ <<: *_obspproc
+
+ target:
+ flatten: false
+ multiple: true
+ type: any
+
+ fieldset:
+ flatten: false
+ multiple: false
+ type: any
+
+ _defaults:
+ class: null
+ date: null
+ domain: null
+ expver: null
+ levelist: null
+ levtype: null
+ param: null
+ step: null
+ stream: null
+ time: null
+ type: null
+
+ _options:
+ param:
+ # expand_with: # In case not type/stream/levtype is provided
+ # type: an
+ # stream: oper
+ # levtype: pl
+ first_rule: true
+##############################################################
+
+get:
+
+ tape:
+ flatten: false
+ multiple: false
+ type: any
+
+ database:
+ flatten: false
+ multiple: true
+ type: any
+
+ target:
+ flatten: false
+ multiple: true
+ type: any
+
+##############################################################
+
+list:
+
+ <<: *_field
+ <<: *_observation
+
+ database:
+ flatten: false
+ multiple: true
+ type: any
+
+ target:
+ flatten: false
+ multiple: true
+ type: any
+
+ _defaults:
+ # class: null
+ date: null
+ domain: null
+ expver: null
+ levelist: null
+ levtype: null
+ param: null
+ step: null
+ stream: null
+ time: null
+ type: null
+
+##############################################################
+
+compute:
+ formula:
+ flatten: false
+ multiple: false
+ type: any
+
+ fieldset:
+ flatten: false
+ multiple: false
+ type: any
+
+##############################################################
+
+write:
+
+ fieldset:
+ flatten: false
+ multiple: false
+ type: any
+
+ target:
+ flatten: false
+ multiple: true
+ type: any
+
+##############################################################
+
+pointdb:
+ lat:
+ multiple: false
+ type: float
+
+ lon:
+ multiple: false
+ type: float
+
+ <<: *_field
+
+ _defaults:
+ class: null
+ date: null
+ domain: null
+ expver: null
+ levelist: null
+ levtype: null
+ param: null
+ step: null
+ stream: null
+ time: null
+ type: null
+
+ _options:
+ param:
+ # expand_with: # In case not type/stream/levtype is provided
+ # type: an
+ # stream: oper
+ # levtype: pl
+ first_rule: true
+
+end: {}
diff --git a/backend/main.py b/backend/main.py
index 7f31a7a..ea3ca85 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -1,10 +1,21 @@
from collections import defaultdict
from typing import Any, Dict
+import yaml
+import os
from fastapi import FastAPI, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from fdb_schema import FDBSchemaFile
+from fastapi.responses import RedirectResponse
+from fastapi.templating import Jinja2Templates
+
+import os
+os.environ["FDB5_CONFIG_FILE"] = "/home/eouser/destine_remoteFDB_config.yaml"
+
+import pyfdb
+
+fdb = pyfdb.FDB()
app = FastAPI()
app.add_middleware(
@@ -16,17 +27,30 @@ app.add_middleware(
)
app.mount("/app", StaticFiles(directory="../webapp"), name="static")
+templates = Jinja2Templates(directory="../webapp")
+
+config = {
+ "message": "",
+ "fdb_schema": "standard_fdb_schema",
+ "mars_language": "language.yaml"
+}
+if os.path.exists("../config.yaml"):
+ with open("../config.yaml", "r") as f:
+ config = config | yaml.safe_load(f)
+
+
+@app.get("/")
+async def redirect_to_app(request: Request):
+ return templates.TemplateResponse("index.html", {"request": request, "config": config})
-language_yaml = "./language.yaml"
import yaml
-with open(language_yaml, "r") as f:
+with open(config["mars_language"], "r") as f:
mars_language = yaml.safe_load(f)["_field"]
###### Load FDB Schema
-schema = FDBSchemaFile("./standard_fdb_schema")
-# schema = FDBSchemaFile("./test_schema")
+schema = FDBSchemaFile(config["fdb_schema"])
def request_to_dict(request: Request) -> Dict[str, Any]:
# Convert query parameters to dictionary format
@@ -37,42 +61,6 @@ def request_to_dict(request: Request) -> Dict[str, Any]:
request_dict[key] = value.split(",")
return request_dict
-@app.get("/simple")
-async def get_tree(request: Request):
- request_dict = request_to_dict(request)
- print(request_dict)
- target = next((k for k,v in request_dict.items() if v == "????"), None)
- if not target:
- return {"error": "No target found in request, there must be one key with value '????'"}
-
- current_query_params = "&".join(f"{k}={v}" for k, v in request_dict.items() if k != target)
- if len(current_query_params) > 1:
- current_query_params += "&"
-
- stac_collection = {
- "type": "Collection",
- "stac_version": "1.0.0",
- "id": target,
- "title" : target.capitalize(),
- "key_type": mars_language.get(target, {}).get("type", ""),
- "description": mars_language.get(target, {}).get("description", ""),
- "values": mars_language.get(target, {}).get("values", ""),
- "links": [
- {
- "title": str(value[-1] if isinstance(value, list) else value),
- "href": f"/tree?{current_query_params}{target}={value[0] if isinstance(value, list) else value}",
- "rel": "child",
- "type": "application/json",
-
- }
-
- for value in mars_language.get(target, {}).get("values", [])
- ]
- }
-
- return stac_collection
-
-
@app.get("/tree")
async def get_tree(request: Request):
# Convert query parameters to dictionary format
@@ -97,24 +85,53 @@ async def get_tree(request: Request):
def make_link(key_name, paths):
"""Take a MARS Key and information about which paths matched up to this point and use it to make a STAC Link"""
first_path = [str(p) for p in paths[0]]
- href = f"/simple?{'&'.join(first_path)}{'&' if first_path else ''}{key_name}=????"
+ href_template = f"/tree?{'&'.join(first_path)}{'&' if first_path else ''}{key_name}={{}}"
optional = [p[-1].key_spec.is_optional() for p in paths if len(p) > 0]
optional_str = "Yes" if all(optional) and len(optional) > 0 else ("Sometimes" if any(optional) else "No")
+ values_from_mars_language = mars_language.get(key_name, {}).get("values", [])
+ values = [v[0] if isinstance(v, list) else v for v in values_from_mars_language]
+
+ if all(isinstance(v, list) for v in values_from_mars_language):
+ value_descriptions = [v[-1] if len(v) > 1 else "" for v in values_from_mars_language]
+ else:
+ value_descriptions = [""] * len(values)
return {
"title": key_name,
- "optional": optional_str,
- # "optional_by_path": optional,
- "href": href,
+ "generalized_datacube:href_template": href_template,
"rel": "child",
"type": "application/json",
- "paths": set(tuple(f"{m.key}={m.value}" for m in p) for p in paths),
- # "description": mars_language.get(key_name, {}).get("description", ""),
- # "values": mars_language.get(key_name, {}).get("values", "")
+ "generalized_datacube:dimension" : {
+ "type" : mars_language.get(key_name, {}).get("type", ""),
+ "description": mars_language.get(key_name, {}).get("description", ""),
+ "values" : values,
+ "value_descriptions" : value_descriptions,
+ "optional" : any(optional),
+ "multiple": True,
+ }
+
+
+ # "paths": set(tuple(f"{m.key}={m.value}" for m in p) for p in paths),
}
+ def value_descriptions(key, values):
+ return {
+ v[0] : v[-1] for v in mars_language.get(key, {}).get("values", [])
+ if len(v) > 1 and v[0] in values
+ }
+
+ descriptions = {
+ key : {
+ "key" : key,
+ "values" : values,
+ "description" : mars_language.get(key, {}).get("description", ""),
+ "value_descriptions" : value_descriptions(key,values),
+ }
+ for key, values in request_dict.items()
+ }
+
# Format the response as a STAC collection
stac_collection = {
"type": "Collection",
@@ -124,7 +141,19 @@ async def get_tree(request: Request):
"links": [
make_link(key_name, paths)
for key_name, paths in key_frontier.items()
- ]
+ ],
+ "debug": {
+ "request": request_dict,
+ "descriptions": descriptions,
+ "matches" : matches,
+ # "paths" : [
+ # {
+ # "path" : {o.key : o.str_value() for o in path},
+ # "list" : [i["keys"] for i in fdb.list({o.key : o.str_value() for o in path}, keys=True)],
+ # "key" : key,
+ # } for key, paths in key_frontier.items() for path in paths
+ # ]
+ }
}
return stac_collection
\ No newline at end of file
diff --git a/webapp/app.js b/backend/webapp/app.js
similarity index 56%
rename from webapp/app.js
rename to backend/webapp/app.js
index 247778f..78d8817 100644
--- a/webapp/app.js
+++ b/backend/webapp/app.js
@@ -65,26 +65,24 @@ function goToNextUrl() {
const key_type = item.dataset.keyType;
let values = [];
- if (key === "date") {
- const datePicker = item.querySelector("input[type='date']");
- //format date as YYYYMMDD
+ const datePicker = item.querySelector("input[type='date']");
+ if (datePicker) {
values.push(datePicker.value.replace(/-/g, ""));
- } else if (key === "time") {
- const timePicker = item.querySelector("input[type='time']");
- //format time as HHMM
- console.log("replace", timePicker.value.replace(":", ""));
+ }
+
+ const timePicker = item.querySelector("input[type='time']");
+ if (timePicker) {
values.push(timePicker.value.replace(":", ""));
- } else if (key_type === "enum") {
- values.push(
- ...Array.from(
- item.querySelectorAll("input[type='checkbox']:checked")
- ).map((checkbox) => checkbox.value)
- );
- } else {
- const any = item.querySelector("input[type='text']");
- if (any.value !== "") {
- values.push(any.value);
- }
+ }
+
+ const enum_checkboxes = item.querySelectorAll("input[type='checkbox']:checked");
+ if (enum_checkboxes.length > 0) {
+ values.push(...Array.from(enum_checkboxes).map((checkbox) => checkbox.value));
+ }
+
+ const any = item.querySelector("input[type='text']");
+ if (any && any.value !== "") {
+ values.push(any.value);
}
// Keep track of whether any new keys are selected
@@ -127,53 +125,27 @@ async function createCatalogItem(link, itemsContainer) {
itemsContainer.appendChild(itemDiv);
try {
- // Fetch details for each item/collection asynchronously
- let base_url = new URL(window.location.href);
- base_url.pathname = "/tree";
- let url = new URL(link.href, base_url);
- console.log("Fetching item details:", url);
- const response = await fetch(url);
- const itemData = await response.json();
// Update the item div with real content
itemDiv.classList.remove("loading");
- itemDiv.innerHTML = ""; // Clear "Loading..." text
+
+ const dimension = link["generalized_datacube:dimension"];
// add data-key attribute to the itemDiv
- itemDiv.dataset.key = itemData.id;
- itemDiv.dataset.keyType = itemData.key_type;
+ itemDiv.dataset.key = link.title;
+ itemDiv.dataset.keyType = dimension.type;
+
+ itemDiv.innerHTML = `
+ ${link.title || "No title available"}
+ Key Type: ${itemDiv.dataset.keyType || "Unknown"}
+ Optional: ${dimension.optional ? "Yes" : "No"}
+ ${dimension.description ? dimension.description.slice(0, 100) : "No description available"}...
+ `;
- const title = document.createElement("h3");
- title.className = "item-title";
- title.textContent = itemData.title || "No title available";
- itemDiv.appendChild(title);
- const key_type = document.createElement("p");
- key_type.className = "item-type";
- key_type.textContent = `Key Type: ${itemData.key_type || "Unknown"}`;
- itemDiv.appendChild(key_type);
-
- const optional = document.createElement("p");
- optional.className = "item-type";
- optional.textContent = `Optional: ${link.optional || "Unknown"}`;
- itemDiv.appendChild(optional);
-
- // const id = document.createElement("p");
- // id.className = "item-id";
- // id.textContent = `ID: ${itemData.id || link.href.split("/").pop()}`;
- // itemDiv.appendChild(id);
-
- const description = document.createElement("p");
- description.className = "item-description";
- const descText = itemData.description
- ? itemData.description.slice(0, 100)
- : "No description available";
- description.textContent = `${descText}...`;
- itemDiv.appendChild(description);
-
- if (itemData.key_type === "date" || itemData.key_type === "time") {
+ if (dimension.type === "date" || dimension.type === "time") {
// Render a date picker for the "date" key
- const picker = ``;
+ const picker = ``;
//convert picker to HTML node
const pickerNode = document
.createRange()
@@ -182,25 +154,49 @@ async function createCatalogItem(link, itemsContainer) {
}
// Otherwise create a scrollable list with checkboxes for values if available
else if (
- itemData.key_type === "enum" &&
- itemData.values &&
- itemData.values.length > 0
+ // dimension.type === "enum" &&
+ dimension.values &&
+ dimension.values.length > 0
) {
- const listContainer = renderCheckboxList(itemData);
+ const listContainer = renderCheckboxList(link);
itemDiv.appendChild(listContainer);
} else {
- const any = ``;
+ const any = ``;
const anyNode = document.createRange().createContextualFragment(any);
itemDiv.appendChild(anyNode);
}
} catch (error) {
console.error("Error loading item data:", error);
-
- // In case of an error, display an error message
- itemDiv.innerHTML = "Error loading item details
";
+ itemDiv.innerHTML = `Error loading item details: ${error}
`;
}
}
+function renderCheckboxList(link) {
+ const dimension = link["generalized_datacube:dimension"];
+ const value_descriptions = dimension.value_descriptions;
+
+ const listContainerHTML = `
+
+
+
+
-
-
-
- `;
- })
- .join("");
-
- scrollableList.innerHTML = checkboxesHtml;
-
- listContainer.appendChild(listLabel);
- listContainer.appendChild(scrollableList);
- return listContainer;
-}
diff --git a/webapp/index.html b/backend/webapp/index.html
similarity index 51%
rename from webapp/index.html
rename to backend/webapp/index.html
index c54f520..183e710 100644
--- a/webapp/index.html
+++ b/backend/webapp/index.html
@@ -4,7 +4,11 @@
STAC Viewer
-
+
+
+
+
+
@@ -23,11 +27,18 @@
+ {{ config.get('message', '')}}
+
Request
+ Hover over a key or value for more info.
+
+{
+}
+
Raw STAC
-
+