Working climate-dt deployment

.gitignore

@@ -5,3 +5,4 @@ config.yaml
 *.json
 raw_list
 *.egg-info/
+deps/

chart/templates/ingress.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.stacServer.ingress.enabled }}
+{{- if .Values.ingress.enabled }}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -6,20 +6,31 @@ metadata:
 spec:
   ingressClassName: nginx
   rules:
-    - host: {{ .Values.stacServer.ingress.hostname }}
+    - host: {{ .Values.ingress.hostname }}
       http:
         paths:
-          - path: /
+        {{- if .Values.stacServer.enabled }}
+          - path: /api
             pathType: Prefix
             backend:
               service:
                 name: stac-server
                 port:
                   number: {{ .Values.stacServer.servicePort }}
+        {{- end }}
+        {{- if .Values.webQueryBuilder.enabled }}
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: web-query-builder
+                port:
+                  number: {{ .Values.webQueryBuilder.servicePort }}
+        {{- end }}
   tls:
     - hosts:
-        - {{ .Values.stacServer.ingress.hostname }}
+        - {{ .Values.ingress.hostname }}
-      secretName: lumi-wildcard-tls
+      secretName: {{ .Values.ingress.tlsSecretName }}
 {{- end }}
 
 

chart/templates/stac-server-configmap.yaml

@@ -0,0 +1,11 @@
+# apiVersion: v1
+# kind: ConfigMap
+# metadata:
+#   name: stack-server
+# data:
+#   file1.txt: |-
+#     {{ .Files.Get "files/file1.txt" | nindent 2 }}
+#   file2.txt: |-
+#     {{ .Files.Get "files/file2.txt" | nindent 2 }}
+#   file3.txt: |-
+#     {{ .Files.Get "files/file3.txt" | nindent 2 }}

chart/templates/web-query-builder-deployment.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: web-query-builder
+spec:
+  replicas: {{ .Values.webQueryBuilder.replicas }}
+  selector:
+    matchLabels:
+      app: web-query-builder
+  template:
+    metadata:
+      labels:
+        app: web-query-builder
+    spec:
+      containers:
+        - name: web-query-builder
+          image: "{{ .Values.webQueryBuilder.image.repository }}:{{ .Values.webQueryBuilder.image.tag }}"
+          imagePullPolicy: {{ .Values.webQueryBuilder.image.pullPolicy }}
+          env:
+            - name: API_HOST
+              value: stac-server
+          ports:
+            - containerPort: {{ .Values.webQueryBuilder.servicePort }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: web-query-builder
+spec:
+  selector:
+    app: web-query-builder
+  ports:
+    - protocol: TCP
+      port: {{ .Values.webQueryBuilder.servicePort }}
+      targetPort: {{ .Values.webQueryBuilder.servicePort }}
+  type: ClusterIP

chart/values.yaml

@@ -1,5 +1,3 @@
-# values.yaml
-
 redis:
   servicePort: 6379
   pvc:
@@ -11,15 +9,25 @@ redis:
   service:
     port: 6379
 
-# See https://eccr.ecmwf.int/harbor/projects/258/repositories
 stacServer:
+  enabled: true
   image:
     repository: "eccr.ecmwf.int/qubed/stac_server"
     tag: "latest"
-    pullPolicy: IfNotPresent
+    pullPolicy: Always
-  servicePort: 8080
+  servicePort: 80
   environment:
     REDIS_HOST: "redis"
-  ingress:
+
-    enabled: True
+webQueryBuilder:
-    hostname: "climate-catalogue.lumi.apps.dte.destination-earth.eu"
+  enabled: true
+  image:
+    repository: "eccr.ecmwf.int/qubed/web_query_builder"
+    tag: "latest"
+    pullPolicy: Always
+  servicePort: 80
+
+ingress:
+  enabled: True
+  tlsSecretName: "lumi-wildcard-tls"
+  hostname: "climate-catalogue.lumi.apps.dte.destination-earth.eu"

config/local/schema

@@ -0,0 +1,11 @@
+[ class=od, stream, date, time
+       [ domain, type, levtype, dbase, rki, rty, ty
+               [ step, levelist?, param ]]
+]
+
+[ class=ensemble, number, stream, date, time,
+       [ domain, type, levtype, dbase, rki, rty, ty
+               [ step, levelist?, param ]]
+]
+
+[ class, foo]

dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12-slim AS stac_server
+FROM python:3.12-slim AS base
 
 RUN apt-get update && apt-get install -y \
     build-essential \
@@ -7,6 +7,9 @@ RUN apt-get update && apt-get install -y \
     git \
     && apt-get clean
 
+RUN pip install uv
+
+# Allows cloning private repos using RUN --mount=type=ssh git clone
 RUN mkdir -p -m 0600 ~/.ssh && \
     ssh-keyscan -H github.com >> ~/.ssh/known_hosts
 
@@ -15,20 +18,38 @@ RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
 ENV PATH="/root/.cargo/bin:${PATH}"
 
 WORKDIR /code
+
+FROM base AS stac_server
+
 COPY stac_server/requirements.txt /code/requirements.txt
 RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
 
 # Todo: don't embed this here, mount them at runtime
-COPY config/destinE/schema /config/schema
+# ENV CONFIG_DIR=/config/
-COPY config/destinE/language.yaml /config/language.yaml
+# COPY config/destinE/config.yaml /config/config.yaml
+# COPY config/destinE/schema /config/schema
+# COPY config/destinE/language.yaml /config/language.yaml
 
 COPY ./tree_compresser /code/tree_compresser 
 
 # Clone the rsfdb and rsfindlibs repos manually because they're private
-RUN --mount=type=ssh git clone ssh://git@github.com/ecmwf/rsfdb.git
+
-RUN --mount=type=ssh git clone ssh://git@github.com/ecmwf/rsfindlibs.git
+# RUN --mount=type=ssh git clone ssh://git@github.com/ecmwf/rsfdb.git
+# RUN --mount=type=ssh git clone ssh://git@github.com/ecmwf/rsfindlibs.git
+COPY stac_server/deps/rsfdb /code/rsfdb
+COPY stac_server/deps/rsfindlibs /code/rsfindlibs
+
 RUN pip install --no-cache-dir -e /code/tree_compresser 
 COPY ./stac_server /code/stac_server
 
 WORKDIR /code/stac_server
-CMD ["fastapi", "dev", "main.py", "--proxy-headers", "--port", "8080", "--host", "0.0.0.0"]
+CMD ["fastapi", "dev", "main.py", "--proxy-headers", "--port", "80", "--host", "0.0.0.0"]
+
+FROM base AS web_query_builder
+
+COPY web_query_builder/requirements.txt /code/requirements.txt
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY web_query_builder /code/web_query_builder
+WORKDIR /code/web_query_builder
+CMD ["flask", "run", "--host", "0.0.0.0", "--port", "80"]

scripts/build_images.sh

@@ -2,12 +2,14 @@ set -e
 
 sudo docker login eccr.ecmwf.int
 
-# Uses ssh agent to check out private repos
+sudo docker build \
-# Make sure that ssh agent is running, your key is added 
-# and potentially that you're using ssh-forwarding if building on a remote machine
-sudo DOCKER_BUILDKIT=1 docker build \
-    --ssh default=${SSH_AUTH_SOCK} \
     --tag=eccr.ecmwf.int/qubed/stac_server:latest \
     --target=stac_server \
     .
-sudo docker --debug push eccr.ecmwf.int/qubed/stac_server:latest
+sudo docker push eccr.ecmwf.int/qubed/stac_server:latest
+
+sudo docker build \
+    --tag=eccr.ecmwf.int/qubed/web_query_builder:latest \
+    --target=web_query_builder \
+    .
+sudo docker push eccr.ecmwf.int/qubed/web_query_builder:latest

scripts/load_redis.py

@@ -0,0 +1,19 @@
+#! .venv/bin/python
+
+import redis
+import yaml
+import json
+
+print("Opening redis connection")
+r = redis.Redis(host="redis", port=6379, db=0)
+
+print("Loading data from local files")
+with open("config/climate-dt/compressed_tree.json") as f:
+    compressed_catalog = json.load(f)
+
+with open("config/climate-dt/language.yaml") as f:
+    mars_language = yaml.safe_load(f)["_field"]
+
+print("Storing data in redis")
+r.set('compressed_catalog', json.dumps(compressed_catalog))
+r.set('mars_language', json.dumps(mars_language))

scripts/setup.sh

@@ -0,0 +1,3 @@
+python3 -m venv .venv
+source .venv/bin/activate
+pip install pyyaml redis

stac_server/main.py

@@ -2,6 +2,7 @@ import json
 import os
 from collections import defaultdict
 from typing import Any, Dict
+from pathlib import Path
 
 import redis
 import yaml
@@ -23,40 +24,33 @@ app.add_middleware(
 async def favicon():
     return FileResponse("favicon.ico")
 
-with open(os.environ.get("CONFIG_DIR", ".") + "/config.yaml", "r") as f:
-    config = yaml.safe_load(f)
 
-if "local_cache" in config:
+if "LOCAL_CACHE" in os.environ:
-    print("Getting cache from local file")
+    print("Getting data from local file")
-    with open(config["local_cache"], "r") as f:
+
-        json_data = f.read()
+    base = Path(os.environ["LOCAL_CACHE"])
-    print("Found compressed catalog in local file")
+    with open(base / "compressed_tree.json", "r") as f:
+        json_tree = f.read()
+
+
+    with open(base / "language.yaml", "r") as f:
+        mars_language = yaml.safe_load(f)["_field"]
+    
 else:
     print("Getting cache from redis")
-    r = redis.Redis(host=os.environ.get("REDIS_HOST", "localhost"), port=6379, db=0)
+    r = redis.Redis(host="redis", port=6379, db=0)
-    json_data = r.get('compressed_catalog')
+    json_tree = r.get('compressed_catalog')
+    assert json_tree, "No compressed tree found in redis"
+    mars_language = json.loads(r.get('mars_language'))
 
-print("Loading tree to json")
+print("Loading tree from json")
-if not json_data:
+c_tree = CompressedTree.from_json(json.loads(json_tree))
-    c_tree = CompressedTree.from_json({})
-else:
-    compressed_tree_json = json.loads(json_data)
-    c_tree = CompressedTree.from_json(compressed_tree_json)
 
 print("Partialy decompressing tree, shoud be able to skip this step in future.")
 tree = c_tree.reconstruct_compressed_ecmwf_style()
 
 print("Ready to serve requests!")
 
-base = os.environ.get("CONFIG_DIR", ".")
-config = {
-    "fdb_schema": f"{base}/schema",
-    "mars_language": f"{base}/language.yaml",
-} 
-
-with open(config["mars_language"], "r") as f:
-    mars_language = yaml.safe_load(f)["_field"]
-
 def request_to_dict(request: Request) -> Dict[str, Any]:
     # Convert query parameters to dictionary format
     request_dict = dict(request.query_params)
@@ -114,7 +108,7 @@ def get_leaves(tree):
             for leaf in get_leaves(v):
                 yield leaf
 
-@app.get("/match")
+@app.get("/api/match")
 async def get_match(request: Request):
     # Convert query parameters to dictionary format
     request_dict = request_to_dict(request)
@@ -130,7 +124,7 @@ async def get_match(request: Request):
 
     return match_tree
 
-@app.get("/paths")
+@app.get("/api/paths")
 async def api_paths(request: Request):
     request_dict = request_to_dict(request)
     match_tree = match_against_cache(request_dict, tree)
@@ -155,7 +149,7 @@ async def api_paths(request: Request):
             "values": sorted(v["values"], reverse=True),
     } for key, v in by_path.items()]
 
-@app.get("/stac")
+@app.get("/api/stac")
 async def get_STAC(request: Request):
     request_dict = request_to_dict(request)
     paths = await api_paths(request)

web_query_builder/requirements.txt

@@ -5,3 +5,4 @@ python-dotenv
 flask-login
 flask-cors
 cachetools
+uvicorn

web_query_builder/static/app.js

@@ -6,7 +6,7 @@ function getSTACUrlFromQuery() {
 
   // get current window url and remove path part
   let api_url = new URL(window.location.href);
-  api_url.pathname = "/stac";
+  api_url.pathname = "/api/stac";
 
   for (const [key, value] of params.entries()) {
     api_url.searchParams.set(key, value);