integrate stac-server and web app

stac_server/main.py

@@ -11,6 +11,10 @@ from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
 from frozendict import frozendict
 from qubed import Qube
 from qubed.tree_formatters import node_tree_to_html
+from fastapi.templating import Jinja2Templates
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import HTMLResponse
+import json
 
 app = FastAPI()
 security = HTTPBearer()
@@ -22,6 +26,9 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+app.mount("/static", StaticFiles(directory="static"), name="static")
+templates = Jinja2Templates(directory="templates")
+
 qubes: dict[str, Qube] = {}
 # print("Getting climate and extremes dt data from github")
 # try:
@@ -46,10 +53,32 @@ qubes["extremes-dt"] = Qube.empty()
 mars_language = {}
 
 if "LOCAL_CACHE" in os.environ:
-    base = Path(os.environ["LOCAL_CACHE"])
+    print("Getting climate and extremes dt data from local files")
+    with open("../tests/example_qubes/climate_dt.json") as f:
+        qubes["climate-dt"] = Qube.from_json(json.load(f))
 
-    with open(base / "language.yaml", "r") as f:
+    with open("../tests/example_qubes/extremes_dt.json") as f:
+        qubes["climate-dt"] = qubes["climate-dt"] | Qube.from_json(json.load(f))
+
+    with open("../config/climate-dt/language.yaml", "r") as f:
         mars_language = yaml.safe_load(f)["_field"]
+else:
+    print("Getting climate and extremes dt data from github")
+    qubes["climate-dt"] = Qube.from_json(
+        requests.get(
+            "https://github.com/ecmwf/qubed/raw/refs/heads/main/tests/example_qubes/climate_dt.json",
+        timeout=1).json()
+    )
+    qubes["extremes-dt"] = Qube.from_json(
+        requests.get(
+            "https://github.com/ecmwf/qubed/raw/refs/heads/main/tests/example_qubes/extremes_dt.json",
+        timeout=1).json()
+    )
+    mars_language = yaml.safe_load(
+        requests.get(
+            "https://github.com/ecmwf/qubed/raw/refs/heads/main/config/climate-dt/language.yaml",
+        timeout=1).content
+    )
 
 if "API_KEY" in os.environ:
     api_key = os.environ["API_KEY"]
@@ -91,6 +120,15 @@ def validate_api_key(credentials: HTTPAuthorizationCredentials = Depends(securit
 async def favicon():
     return FileResponse("favicon.ico")
 
+@app.get("/", response_class=HTMLResponse)
+async def read_root(request: Request):
+    return templates.TemplateResponse("index.html", {"request": request, "config": {
+        "message": "Hello from the dev server!",
+    },
+    "api_url": "/api/v1/stac/climate-dt",
+    "request" : request,
+    })
+
 
 @app.get("/api/v1/keys/")
 async def keys():
@@ -137,6 +175,13 @@ def follow_query(request: dict[str, str | list[str]], qube: Qube):
         for key, v in by_path.items()
     ]
 
+@app.get("/api/v1/select/{key}/")
+async def get(
+    key: str = Depends(validate_key),
+    request: dict[str, str | list[str]] = Depends(parse_request),
+):
+    q = qubes[key].select(request)
+    return q.to_json()
 
 @app.get("/api/v1/query/{key}")
 async def query(

stac_server/run.sh

@@ -1,3 +1,3 @@
 parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
 cd "$parent_path"
-LOCAL_CACHE=../config/climate-dt fastapi dev ./main.py --port 8124 --reload
+LOCAL_CACHE=True fastapi dev ./main.py --port 8124 --reload

stac_server/run_prod.sh

@@ -0,0 +1,3 @@
+parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
+cd "$parent_path"
+sudo LOCAL_CACHE=True ../../.venv/bin/fastapi dev ./main.py --port 80 --host=0.0.0.0 --reload

stac_server/static/app.js

@@ -0,0 +1,315 @@
+// app.js
+
+// Take the query string and stick it on the API URL
+function getSTACUrlFromQuery() {
+  const params = new URLSearchParams(window.location.search);
+
+  // get current window url and remove path part
+  if (window.API_URL.startsWith("http")) {
+    // Absolute URL: Use it directly
+    api_url = new URL(window.API_URL);
+  } else {
+    // Relative URL: Combine with the current window's location
+    api_url = new URL(window.location.href);
+    api_url.pathname = window.API_URL;
+  }
+
+  for (const [key, value] of params.entries()) {
+    api_url.searchParams.set(key, value);
+  }
+
+  console.log(api_url.toString());
+  return api_url.toString();
+}
+
+function get_request_from_url() {
+  // Extract the query params in order and split any with a , delimiter
+  // request is an ordered array of [key, [value1, value2, value3, ...]]
+  const url = new URL(window.location.href);
+  const params = new URLSearchParams(url.search);
+  const request = [];
+  for (const [key, value] of params.entries()) {
+    request.push([key, value.split(",")]);
+  }
+  return request;
+}
+
+function make_url_from_request(request) {
+  const url = new URL(window.location.href);
+  url.search = ""; // Clear existing params
+  const params = new URLSearchParams();
+
+  for (const [key, values] of request) {
+    params.set(key, values.join(","));
+  }
+  url.search = params.toString();
+
+  return url.toString().replace(/%2C/g, ",");
+}
+
+function goToPreviousUrl() {
+  let request = get_request_from_url();
+  request.pop();
+  console.log("Request:", request);
+  const url = make_url_from_request(request);
+  console.log("URL:", url);
+  window.location.href = make_url_from_request(request);
+}
+
+// Function to generate a new STAC URL based on current selection
+function goToNextUrl() {
+  const request = get_request_from_url();
+
+  // Get the currently selected key = value,value2,value3 pairs
+  const items = Array.from(document.querySelectorAll("div#items > div"));
+
+  let any_new_keys = false;
+  const new_keys = items.map((item) => {
+    const key = item.dataset.key;
+    const key_type = item.dataset.keyType;
+    let values = [];
+
+    const datePicker = item.querySelector("input[type='date']");
+    if (datePicker) {
+      values.push(datePicker.value.replace(/-/g, ""));
+    }
+
+    const timePicker = item.querySelector("input[type='time']");
+    if (timePicker) {
+      values.push(timePicker.value.replace(":", ""));
+    }
+
+    const enum_checkboxes = item.querySelectorAll(
+      "input[type='checkbox']:checked"
+    );
+    if (enum_checkboxes.length > 0) {
+      values.push(
+        ...Array.from(enum_checkboxes).map((checkbox) => checkbox.value)
+      );
+    }
+
+    const any = item.querySelector("input[type='text']");
+    if (any && any.value !== "") {
+      values.push(any.value);
+    }
+
+    // Keep track of whether any new keys are selected
+    if (values.length > 0) {
+      any_new_keys = true;
+    }
+
+    return { key, values };
+  });
+
+  // if not new keys are selected, do nothing
+  if (!any_new_keys) {
+    return;
+  }
+
+  // Update the request with the new keys
+  for (const { key, values } of new_keys) {
+    // Find the index of the existing key in the request array
+    const existingIndex = request.findIndex(
+      ([existingKey, existingValues]) => existingKey === key
+    );
+
+    if (existingIndex !== -1) {
+      // If the key already exists,
+      // and the values aren't already in there,
+      // append the values
+      request[existingIndex][1] = [...request[existingIndex][1], ...values];
+    } else {
+      // If the key doesn't exist, add a new entry
+      request.push([key, values]);
+    }
+  }
+
+  const url = make_url_from_request(request);
+  window.location.href = url;
+}
+
+async function createCatalogItem(link, itemsContainer) {
+  const itemDiv = document.createElement("div");
+  itemDiv.className = "item loading";
+  itemDiv.textContent = "Loading...";
+  itemsContainer.appendChild(itemDiv);
+
+  try {
+    // Update the item div with real content
+    itemDiv.classList.remove("loading");
+
+    const variables = link["variables"];
+    const key = Object.keys(variables)[0];
+    const variable = variables[key];
+
+    // add data-key attribute to the itemDiv
+    itemDiv.dataset.key = link.title;
+    itemDiv.dataset.keyType = variable.type;
+
+    itemDiv.innerHTML = `
+      <h3 class="item-title">${link.title || "No title available"}</h3>
+      <p class="item-type">Key Type: ${itemDiv.dataset.keyType || "Unknown"}</p>
+      <p class="item-description">${
+        variable.description ? variable.description.slice(0, 100) : ""
+      }</p>
+    `;
+
+    if (variable.enum && variable.enum.length > 0) {
+      const listContainer = renderCheckboxList(link);
+      itemDiv.appendChild(listContainer);
+    } else {
+      const any = `<input type="text" name="${link.title}">`;
+      const anyNode = document.createRange().createContextualFragment(any);
+      itemDiv.appendChild(anyNode);
+    }
+  } catch (error) {
+    console.error("Error loading item data:", error);
+    itemDiv.innerHTML = `<p>Error loading item details: ${error}</p>`;
+  }
+}
+
+function renderCheckboxList(link) {
+  const variables = link["variables"];
+  const key = Object.keys(variables)[0];
+  const variable = variables[key];
+  const value_descriptions = variable.value_descriptions || [];
+
+  const listContainerHTML = `
+      <div class="item-list-container">
+        <label class="list-label">Select one or more values:</label>
+        <div class="scrollable-list">
+          ${variable.enum
+            .map((value, index) => {
+              const labelText = value_descriptions[index]
+                ? `${value} - ${value_descriptions[index]}`
+                : value;
+              return `
+                <div class="checkbox-container">
+                  <label class="checkbox-label">
+                  <input type="checkbox" class="item-checkbox" value="${value}" ${
+                variable.enum.length === 1 ? "checked" : ""
+              }>
+                  ${labelText}
+                  </label>
+                </div>
+              `;
+            })
+            .join("")}
+        </div>
+      </div>
+    `;
+
+  return document.createRange().createContextualFragment(listContainerHTML)
+    .firstElementChild;
+}
+
+// Render catalog items in the sidebar
+function renderCatalogItems(links) {
+  const itemsContainer = document.getElementById("items");
+  itemsContainer.innerHTML = ""; // Clear previous items
+
+  console.log("Number of Links:", links);
+  const children = links.filter(
+    (link) => link.rel === "child" || link.rel === "items"
+  );
+  console.log("Number of Children:", children.length);
+
+  children.forEach((link) => {
+    createCatalogItem(link, itemsContainer);
+  });
+}
+
+function renderRequestBreakdown(request, descriptions) {
+  const container = document.getElementById("request-breakdown");
+  const format_value = (key, value) => {
+    return `<span class="value" title="${descriptions[key]["value_descriptions"][value]}">"${value}"</span>`;
+  };
+
+  const format_values = (key, values) => {
+    if (values.length === 1) {
+      return format_value(key, values[0]);
+    }
+    return `[${values.map((v) => format_value(key, v)).join(", ")}]`;
+  };
+
+  let html =
+    `{\n` +
+    request
+      .map(
+        ([key, values]) =>
+          `    <span class="key" title="${
+            descriptions[key]["description"]
+          }">"${key}"</span>: ${format_values(key, values)},`
+      )
+      .join("\n") +
+    `\n}`;
+  container.innerHTML = html;
+}
+
+function renderRawSTACResponse(catalog) {
+  const itemDetails = document.getElementById("raw-stac");
+  // create new object without debug key
+  let just_stac = Object.assign({}, catalog);
+  delete just_stac.debug;
+  itemDetails.textContent = JSON.stringify(just_stac, null, 2);
+
+  const debug_container = document.getElementById("debug");
+  debug_container.textContent = JSON.stringify(catalog.debug, null, 2);
+
+  const qube_container = document.getElementById("qube");
+  qube_container.innerHTML = catalog.debug.qube;
+}
+
+// Fetch STAC catalog and display items
+async function fetchCatalog(request, stacUrl) {
+  try {
+    const response = await fetch(stacUrl);
+    const catalog = await response.json();
+
+    // Render the request breakdown in the sidebar
+    renderRequestBreakdown(request, catalog.debug.descriptions);
+
+    // Show the raw STAC in the sidebar
+    renderRawSTACResponse(catalog);
+
+    // Render the items from the catalog
+    if (catalog.links) {
+      console.log("Fetched STAC catalog:", stacUrl, catalog.links);
+      renderCatalogItems(catalog.links);
+    }
+
+    // Highlight the request and raw STAC
+    hljs.highlightElement(document.getElementById("raw-stac"));
+    hljs.highlightElement(document.getElementById("debug"));
+    hljs.highlightElement(document.getElementById("example-python"));
+  } catch (error) {
+    console.error("Error fetching STAC catalog:", error);
+  }
+}
+
+// Initialize the viewer by fetching the STAC catalog
+function initializeViewer() {
+  const stacUrl = getSTACUrlFromQuery();
+  const request = get_request_from_url();
+
+  if (stacUrl) {
+    console.log("Fetching STAC catalog from query string URL:", stacUrl);
+    fetchCatalog(request, stacUrl);
+  } else {
+    console.error("No STAC URL provided in the query string.");
+  }
+
+  // Add event listener for the "Generate STAC URL" button
+  const generateUrlBtn = document.getElementById("next-btn");
+  generateUrlBtn.addEventListener("click", goToNextUrl);
+
+  const previousUrlBtn = document.getElementById("previous-btn");
+  previousUrlBtn.addEventListener("click", goToPreviousUrl);
+
+  // Add event listener for the "Raw STAC" button
+  const stacAnchor = document.getElementById("stac-anchor");
+  stacAnchor.href = getSTACUrlFromQuery();
+}
+
+// Call initializeViewer on page load
+initializeViewer();

stac_server/static/qube_styles.css

@@ -0,0 +1,50 @@
+pre#qube {
+    font-family: monospace;
+    white-space: pre;
+    font-family: SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;
+    font-size: 12px;
+    line-height: 1.4;
+
+    details {
+        margin-left: 0;
+    }
+
+    .qubed-node a {
+        margin-left: 10px;
+        text-decoration: none;
+    }
+
+    summary {
+        list-style: none;
+        cursor: pointer;
+        text-overflow: ellipsis;
+        overflow: hidden;
+        text-wrap: nowrap;
+        display: block;
+    }
+
+    summary:hover,span.leaf:hover {
+        background-color: #f0f0f0;
+    }
+
+    details > summary::after {
+        content: ' ▲';
+    }
+
+    details:not([open]) > summary::after {
+        content: " ▼";
+    }
+
+    .leaf {
+        text-overflow: ellipsis;
+        overflow: hidden;
+        text-wrap: nowrap;
+        display: block;
+    }
+
+    summary::-webkit-details-marker {
+      display: none;
+      content: "";
+    }
+
+}

stac_server/static/styles.css

@@ -0,0 +1,217 @@
+html,
+body {
+    min-height: 100vh;
+    height: 100%;
+}
+
+body {
+    font-family: Arial, sans-serif;
+    margin: 0;
+    padding-left: 0.5em;
+    padding-right: 0.5em;
+
+}
+
+#viewer {
+    display: flex;
+    flex-direction: row;
+    height: fit-content;
+    min-height: 100vh;
+}
+
+#catalog-list {
+    width: 30%;
+    padding: 10px;
+    overflow-y: scroll;
+    background-color: #f4f4f4;
+    border-right: 1px solid #ddd;
+}
+
+#catalog-list h2 {
+    margin-top: 0;
+}
+
+#details {
+    width: 70%;
+    padding: 10px;
+}
+
+.sidebar-header {
+    display: flex;
+    justify-content: center;
+    margin-bottom: 10px;
+    flex-wrap: wrap;
+    gap: 0.5em;
+}
+
+.sidebar-header button {
+    width: 7em;
+    height: 2em;
+    padding: 0;
+}
+
+canvas {
+    width: 100%;
+    height: 300px;
+    border: 1px solid #ccc;
+    margin-top: 20px;
+}
+
+/* Updated CSS for the item elements in the catalog list */
+.item {
+    background-color: white;
+    border: 1px solid #ddd;
+    padding: 10px;
+    margin-bottom: 10px;
+    border-radius: 5px;
+    transition: background-color 0.2s ease;
+}
+
+.item-title {
+    font-size: 18px;
+    margin: 0;
+    color: #333;
+}
+
+.item-type {
+    font-size: 14px;
+    margin: 5px 0;
+    color: #666;
+}
+
+.item-id,
+.item-key-type {
+    font-size: 12px;
+    color: #999;
+}
+
+.item-description {
+    font-size: 13px;
+    margin: 5px 0;
+    color: #444;
+    font-style: italic;
+}
+
+.item.selected {
+    background-color: #d4e9ff;
+    /* Lighter blue for selection */
+    border-color: #003399;
+    /* Keep the original ECMWF blue for the border */
+}
+
+summary h2 {
+    display: inline;
+}
+
+.json-pre {
+    white-space: pre-wrap;
+    /* background-color: #f9f9f9; */
+    border: 1px solid #ccc;
+    border-radius: 5px;
+    padding: 10px;
+}
+
+
+/* Button styles */
+button {
+    height: 3em;
+    padding: 10px 20px;
+    /* Padding around button text */
+    margin: 0 5px;
+    /* Margin between buttons */
+    background-color: #003399;
+    /* ECMWF blue */
+    color: white;
+    /* White text color */
+    border: none;
+    /* Remove default button border */
+    cursor: pointer;
+    /* Pointer cursor on hover */
+    border-radius: 5px;
+    /* Rounded corners */
+    transition: background-color 0.3s ease;
+    /* Smooth background color transition */
+}
+
+button:hover {
+    background-color: #001f66;
+    /* Darker shade of ECMWF blue on hover */
+}
+
+.item-list-container {
+    margin-top: 20px;
+    margin-bottom: 20px;
+}
+
+.scrollable-list {
+    max-height: 200px;
+    overflow-y: auto;
+    padding: 10px;
+    border: 1px solid #ccc;
+    border-radius: 4px;
+    background-color: #fff;
+    box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1);
+}
+
+.checkbox-container {
+    display: flex;
+    align-items: center;
+    margin-bottom: 10px;
+}
+
+.item-checkbox {
+    margin-right: 10px;
+    cursor: pointer;
+}
+
+.checkbox-label {
+    font-size: 16px;
+    color: #333;
+}
+
+.checkbox-container:hover .checkbox-label {
+    color: #003399;
+}
+
+.list-label {
+    font-weight: bold;
+    margin-bottom: 0.5em;
+    display: block;
+    color: #003399;
+}
+
+span.key,
+span.value {
+    color: #ba2121;
+    ;
+}
+
+span.key {
+    font-weight: bold;
+}
+
+span.key:hover,
+span.value:hover {
+    color: #ff2a2a;
+    cursor: pointer;
+}
+
+/* Change layout for narrow viewport */
+@media (max-width: 800px) {
+    #viewer {
+        flex-direction: column;
+    }
+
+    #catalog-list {
+        width: 100%;
+        border-right: none;
+    }
+
+    #details {
+        width: 100%;
+    }
+}
+
+details h2 {
+    font-size: medium;
+}

stac_server/templates/index.html

@@ -0,0 +1,78 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>ECMWF DestinE STAC Viewer</title>
+    <link rel="stylesheet" href="/static/styles.css" />
+    <link rel="stylesheet" href="/static/qube_styles.css" />
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github.min.css">
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/json.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/python.min.js"></script>
+    <link rel="icon" href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 100 100%22><text y=%22.9em%22 font-size=%2290%22>📚</text></svg>">
+
+</head>
+<body>
+    <div id="viewer">
+        <div id="catalog-list">
+            <h2>STAC Items</h2>
+            <p>{{ config.get('message', '')}}</p>
+            <p>Select one <strong>or multiple</strong> items and then click next to iteratively build up a full request.</p>
+            <p>Last database update: <time>{{config.get('last_database_update', '')}}</time></p>
+            <div class="sidebar-header">
+                <button id="previous-btn">Previous</button>
+                <a id="stac-anchor"><button id="stac-btn">Raw STAC</button></a>
+                <button id="next-btn">Next</button>
+            </div>
+
+            <div id="items">
+                <!-- Items from the STAC catalog will be rendered here -->
+            </div>
+        </div>
+        <div id="details">
+            <h2>Current Selection</h2>
+            This is a <a href="https://github.com/ecmwf/datacube-spec/blob/main/spec/selection.md">MARS Selection</a> object in JSON format. Hover over a key or value for more info.
+            <!-- Container for the request part, preloaded to prevent layout shift. -->
+            <pre><code id="request-breakdown" class="language-json">
+{
+}
+            </code></pre>
+
+            <!-- Container to show the current tree -->
+            <h2>Currently Selected Tree</h2></summary>
+            <pre id = "qube"></pre>
+
+            <details>
+                <summary><h2>Example Qubed Code</h2></summary>
+                See the <a href="https://qubed.readthedocs.io/en/latest/">Qubed documentation</a> for more details.
+                <pre><code id="example-python" class="language-python">
+# pip install qubed requests
+import requests
+from qubed import Qube
+qube = Qube.from_json(requests.get("http://stac-catalogs.ecmwf-development.f.ewcloud.host/api/v1/select/climate-dt?{{request.url.query}}").json())
+qube.print()
+                </code></pre>
+            </details>
+
+            <!-- Container fo the raw STAC response -->
+            <details>
+                <summary><h2>Raw STAC Response</h2></summary>
+                <p>See the <a href="https://github.com/ecmwf-projects/catalogs/blob/main/structured_stac.md">STAC Extension Proposal</a> for more details on the format.</p>
+                <pre class="json-pre"><code id="raw-stac" class="language-json"></code></pre>
+            </details>
+
+            <!-- Container for the debug response -->
+            <details>
+                <summary><h2>Debug Info</h2></summary>
+                <pre class="json-pre"><code id="debug" class="language-json"></code></pre>
+            </details>
+        </div>
+    </div>
+
+    <script>
+        window.API_URL = "{{ api_url }}";
+    </script>
+    <script src="/static/app.js"></script>
+</body>
+</html>