add web_query_builder

web_query_builder/app.py

@@ -0,0 +1,55 @@
+from flask import (
+    Flask,
+    render_template,
+    request,
+    redirect,
+    Response,
+)
+import requests
+from flask_cors import CORS
+
+from werkzeug.middleware.proxy_fix import ProxyFix
+
+app = Flask(__name__)
+CORS(app, resources={r"/api/*": {"origins": "*"}})
+
+# This is required because when running in k8s the flask server sits behind a TLS proxy
+# So flask speaks http while the client speaks https
+# Client <-- https ---> Proxy <---- http ---> Flask server
+# For the Oauth flow, flask needs to provide a callback url and it needs to use the right scheme=https
+# This line tells flask to look at HTTP headers set by the TLS proxy to figure out what the original 
+# Traffic looked like.
+# See https://flask.palletsprojects.com/en/3.0.x/deploying/proxy_fix/
+app.wsgi_app = ProxyFix(
+    app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1
+)
+
+config = {}
+
+@app.route("/")
+def index():
+    return render_template("index.html", request = request, config = config)
+
+
+
+# @app.route('/stac', methods=["GET", "POST"])  # ref. https://medium.com/@zwork101/making-a-flask-proxy-server-online-in-10-lines-of-code-44b8721bca6
+# def redirect_to_API_HOST():  #NOTE var :subpath will be unused as all path we need will be read from :request ie from flask import request
+#     url = f'http://localhost:8124/stac'
+#     res = requests.request(  # ref. https://stackoverflow.com/a/36601467/248616
+#         method          = request.method,
+#         url             = url,
+#         headers         = {k:v for k,v in request.headers if k.lower() != 'host'}, # exclude 'host' header
+#         data            = request.get_data(),
+#         cookies         = request.cookies,
+#         allow_redirects = False,
+#     )
+
+#     excluded_headers = ['content-encoding', 'content-length', 'transfer-encoding', 'connection']  #NOTE we here exclude all "hop-by-hop headers" defined by RFC 2616 section 13.5.1 ref. https://www.rfc-editor.org/rfc/rfc2616#section-13.5.1
+#     headers          = [
+#         (k,v) for k,v in res.raw.headers.items()
+#         if k.lower() not in excluded_headers
+#     ]
+
+#     response = Response(res.content, res.status_code, headers)
+#     return response
+

web_query_builder/requirements.txt

@@ -0,0 +1,7 @@
+flask==3
+pyyaml
+flask_dance
+python-dotenv
+flask-login
+flask-cors
+cachetools

web_query_builder/run.sh

@@ -0,0 +1 @@
+flask run --debug --port=5005

web_query_builder/static/app.js

@@ -0,0 +1,308 @@
+// app.js
+
+// Take the query string and stick it on the API URL
+function getSTACUrlFromQuery() {
+  const params = new URLSearchParams(window.location.search);
+
+  // get current window url and remove path part
+  let api_url = new URL(window.location.href);
+  api_url.pathname = "/stac";
+
+  for (const [key, value] of params.entries()) {
+    api_url.searchParams.set(key, value);
+  }
+
+  console.log(api_url.toString());
+  return api_url.toString();
+}
+
+function get_request_from_url() {
+  // Extract the query params in order and split any with a , delimiter
+  // request is an ordered array of [key, [value1, value2, value3, ...]]
+  const url = new URL(window.location.href);
+  const params = new URLSearchParams(url.search);
+  const request = [];
+  for (const [key, value] of params.entries()) {
+    request.push([key, value.split(",")]);
+  }
+  return request;
+}
+
+function make_url_from_request(request) {
+  const url = new URL(window.location.href);
+  url.search = ""; // Clear existing params
+  const params = new URLSearchParams();
+
+  for (const [key, values] of request) {
+    params.set(key, values.join(","));
+  }
+  url.search = params.toString();
+
+  return url.toString().replace(/%2C/g, ",");
+}
+
+function goToPreviousUrl() {
+  let request = get_request_from_url();
+  request.pop();
+  console.log("Request:", request);
+  const url = make_url_from_request(request);
+  console.log("URL:", url);
+  window.location.href = make_url_from_request(request);
+}
+
+// Function to generate a new STAC URL based on current selection
+function goToNextUrl() {
+  const request = get_request_from_url();
+
+  // Get the currently selected key = value,value2,value3 pairs
+  const items = Array.from(document.querySelectorAll("div#items > div"));
+
+  let any_new_keys = false;
+  const new_keys = items.map((item) => {
+    const key = item.dataset.key;
+    const key_type = item.dataset.keyType;
+    let values = [];
+
+    const datePicker = item.querySelector("input[type='date']");
+    if (datePicker) {
+      values.push(datePicker.value.replace(/-/g, ""));
+    }
+
+    const timePicker = item.querySelector("input[type='time']");
+    if (timePicker) {
+      values.push(timePicker.value.replace(":", ""));
+    }
+
+    const enum_checkboxes = item.querySelectorAll("input[type='checkbox']:checked");
+    if (enum_checkboxes.length > 0) {
+      values.push(...Array.from(enum_checkboxes).map((checkbox) => checkbox.value));
+    }
+
+    const any = item.querySelector("input[type='text']");
+    if (any && any.value !== "") {
+      values.push(any.value);
+    }
+
+    // Keep track of whether any new keys are selected
+    if (values.length > 0) {
+      any_new_keys = true;
+    }
+
+    return { key, values };
+  });
+
+  // if not new keys are selected, do nothing
+  if (!any_new_keys) {
+    return;
+  }
+
+  // Update the request with the new keys
+  for (const { key, values } of new_keys) {
+    // Find the index of the existing key in the request array
+    const existingIndex = request.findIndex(
+      ([existingKey, existingValues]) => existingKey === key
+    );
+
+    if (existingIndex !== -1) {
+      // If the key already exists, 
+      // and the values aren't already in there,
+      // append the values
+      request[existingIndex][1] = [...request[existingIndex][1], ...values];
+    } else {
+      // If the key doesn't exist, add a new entry
+      request.push([key, values]);
+    }
+  }
+
+  const url = make_url_from_request(request);
+  window.location.href = url;
+}
+
+async function createCatalogItem(link, itemsContainer) {
+  const itemDiv = document.createElement("div");
+  itemDiv.className = "item loading";
+  itemDiv.textContent = "Loading...";
+  itemsContainer.appendChild(itemDiv);
+
+  try {
+
+    // Update the item div with real content
+    itemDiv.classList.remove("loading");
+
+    const dimension = link["generalized_datacube:dimension"];
+
+    // add data-key attribute to the itemDiv
+    itemDiv.dataset.key = link.title;
+    itemDiv.dataset.keyType = dimension.type;
+    
+    itemDiv.innerHTML = `
+      <h3 class="item-title">${link.title || "No title available"}</h3>
+      <p class="item-type">Key Type: ${itemDiv.dataset.keyType || "Unknown"}</p>
+      <!-- <p class="item-type">Paths: ${dimension.paths}</p> -->
+      <p class="item-type">Optional: ${dimension.optional ? "Yes" : "No"}</p>
+      <p class="item-description">${dimension.description ? dimension.description.slice(0, 100) : "No description available"}...</p>
+    `;
+
+
+    // if (dimension.type === "date" || dimension.type === "time") {
+    //   // Render a date picker for the "date" key
+    //   const picker = `<input type="${link.title}" name="${link.title}">`;
+    //   //convert picker to HTML node
+    //   const pickerNode = document
+    //     .createRange()
+    //     .createContextualFragment(picker);
+    //   itemDiv.appendChild(pickerNode);
+    // }
+    // Otherwise create a scrollable list with checkboxes for values if available
+    if (
+    //   dimension.type === "enum" &&
+      dimension.values &&
+      dimension.values.length > 0
+    ) {
+      const listContainer = renderCheckboxList(link);
+      itemDiv.appendChild(listContainer);
+    } else {
+      const any = `<input type="text" name="${link.title}">`;
+      const anyNode = document.createRange().createContextualFragment(any);
+      itemDiv.appendChild(anyNode);
+    }
+  } catch (error) {
+    console.error("Error loading item data:", error);
+    itemDiv.innerHTML = `<p>Error loading item details: ${error}</p>`;
+  }
+}
+
+function renderCheckboxList(link) {
+    const dimension = link["generalized_datacube:dimension"];
+    const value_descriptions = dimension.value_descriptions || [];
+  
+    const listContainerHTML = `
+      <div class="item-list-container">
+        <label class="list-label">Select one or more values:</label>
+        <div class="scrollable-list">
+          ${dimension.values
+            .map((value, index) => {
+              const labelText = value_descriptions[index] ? `${value} - ${value_descriptions[index]}` : value;
+              return `
+                <div class="checkbox-container">
+                  <label class="checkbox-label">
+                  <input type="checkbox" class="item-checkbox" value="${value}" ${dimension.values.length === 1? 'checked' : ''}>
+                  ${labelText}
+                  </label>
+                </div>
+              `;
+            })
+            .join("")}
+        </div>
+      </div>
+    `;
+  
+    return document.createRange().createContextualFragment(listContainerHTML).firstElementChild;
+  }
+
+// Render catalog items in the sidebar
+function renderCatalogItems(links) {
+  const itemsContainer = document.getElementById("items");
+  itemsContainer.innerHTML = ""; // Clear previous items
+
+  console.log("Number of Links:", links);
+  const children = links.filter(
+    (link) => link.rel === "child" || link.rel === "items"
+  );
+  console.log("Number of Children:", children.length);
+
+  children.forEach((link) => {
+    createCatalogItem(link, itemsContainer);
+  });
+}
+
+function renderRequestBreakdown(request, descriptions) {
+    const container = document.getElementById("request-breakdown");
+    const format_value = (key, value) => {
+      return `<span class="value" title="${descriptions[key]['value_descriptions'][value]}">"${value}"</span>`;
+    };
+    
+    const format_values = (key, values) => {
+      if (values.length === 1) {
+        return format_value(key, values[0]);
+      }
+      return `[${values.map((v) => 
+        format_value(key, v)
+    ).join(", ")}]`;
+    };
+  
+    let html = `{\n` +
+      request
+        .map(
+          ([key, values]) =>
+            `    <span class="key" title="${descriptions[key]['description']}">"${key}"</span>: ${format_values(key, values)},`
+        )
+        .join("\n") +
+      `\n}`;
+    container.innerHTML = html;
+  }
+
+function renderRawSTACResponse(catalog) {
+  const itemDetails = document.getElementById("raw-stac");
+  // create new object without debug key
+    let just_stac = Object.assign({}, catalog);
+    delete just_stac.debug;
+  itemDetails.textContent = JSON.stringify(just_stac, null, 2);
+
+  const debug_container = document.getElementById("debug");
+  // create new object without debug key
+  debug_container.textContent = JSON.stringify(catalog.debug, null, 2);
+}
+
+// Fetch STAC catalog and display items
+async function fetchCatalog(request, stacUrl) {
+  try {
+    const response = await fetch(stacUrl);
+    const catalog = await response.json();
+
+    // Render the request breakdown in the sidebar
+    renderRequestBreakdown(request, catalog.debug.descriptions);
+
+    // Show the raw STAC in the sidebar
+    renderRawSTACResponse(catalog);
+
+    // Render the items from the catalog
+    if (catalog.links) {
+      console.log("Fetched STAC catalog:", stacUrl, catalog.links);
+      renderCatalogItems(catalog.links);
+    }
+
+    // Highlight the request and raw STAC
+    hljs.highlightElement(document.getElementById("raw-stac"));
+    hljs.highlightElement(document.getElementById("debug"));
+  } catch (error) {
+    console.error("Error fetching STAC catalog:", error);
+  }
+}
+
+// Initialize the viewer by fetching the STAC catalog
+function initializeViewer() {
+  const stacUrl = getSTACUrlFromQuery();
+  const request = get_request_from_url();
+
+  if (stacUrl) {
+    console.log("Fetching STAC catalog from query string URL:", stacUrl);
+    fetchCatalog(request, stacUrl);
+  } else {
+    console.error("No STAC URL provided in the query string.");
+  }
+
+  // Add event listener for the "Generate STAC URL" button
+  const generateUrlBtn = document.getElementById("next-btn");
+  generateUrlBtn.addEventListener("click", goToNextUrl);
+
+  const previousUrlBtn = document.getElementById("previous-btn");
+  previousUrlBtn.addEventListener("click", goToPreviousUrl);
+
+  // Add event listener for the "Raw STAC" button
+  const stacAnchor = document.getElementById("stac-anchor");
+  stacAnchor.href = getSTACUrlFromQuery();
+}
+
+// Call initializeViewer on page load
+initializeViewer();

web_query_builder/static/styles.css

@@ -0,0 +1,211 @@
+html,
+body {
+    min-height: 100vh;
+    height: 100%;
+}
+
+body {
+    font-family: Arial, sans-serif;
+    margin: 0;
+    padding-left: 0.5em;
+    padding-right: 0.5em;
+
+}
+
+#viewer {
+    display: flex;
+    flex-direction: row;
+    height: fit-content;
+    min-height: 100vh;
+}
+
+#catalog-list {
+    width: 30%;
+    padding: 10px;
+    overflow-y: scroll;
+    background-color: #f4f4f4;
+    border-right: 1px solid #ddd;
+}
+
+#catalog-list h2 {
+    margin-top: 0;
+}
+
+#details {
+    width: 70%;
+    padding: 10px;
+}
+
+.sidebar-header {
+    display: flex;
+    justify-content: center;
+    margin-bottom: 10px;
+    flex-wrap: wrap;
+    gap: 0.5em;
+}
+
+.sidebar-header button {
+    width: 10em;
+}
+
+canvas {
+    width: 100%;
+    height: 300px;
+    border: 1px solid #ccc;
+    margin-top: 20px;
+}
+
+/* Updated CSS for the item elements in the catalog list */
+.item {
+    background-color: white;
+    border: 1px solid #ddd;
+    padding: 10px;
+    margin-bottom: 10px;
+    border-radius: 5px;
+    transition: background-color 0.2s ease;
+}
+
+.item-title {
+    font-size: 18px;
+    margin: 0;
+    color: #333;
+}
+
+.item-type {
+    font-size: 14px;
+    margin: 5px 0;
+    color: #666;
+}
+
+.item-id,
+.item-key-type {
+    font-size: 12px;
+    color: #999;
+}
+
+.item-description {
+    font-size: 13px;
+    margin: 5px 0;
+    color: #444;
+    font-style: italic;
+}
+
+.item.selected {
+    background-color: #d4e9ff;
+    /* Lighter blue for selection */
+    border-color: #003399;
+    /* Keep the original ECMWF blue for the border */
+}
+
+summary h2 {
+    display: inline;
+}
+
+.json-pre {
+    white-space: pre-wrap;
+    /* background-color: #f9f9f9; */
+    border: 1px solid #ccc;
+    border-radius: 5px;
+    padding: 10px;
+}
+
+
+/* Button styles */
+button {
+    height: 3em;
+    padding: 10px 20px;
+    /* Padding around button text */
+    margin: 0 5px;
+    /* Margin between buttons */
+    background-color: #003399;
+    /* ECMWF blue */
+    color: white;
+    /* White text color */
+    border: none;
+    /* Remove default button border */
+    cursor: pointer;
+    /* Pointer cursor on hover */
+    border-radius: 5px;
+    /* Rounded corners */
+    transition: background-color 0.3s ease;
+    /* Smooth background color transition */
+}
+
+button:hover {
+    background-color: #001f66;
+    /* Darker shade of ECMWF blue on hover */
+}
+
+.item-list-container {
+    margin-top: 20px;
+    margin-bottom: 20px;
+}
+
+.scrollable-list {
+    max-height: 200px;
+    overflow-y: auto;
+    padding: 10px;
+    border: 1px solid #ccc;
+    border-radius: 4px;
+    background-color: #fff;
+    box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1);
+}
+
+.checkbox-container {
+    display: flex;
+    align-items: center;
+    margin-bottom: 10px;
+}
+
+.item-checkbox {
+    margin-right: 10px;
+    cursor: pointer;
+}
+
+.checkbox-label {
+    font-size: 16px;
+    color: #333;
+}
+
+.checkbox-container:hover .checkbox-label {
+    color: #003399;
+}
+
+.list-label {
+    font-weight: bold;
+    margin-bottom: 0.5em;
+    display: block;
+    color: #003399;
+}
+
+span.key,
+span.value {
+    color: #ba2121;
+    ;
+}
+
+span.key {
+    font-weight: bold;
+}
+
+span.key:hover,
+span.value:hover {
+    color: #ff2a2a;
+    cursor: pointer;
+}
+
+/* Change layout for narrow viewport */
+@media (max-width: 800px) {
+    #viewer {
+        flex-direction: column;
+    }
+
+    #catalog-list {
+        width: 100%;
+        border-right: none;
+    }
+
+    #details {
+        width: 100%;
+    }
+}

web_query_builder/templates/index.html

@@ -0,0 +1,58 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>ECMWF DestinE STAC Viewer</title>
+    <link rel="stylesheet" href="/static/styles.css" />
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github.min.css">
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/json.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/python.min.js"></script>
+    <link rel="icon" href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 100 100%22><text y=%22.9em%22 font-size=%2290%22>📚</text></svg>">
+
+</head>
+<body>
+    <div id="viewer">
+        <div id="catalog-list">
+            <h2>STAC Items</h2>
+            <p>{{ config.get('message', '')}}</p>
+            <p>Select one <strong>or multiple</strong> items and then click next to iteratively build up a full request.</p>
+            <p>Last database update: <time>{{config.get('last_database_update', '')}}</time></p>
+            <div class="sidebar-header">
+                <button id="previous-btn">Previous</button>
+                <a id="stac-anchor"><button id="stac-btn">Raw STAC</button></a>
+                <button id="next-btn">Next</button>
+            </div>
+            
+            <div id="items">
+                <!-- Items from the STAC catalog will be rendered here -->
+            </div>
+        </div>
+        <div id="details">
+            <h2>Current Request</h2>
+            Hover over a key or value for more info.
+            <!-- Container for the request part, preloaded to prevent layout shift. -->
+            <pre><code id="request-breakdown" class="language-json">
+{
+}
+            </code></pre>
+
+            <!-- Container fo the raw STAC response -->
+            <details open>
+                <summary><h2>Raw STAC Response</h2></summary>
+                <p>See the <a href="https://github.com/ecmwf-projects/catalogs/blob/main/structured_stac.md">extension proposal</a> for more details on the format.</p>
+                <pre class="json-pre"><code id="raw-stac" class="language-json"></code></pre>
+            </details>
+
+            <!-- Container for the debug response -->
+            <details>
+                <summary><h2>Debug Info</h2></summary>
+                <pre class="json-pre"><code id="debug" class="language-json"></code></pre>
+            </details>
+        </div>
+    </div>
+
+    <script src="/static/app.js"></script>
+</body>
+</html>