Integrate web_query_builder and stac server backend

This commit is contained in:
Tom 2025-03-31 16:36:04 +01:00
parent 4502a942cb
commit ab2f8cf3f3
16 changed files with 14 additions and 735 deletions

View File

@ -1,5 +1,5 @@
apiVersion: v2
name: stac-server
name: qubed
description: A Helm chart for the STAC Server with frontend, STAC API and caching service.
type: application
version: 0.1.0

View File

@ -10,7 +10,7 @@ spec:
http:
paths:
{{- if .Values.stacServer.enabled }}
- path: /api
- path: /
pathType: Prefix
backend:
service:
@ -18,15 +18,6 @@ spec:
port:
number: {{ .Values.stacServer.servicePort }}
{{- end }}
{{- if .Values.webQueryBuilder.enabled }}
- path: /
pathType: Prefix
backend:
service:
name: web-query-builder
port:
number: {{ .Values.webQueryBuilder.servicePort }}
{{- end }}
tls:
- hosts:
- {{ .Values.ingress.hostname }}

View File

@ -6,14 +6,6 @@ stacServer:
pullPolicy: Always
servicePort: 80
webQueryBuilder:
enabled: true
image:
repository: "eccr.ecmwf.int/qubed/web_query_builder"
tag: "latest"
pullPolicy: Always
servicePort: 80
ingress:
enabled: True
tlsSecretName: "lumi-wildcard-tls"

View File

@ -34,12 +34,3 @@ COPY ./stac_server /code/stac_server
WORKDIR /code/stac_server
CMD ["fastapi", "dev", "main.py", "--proxy-headers", "--port", "80", "--host", "0.0.0.0"]
FROM base AS web_query_builder
COPY web_query_builder/requirements.txt /code/requirements.txt
RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
COPY web_query_builder /code/web_query_builder
WORKDIR /code/web_query_builder
CMD ["flask", "run", "--host", "0.0.0.0", "--port", "80"]

View File

@ -7,9 +7,3 @@ sudo docker build \
--target=stac_server \
.
sudo docker push eccr.ecmwf.int/qubed/stac_server:latest
sudo docker build \
--tag=eccr.ecmwf.int/qubed/web_query_builder:latest \
--target=web_query_builder \
.
sudo docker push eccr.ecmwf.int/qubed/web_query_builder:latest

View File

@ -176,7 +176,6 @@ function renderCheckboxList(link) {
const listContainerHTML = `
<div class="item-list-container">
<label class="list-label">Select one or more values:</label>
<div class="scrollable-list">
${variable.enum
.map((value, index) => {

View File

@ -2,6 +2,9 @@ html,
body {
min-height: 100vh;
height: 100%;
--accent-color: #003399;
--background-grey: #f4f4f4;
}
body {
@ -23,7 +26,7 @@ body {
width: 30%;
padding: 10px;
overflow-y: scroll;
background-color: #f4f4f4;
background-color: var(--background-grey);
border-right: 1px solid #ddd;
}
@ -65,6 +68,7 @@ canvas {
margin-bottom: 10px;
border-radius: 5px;
transition: background-color 0.2s ease;
box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1);
}
.item-title {
@ -93,10 +97,8 @@ canvas {
}
.item.selected {
background-color: #d4e9ff;
/* Lighter blue for selection */
border-color: #003399;
/* Keep the original ECMWF blue for the border */
background-color: var(--background-grey);
border-color: var(--accent-color);
}
summary h2 {
@ -119,7 +121,7 @@ button {
/* Padding around button text */
margin: 0 5px;
/* Margin between buttons */
background-color: #003399;
background-color: var(--accent-color);
/* ECMWF blue */
color: white;
/* White text color */
@ -140,7 +142,6 @@ button:hover {
.item-list-container {
margin-top: 20px;
margin-bottom: 20px;
}
.scrollable-list {
@ -150,7 +151,6 @@ button:hover {
border: 1px solid #ccc;
border-radius: 4px;
background-color: #fff;
box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1);
}
.checkbox-container {
@ -170,14 +170,14 @@ button:hover {
}
.checkbox-container:hover .checkbox-label {
color: #003399;
color: var(--accent-color);
}
.list-label {
font-weight: bold;
margin-bottom: 0.5em;
display: block;
color: #003399;
color: var(--accent-color);
}
span.key,
@ -214,4 +214,4 @@ span.value:hover {
details h2 {
font-size: medium;
}
}

View File

@ -41,6 +41,7 @@
<!-- Container to show the current tree -->
<h2>Currently Selected Tree</h2></summary>
<p>This shows the data <a href="https://qubed.readthedocs.io/en/latest/quickstart.html">qube</a> that matches with the current query. The leaves are the next set if available selections you can make. </p>
<pre id = "qube"></pre>
<details>

View File

@ -1 +0,0 @@
API_HOST=localhost:8124

View File

@ -1,33 +0,0 @@
import os
from flask import (
Flask,
render_template,
request,
)
from flask_cors import CORS
from werkzeug.middleware.proxy_fix import ProxyFix
app = Flask(__name__)
CORS(app, resources={r"/api/*": {"origins": "*"}})
# This is required because when running in k8s the flask server sits behind a TLS proxy
# So flask speaks http while the client speaks https
# Client <-- https ---> Proxy <---- http ---> Flask server
# For the Oauth flow, flask needs to provide a callback url and it needs to use the right scheme=https
# This line tells flask to look at HTTP headers set by the TLS proxy to figure out what the original
# Traffic looked like.
# See https://flask.palletsprojects.com/en/3.0.x/deploying/proxy_fix/
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1)
config = {}
@app.route("/")
def index():
return render_template(
"index.html",
request=request,
config=config,
api_url=os.environ.get("API_URL", "/api/v1/stac"),
)

View File

@ -1,8 +0,0 @@
flask==3
pyyaml
flask_dance
python-dotenv
flask-login
flask-cors
cachetools
uvicorn

View File

@ -1,2 +0,0 @@
export API_URL="http://127.0.0.1:8124/api/v1/stac/climate-dt"
flask run --debug --port=5006

View File

@ -1,314 +0,0 @@
// app.js
// Take the query string and stick it on the API URL
function getSTACUrlFromQuery() {
const params = new URLSearchParams(window.location.search);
// get current window url and remove path part
if (window.API_URL.startsWith("http")) {
// Absolute URL: Use it directly
api_url = new URL(window.API_URL);
} else {
// Relative URL: Combine with the current window's location
api_url = new URL(window.location.href);
api_url.pathname = window.API_URL;
}
for (const [key, value] of params.entries()) {
api_url.searchParams.set(key, value);
}
console.log(api_url.toString());
return api_url.toString();
}
function get_request_from_url() {
// Extract the query params in order and split any with a , delimiter
// request is an ordered array of [key, [value1, value2, value3, ...]]
const url = new URL(window.location.href);
const params = new URLSearchParams(url.search);
const request = [];
for (const [key, value] of params.entries()) {
request.push([key, value.split(",")]);
}
return request;
}
function make_url_from_request(request) {
const url = new URL(window.location.href);
url.search = ""; // Clear existing params
const params = new URLSearchParams();
for (const [key, values] of request) {
params.set(key, values.join(","));
}
url.search = params.toString();
return url.toString().replace(/%2C/g, ",");
}
function goToPreviousUrl() {
let request = get_request_from_url();
request.pop();
console.log("Request:", request);
const url = make_url_from_request(request);
console.log("URL:", url);
window.location.href = make_url_from_request(request);
}
// Function to generate a new STAC URL based on current selection
function goToNextUrl() {
const request = get_request_from_url();
// Get the currently selected key = value,value2,value3 pairs
const items = Array.from(document.querySelectorAll("div#items > div"));
let any_new_keys = false;
const new_keys = items.map((item) => {
const key = item.dataset.key;
const key_type = item.dataset.keyType;
let values = [];
const datePicker = item.querySelector("input[type='date']");
if (datePicker) {
values.push(datePicker.value.replace(/-/g, ""));
}
const timePicker = item.querySelector("input[type='time']");
if (timePicker) {
values.push(timePicker.value.replace(":", ""));
}
const enum_checkboxes = item.querySelectorAll(
"input[type='checkbox']:checked"
);
if (enum_checkboxes.length > 0) {
values.push(
...Array.from(enum_checkboxes).map((checkbox) => checkbox.value)
);
}
const any = item.querySelector("input[type='text']");
if (any && any.value !== "") {
values.push(any.value);
}
// Keep track of whether any new keys are selected
if (values.length > 0) {
any_new_keys = true;
}
return { key, values };
});
// if not new keys are selected, do nothing
if (!any_new_keys) {
return;
}
// Update the request with the new keys
for (const { key, values } of new_keys) {
// Find the index of the existing key in the request array
const existingIndex = request.findIndex(
([existingKey, existingValues]) => existingKey === key
);
if (existingIndex !== -1) {
// If the key already exists,
// and the values aren't already in there,
// append the values
request[existingIndex][1] = [...request[existingIndex][1], ...values];
} else {
// If the key doesn't exist, add a new entry
request.push([key, values]);
}
}
const url = make_url_from_request(request);
window.location.href = url;
}
async function createCatalogItem(link, itemsContainer) {
const itemDiv = document.createElement("div");
itemDiv.className = "item loading";
itemDiv.textContent = "Loading...";
itemsContainer.appendChild(itemDiv);
try {
// Update the item div with real content
itemDiv.classList.remove("loading");
const variables = link["variables"];
const key = Object.keys(variables)[0];
const variable = variables[key];
// add data-key attribute to the itemDiv
itemDiv.dataset.key = link.title;
itemDiv.dataset.keyType = variable.type;
itemDiv.innerHTML = `
<h3 class="item-title">${link.title || "No title available"}</h3>
<p class="item-type">Key Type: ${itemDiv.dataset.keyType || "Unknown"}</p>
<p class="item-description">${
variable.description ? variable.description.slice(0, 100) : ""
}</p>
`;
if (variable.enum && variable.enum.length > 0) {
const listContainer = renderCheckboxList(link);
itemDiv.appendChild(listContainer);
} else {
const any = `<input type="text" name="${link.title}">`;
const anyNode = document.createRange().createContextualFragment(any);
itemDiv.appendChild(anyNode);
}
} catch (error) {
console.error("Error loading item data:", error);
itemDiv.innerHTML = `<p>Error loading item details: ${error}</p>`;
}
}
function renderCheckboxList(link) {
const variables = link["variables"];
const key = Object.keys(variables)[0];
const variable = variables[key];
const value_descriptions = variable.value_descriptions || [];
const listContainerHTML = `
<div class="item-list-container">
<label class="list-label">Select one or more values:</label>
<div class="scrollable-list">
${variable.enum
.map((value, index) => {
const labelText = value_descriptions[index]
? `${value} - ${value_descriptions[index]}`
: value;
return `
<div class="checkbox-container">
<label class="checkbox-label">
<input type="checkbox" class="item-checkbox" value="${value}" ${
variable.enum.length === 1 ? "checked" : ""
}>
${labelText}
</label>
</div>
`;
})
.join("")}
</div>
</div>
`;
return document.createRange().createContextualFragment(listContainerHTML)
.firstElementChild;
}
// Render catalog items in the sidebar
function renderCatalogItems(links) {
const itemsContainer = document.getElementById("items");
itemsContainer.innerHTML = ""; // Clear previous items
console.log("Number of Links:", links);
const children = links.filter(
(link) => link.rel === "child" || link.rel === "items"
);
console.log("Number of Children:", children.length);
children.forEach((link) => {
createCatalogItem(link, itemsContainer);
});
}
function renderRequestBreakdown(request, descriptions) {
const container = document.getElementById("request-breakdown");
const format_value = (key, value) => {
return `<span class="value" title="${descriptions[key]["value_descriptions"][value]}">"${value}"</span>`;
};
const format_values = (key, values) => {
if (values.length === 1) {
return format_value(key, values[0]);
}
return `[${values.map((v) => format_value(key, v)).join(", ")}]`;
};
let html =
`{\n` +
request
.map(
([key, values]) =>
` <span class="key" title="${
descriptions[key]["description"]
}">"${key}"</span>: ${format_values(key, values)},`
)
.join("\n") +
`\n}`;
container.innerHTML = html;
}
function renderRawSTACResponse(catalog) {
const itemDetails = document.getElementById("raw-stac");
// create new object without debug key
let just_stac = Object.assign({}, catalog);
delete just_stac.debug;
itemDetails.textContent = JSON.stringify(just_stac, null, 2);
const debug_container = document.getElementById("debug");
debug_container.textContent = JSON.stringify(catalog.debug, null, 2);
const qube_container = document.getElementById("qube");
qube_container.innerHTML = catalog.debug.qube;
}
// Fetch STAC catalog and display items
async function fetchCatalog(request, stacUrl) {
try {
const response = await fetch(stacUrl);
const catalog = await response.json();
// Render the request breakdown in the sidebar
renderRequestBreakdown(request, catalog.debug.descriptions);
// Show the raw STAC in the sidebar
renderRawSTACResponse(catalog);
// Render the items from the catalog
if (catalog.links) {
console.log("Fetched STAC catalog:", stacUrl, catalog.links);
renderCatalogItems(catalog.links);
}
// Highlight the request and raw STAC
hljs.highlightElement(document.getElementById("raw-stac"));
hljs.highlightElement(document.getElementById("debug"));
} catch (error) {
console.error("Error fetching STAC catalog:", error);
}
}
// Initialize the viewer by fetching the STAC catalog
function initializeViewer() {
const stacUrl = getSTACUrlFromQuery();
const request = get_request_from_url();
if (stacUrl) {
console.log("Fetching STAC catalog from query string URL:", stacUrl);
fetchCatalog(request, stacUrl);
} else {
console.error("No STAC URL provided in the query string.");
}
// Add event listener for the "Generate STAC URL" button
const generateUrlBtn = document.getElementById("next-btn");
generateUrlBtn.addEventListener("click", goToNextUrl);
const previousUrlBtn = document.getElementById("previous-btn");
previousUrlBtn.addEventListener("click", goToPreviousUrl);
// Add event listener for the "Raw STAC" button
const stacAnchor = document.getElementById("stac-anchor");
stacAnchor.href = getSTACUrlFromQuery();
}
// Call initializeViewer on page load
initializeViewer();

View File

@ -1,50 +0,0 @@
pre#qube {
font-family: monospace;
white-space: pre;
font-family: SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;
font-size: 12px;
line-height: 1.4;
details {
margin-left: 0;
}
.qubed-node a {
margin-left: 10px;
text-decoration: none;
}
summary {
list-style: none;
cursor: pointer;
text-overflow: ellipsis;
overflow: hidden;
text-wrap: nowrap;
display: block;
}
summary:hover,span.leaf:hover {
background-color: #f0f0f0;
}
details > summary::after {
content: ' ▲';
}
details:not([open]) > summary::after {
content: " ▼";
}
.leaf {
text-overflow: ellipsis;
overflow: hidden;
text-wrap: nowrap;
display: block;
}
summary::-webkit-details-marker {
display: none;
content: "";
}
}

View File

@ -1,215 +0,0 @@
html,
body {
min-height: 100vh;
height: 100%;
}
body {
font-family: Arial, sans-serif;
margin: 0;
padding-left: 0.5em;
padding-right: 0.5em;
}
#viewer {
display: flex;
flex-direction: row;
height: fit-content;
min-height: 100vh;
}
#catalog-list {
width: 30%;
padding: 10px;
overflow-y: scroll;
background-color: #f4f4f4;
border-right: 1px solid #ddd;
}
#catalog-list h2 {
margin-top: 0;
}
#details {
width: 70%;
padding: 10px;
}
.sidebar-header {
display: flex;
justify-content: center;
margin-bottom: 10px;
flex-wrap: wrap;
gap: 0.5em;
}
.sidebar-header button {
width: 10em;
}
canvas {
width: 100%;
height: 300px;
border: 1px solid #ccc;
margin-top: 20px;
}
/* Updated CSS for the item elements in the catalog list */
.item {
background-color: white;
border: 1px solid #ddd;
padding: 10px;
margin-bottom: 10px;
border-radius: 5px;
transition: background-color 0.2s ease;
}
.item-title {
font-size: 18px;
margin: 0;
color: #333;
}
.item-type {
font-size: 14px;
margin: 5px 0;
color: #666;
}
.item-id,
.item-key-type {
font-size: 12px;
color: #999;
}
.item-description {
font-size: 13px;
margin: 5px 0;
color: #444;
font-style: italic;
}
.item.selected {
background-color: #d4e9ff;
/* Lighter blue for selection */
border-color: #003399;
/* Keep the original ECMWF blue for the border */
}
summary h2 {
display: inline;
}
.json-pre {
white-space: pre-wrap;
/* background-color: #f9f9f9; */
border: 1px solid #ccc;
border-radius: 5px;
padding: 10px;
}
/* Button styles */
button {
height: 3em;
padding: 10px 20px;
/* Padding around button text */
margin: 0 5px;
/* Margin between buttons */
background-color: #003399;
/* ECMWF blue */
color: white;
/* White text color */
border: none;
/* Remove default button border */
cursor: pointer;
/* Pointer cursor on hover */
border-radius: 5px;
/* Rounded corners */
transition: background-color 0.3s ease;
/* Smooth background color transition */
}
button:hover {
background-color: #001f66;
/* Darker shade of ECMWF blue on hover */
}
.item-list-container {
margin-top: 20px;
margin-bottom: 20px;
}
.scrollable-list {
max-height: 200px;
overflow-y: auto;
padding: 10px;
border: 1px solid #ccc;
border-radius: 4px;
background-color: #fff;
box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1);
}
.checkbox-container {
display: flex;
align-items: center;
margin-bottom: 10px;
}
.item-checkbox {
margin-right: 10px;
cursor: pointer;
}
.checkbox-label {
font-size: 16px;
color: #333;
}
.checkbox-container:hover .checkbox-label {
color: #003399;
}
.list-label {
font-weight: bold;
margin-bottom: 0.5em;
display: block;
color: #003399;
}
span.key,
span.value {
color: #ba2121;
;
}
span.key {
font-weight: bold;
}
span.key:hover,
span.value:hover {
color: #ff2a2a;
cursor: pointer;
}
/* Change layout for narrow viewport */
@media (max-width: 800px) {
#viewer {
flex-direction: column;
}
#catalog-list {
width: 100%;
border-right: none;
}
#details {
width: 100%;
}
}
details h2 {
font-size: medium;
}

View File

@ -1,66 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>ECMWF DestinE STAC Viewer</title>
<link rel="stylesheet" href="/static/styles.css" />
<link rel="stylesheet" href="/static/qube_styles.css" />
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github.min.css">
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/json.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/python.min.js"></script>
<link rel="icon" href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 100 100%22><text y=%22.9em%22 font-size=%2290%22>📚</text></svg>">
</head>
<body>
<div id="viewer">
<div id="catalog-list">
<h2>STAC Items</h2>
<p>{{ config.get('message', '')}}</p>
<p>Select one <strong>or multiple</strong> items and then click next to iteratively build up a full request.</p>
<p>Last database update: <time>{{config.get('last_database_update', '')}}</time></p>
<div class="sidebar-header">
<button id="previous-btn">Previous</button>
<a id="stac-anchor"><button id="stac-btn">Raw STAC</button></a>
<button id="next-btn">Next</button>
</div>
<div id="items">
<!-- Items from the STAC catalog will be rendered here -->
</div>
</div>
<div id="details">
<h2>Current Selection</h2>
This is a <a href="https://github.com/ecmwf/datacube-spec/blob/main/spec/selection.md">MARS Selection</a> object in JSON format. Hover over a key or value for more info.
<!-- Container for the request part, preloaded to prevent layout shift. -->
<pre><code id="request-breakdown" class="language-json">
{
}
</code></pre>
<!-- Container to show the current tree -->
<h2>Currently Selected Tree</h2></summary>
<pre id = "qube"></pre>
<!-- Container fo the raw STAC response -->
<details>
<summary><h2>Raw STAC Response</h2></summary>
<p>See the <a href="https://github.com/ecmwf-projects/catalogs/blob/main/structured_stac.md">STAC Extension Proposal</a> for more details on the format.</p>
<pre class="json-pre"><code id="raw-stac" class="language-json"></code></pre>
</details>
<!-- Container for the debug response -->
<details>
<summary><h2>Debug Info</h2></summary>
<pre class="json-pre"><code id="debug" class="language-json"></code></pre>
</details>
</div>
</div>
<script>
window.API_URL = "{{ api_url }}";
</script>
<script src="/static/app.js"></script>
</body>
</html>