tom/qubed
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

integrate stac-server and web app

Browse Source
This commit is contained in:
Tom Hodson 2025-03-31 12:45:51 +00:00
parent 3017185950
commit 4502a942cb
7 changed files with 711 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
stac_server/main.py
View File

@ -11,6 +11,10 @@ from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from frozendict import frozendict
from qubed import Qube
from qubed.tree_formatters import node_tree_to_html
from fastapi.templating import Jinja2Templates
from fastapi.staticfiles import StaticFiles
from fastapi.responses import HTMLResponse
import json
app = FastAPI()
security = HTTPBearer()
@ -22,6 +26,9 @@ app.add_middleware(
allow_headers=["*"],
)
app.mount("/static", StaticFiles(directory="static"), name="static")
templates = Jinja2Templates(directory="templates")
qubes: dict[str, Qube] = {}
# print("Getting climate and extremes dt data from github")
# try:
@ -46,10 +53,32 @@ qubes["extremes-dt"] = Qube.empty()
mars_language = {}
if "LOCAL_CACHE" in os.environ:
base = Path(os.environ["LOCAL_CACHE"])
print("Getting climate and extremes dt data from local files")
with open("../tests/example_qubes/climate_dt.json") as f:
qubes["climate-dt"] = Qube.from_json(json.load(f))
with open(base / "language.yaml", "r") as f:
with open("../tests/example_qubes/extremes_dt.json") as f:
qubes["climate-dt"] = qubes["climate-dt"] | Qube.from_json(json.load(f))
with open("../config/climate-dt/language.yaml", "r") as f:
mars_language = yaml.safe_load(f)["_field"]
else:
print("Getting climate and extremes dt data from github")
qubes["climate-dt"] = Qube.from_json(
requests.get(
"https://github.com/ecmwf/qubed/raw/refs/heads/main/tests/example_qubes/climate_dt.json",
timeout=1).json()
)
qubes["extremes-dt"] = Qube.from_json(
requests.get(
"https://github.com/ecmwf/qubed/raw/refs/heads/main/tests/example_qubes/extremes_dt.json",
timeout=1).json()
)
mars_language = yaml.safe_load(
requests.get(
"https://github.com/ecmwf/qubed/raw/refs/heads/main/config/climate-dt/language.yaml",
timeout=1).content
)
if "API_KEY" in os.environ:
api_key = os.environ["API_KEY"]
@ -91,6 +120,15 @@ def validate_api_key(credentials: HTTPAuthorizationCredentials = Depends(securit
async def favicon():
return FileResponse("favicon.ico")
@app.get("/", response_class=HTMLResponse)
async def read_root(request: Request):
return templates.TemplateResponse("index.html", {"request": request, "config": {
"message": "Hello from the dev server!",
},
"api_url": "/api/v1/stac/climate-dt",
"request" : request,
})
@app.get("/api/v1/keys/")
async def keys():
@ -137,6 +175,13 @@ def follow_query(request: dict[str, str | list[str]], qube: Qube):
for key, v in by_path.items()
]
@app.get("/api/v1/select/{key}/")
async def get(
key: str = Depends(validate_key),
request: dict[str, str | list[str]] = Depends(parse_request),
):
q = qubes[key].select(request)
return q.to_json()
@app.get("/api/v1/query/{key}")
async def query(

2
stac_server/run.sh
View File

@ -1,3 +1,3 @@
parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
cd "$parent_path"
LOCAL_CACHE=../config/climate-dt fastapi dev ./main.py --port 8124 --reload
LOCAL_CACHE=True fastapi dev ./main.py --port 8124 --reload

3
stac_server/run_prod.sh Executable file
View File

@ -0,0 +1,3 @@
parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
cd "$parent_path"
sudo LOCAL_CACHE=True ../../.venv/bin/fastapi dev ./main.py --port 80 --host=0.0.0.0 --reload

315
stac_server/static/app.js Normal file
View File

@ -0,0 +1,315 @@
// app.js
// Take the query string and stick it on the API URL
function getSTACUrlFromQuery() {
const params = new URLSearchParams(window.location.search);
// get current window url and remove path part
if (window.API_URL.startsWith("http")) {
// Absolute URL: Use it directly
api_url = new URL(window.API_URL);
} else {
// Relative URL: Combine with the current window's location
api_url = new URL(window.location.href);
api_url.pathname = window.API_URL;
}
for (const [key, value] of params.entries()) {
api_url.searchParams.set(key, value);
}
console.log(api_url.toString());
return api_url.toString();
}
function get_request_from_url() {
// Extract the query params in order and split any with a , delimiter
// request is an ordered array of [key, [value1, value2, value3, ...]]
const url = new URL(window.location.href);
const params = new URLSearchParams(url.search);
const request = [];
for (const [key, value] of params.entries()) {
request.push([key, value.split(",")]);
}
return request;
}
function make_url_from_request(request) {
const url = new URL(window.location.href);
url.search = ""; // Clear existing params
const params = new URLSearchParams();
for (const [key, values] of request) {
params.set(key, values.join(","));
}
url.search = params.toString();
return url.toString().replace(/%2C/g, ",");
}
function goToPreviousUrl() {
let request = get_request_from_url();
request.pop();
console.log("Request:", request);
const url = make_url_from_request(request);
console.log("URL:", url);
window.location.href = make_url_from_request(request);
}
// Function to generate a new STAC URL based on current selection
function goToNextUrl() {
const request = get_request_from_url();
// Get the currently selected key = value,value2,value3 pairs
const items = Array.from(document.querySelectorAll("div#items > div"));
let any_new_keys = false;
const new_keys = items.map((item) => {
const key = item.dataset.key;
const key_type = item.dataset.keyType;
let values = [];
const datePicker = item.querySelector("input[type='date']");
if (datePicker) {
values.push(datePicker.value.replace(/-/g, ""));
}
const timePicker = item.querySelector("input[type='time']");
if (timePicker) {
values.push(timePicker.value.replace(":", ""));
}
const enum_checkboxes = item.querySelectorAll(
"input[type='checkbox']:checked"
);
if (enum_checkboxes.length > 0) {
values.push(
...Array.from(enum_checkboxes).map((checkbox) => checkbox.value)
);
}
const any = item.querySelector("input[type='text']");
if (any && any.value !== "") {
values.push(any.value);
}
// Keep track of whether any new keys are selected
if (values.length > 0) {
any_new_keys = true;
}
return { key, values };
});
// if not new keys are selected, do nothing
if (!any_new_keys) {
return;
}
// Update the request with the new keys
for (const { key, values } of new_keys) {
// Find the index of the existing key in the request array
const existingIndex = request.findIndex(
([existingKey, existingValues]) => existingKey === key
);
if (existingIndex !== -1) {
// If the key already exists,
// and the values aren't already in there,
// append the values
request[existingIndex][1] = [...request[existingIndex][1], ...values];
} else {
// If the key doesn't exist, add a new entry
request.push([key, values]);
}
}
const url = make_url_from_request(request);
window.location.href = url;
}
async function createCatalogItem(link, itemsContainer) {
const itemDiv = document.createElement("div");
itemDiv.className = "item loading";
itemDiv.textContent = "Loading...";
itemsContainer.appendChild(itemDiv);
try {
// Update the item div with real content
itemDiv.classList.remove("loading");
const variables = link["variables"];
const key = Object.keys(variables)[0];
const variable = variables[key];
// add data-key attribute to the itemDiv
itemDiv.dataset.key = link.title;
itemDiv.dataset.keyType = variable.type;
itemDiv.innerHTML = `
<h3 class="item-title">${link.title || "No title available"}</h3>
<p class="item-type">Key Type: ${itemDiv.dataset.keyType || "Unknown"}</p>
<p class="item-description">${
variable.description ? variable.description.slice(0, 100) : ""
}</p>
`;
if (variable.enum && variable.enum.length > 0) {
const listContainer = renderCheckboxList(link);
itemDiv.appendChild(listContainer);
} else {
const any = `<input type="text" name="${link.title}">`;
const anyNode = document.createRange().createContextualFragment(any);
itemDiv.appendChild(anyNode);
}
} catch (error) {
console.error("Error loading item data:", error);
itemDiv.innerHTML = `<p>Error loading item details: ${error}</p>`;
}
}
function renderCheckboxList(link) {
const variables = link["variables"];
const key = Object.keys(variables)[0];
const variable = variables[key];
const value_descriptions = variable.value_descriptions || [];
const listContainerHTML = `
<div class="item-list-container">
<label class="list-label">Select one or more values:</label>
<div class="scrollable-list">
${variable.enum
.map((value, index) => {
const labelText = value_descriptions[index]
? `${value} - ${value_descriptions[index]}`
: value;
return `
<div class="checkbox-container">
<label class="checkbox-label">
<input type="checkbox" class="item-checkbox" value="${value}" ${
variable.enum.length === 1 ? "checked" : ""
}>
${labelText}
</label>
</div>
`;
})
.join("")}
</div>
</div>
`;
return document.createRange().createContextualFragment(listContainerHTML)
.firstElementChild;
}
// Render catalog items in the sidebar
function renderCatalogItems(links) {
const itemsContainer = document.getElementById("items");
itemsContainer.innerHTML = ""; // Clear previous items
console.log("Number of Links:", links);
const children = links.filter(
(link) => link.rel === "child" || link.rel === "items"
);
console.log("Number of Children:", children.length);
children.forEach((link) => {
createCatalogItem(link, itemsContainer);
});
}
function renderRequestBreakdown(request, descriptions) {
const container = document.getElementById("request-breakdown");
const format_value = (key, value) => {
return `<span class="value" title="${descriptions[key]["value_descriptions"][value]}">"${value}"</span>`;
};
const format_values = (key, values) => {
if (values.length === 1) {
return format_value(key, values[0]);
}
return `[${values.map((v) => format_value(key, v)).join(", ")}]`;
};
let html =
`{\n` +
request
.map(
([key, values]) =>
` <span class="key" title="${
descriptions[key]["description"]
}">"${key}"</span>: ${format_values(key, values)},`
)
.join("\n") +
`\n}`;
container.innerHTML = html;
}
function renderRawSTACResponse(catalog) {
const itemDetails = document.getElementById("raw-stac");
// create new object without debug key
let just_stac = Object.assign({}, catalog);
delete just_stac.debug;
itemDetails.textContent = JSON.stringify(just_stac, null, 2);
const debug_container = document.getElementById("debug");
debug_container.textContent = JSON.stringify(catalog.debug, null, 2);
const qube_container = document.getElementById("qube");
qube_container.innerHTML = catalog.debug.qube;
}
// Fetch STAC catalog and display items
async function fetchCatalog(request, stacUrl) {
try {
const response = await fetch(stacUrl);
const catalog = await response.json();
// Render the request breakdown in the sidebar
renderRequestBreakdown(request, catalog.debug.descriptions);
// Show the raw STAC in the sidebar
renderRawSTACResponse(catalog);
// Render the items from the catalog
if (catalog.links) {
console.log("Fetched STAC catalog:", stacUrl, catalog.links);
renderCatalogItems(catalog.links);
}
// Highlight the request and raw STAC
hljs.highlightElement(document.getElementById("raw-stac"));
hljs.highlightElement(document.getElementById("debug"));
hljs.highlightElement(document.getElementById("example-python"));
} catch (error) {
console.error("Error fetching STAC catalog:", error);
}
}
// Initialize the viewer by fetching the STAC catalog
function initializeViewer() {
const stacUrl = getSTACUrlFromQuery();
const request = get_request_from_url();
if (stacUrl) {
console.log("Fetching STAC catalog from query string URL:", stacUrl);
fetchCatalog(request, stacUrl);
} else {
console.error("No STAC URL provided in the query string.");
}
// Add event listener for the "Generate STAC URL" button
const generateUrlBtn = document.getElementById("next-btn");
generateUrlBtn.addEventListener("click", goToNextUrl);
const previousUrlBtn = document.getElementById("previous-btn");
previousUrlBtn.addEventListener("click", goToPreviousUrl);
// Add event listener for the "Raw STAC" button
const stacAnchor = document.getElementById("stac-anchor");
stacAnchor.href = getSTACUrlFromQuery();
}
// Call initializeViewer on page load
initializeViewer();

50
stac_server/static/qube_styles.css Normal file
View File

@ -0,0 +1,50 @@
pre#qube {
font-family: monospace;
white-space: pre;
font-family: SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;
font-size: 12px;
line-height: 1.4;
details {
margin-left: 0;
}
.qubed-node a {
margin-left: 10px;
text-decoration: none;
}
summary {
list-style: none;
cursor: pointer;
text-overflow: ellipsis;
overflow: hidden;
text-wrap: nowrap;
display: block;
}
summary:hover,span.leaf:hover {
background-color: #f0f0f0;
}
details > summary::after {
content: ' ▲';
}
details:not([open]) > summary::after {
content: " ▼";
}
.leaf {
text-overflow: ellipsis;
overflow: hidden;
text-wrap: nowrap;
display: block;
}
summary::-webkit-details-marker {
display: none;
content: "";
}
}

217
stac_server/static/styles.css Normal file
View File

@ -0,0 +1,217 @@
html,
body {
min-height: 100vh;
height: 100%;
}
body {
font-family: Arial, sans-serif;
margin: 0;
padding-left: 0.5em;
padding-right: 0.5em;
}
#viewer {
display: flex;
flex-direction: row;
height: fit-content;
min-height: 100vh;
}
#catalog-list {
width: 30%;
padding: 10px;
overflow-y: scroll;
background-color: #f4f4f4;
border-right: 1px solid #ddd;
}
#catalog-list h2 {
margin-top: 0;
}
#details {
width: 70%;
padding: 10px;
}
.sidebar-header {
display: flex;
justify-content: center;
margin-bottom: 10px;
flex-wrap: wrap;
gap: 0.5em;
}
.sidebar-header button {
width: 7em;
height: 2em;
padding: 0;
}
canvas {
width: 100%;
height: 300px;
border: 1px solid #ccc;
margin-top: 20px;
}
/* Updated CSS for the item elements in the catalog list */
.item {
background-color: white;
border: 1px solid #ddd;
padding: 10px;
margin-bottom: 10px;
border-radius: 5px;
transition: background-color 0.2s ease;
}
.item-title {
font-size: 18px;
margin: 0;
color: #333;
}
.item-type {
font-size: 14px;
margin: 5px 0;
color: #666;
}
.item-id,
.item-key-type {
font-size: 12px;
color: #999;
}
.item-description {
font-size: 13px;
margin: 5px 0;
color: #444;
font-style: italic;
}
.item.selected {
background-color: #d4e9ff;
/* Lighter blue for selection */
border-color: #003399;
/* Keep the original ECMWF blue for the border */
}
summary h2 {
display: inline;
}
.json-pre {
white-space: pre-wrap;
/* background-color: #f9f9f9; */
border: 1px solid #ccc;
border-radius: 5px;
padding: 10px;
}
/* Button styles */
button {
height: 3em;
padding: 10px 20px;
/* Padding around button text */
margin: 0 5px;
/* Margin between buttons */
background-color: #003399;
/* ECMWF blue */
color: white;
/* White text color */
border: none;
/* Remove default button border */
cursor: pointer;
/* Pointer cursor on hover */
border-radius: 5px;
/* Rounded corners */
transition: background-color 0.3s ease;
/* Smooth background color transition */
}
button:hover {
background-color: #001f66;
/* Darker shade of ECMWF blue on hover */
}
.item-list-container {
margin-top: 20px;
margin-bottom: 20px;
}
.scrollable-list {
max-height: 200px;
overflow-y: auto;
padding: 10px;
border: 1px solid #ccc;
border-radius: 4px;
background-color: #fff;
box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1);
}
.checkbox-container {
display: flex;
align-items: center;
margin-bottom: 10px;
}
.item-checkbox {
margin-right: 10px;
cursor: pointer;
}
.checkbox-label {
font-size: 16px;
color: #333;
}
.checkbox-container:hover .checkbox-label {
color: #003399;
}
.list-label {
font-weight: bold;
margin-bottom: 0.5em;
display: block;
color: #003399;
}
span.key,
span.value {
color: #ba2121;
;
}
span.key {
font-weight: bold;
}
span.key:hover,
span.value:hover {
color: #ff2a2a;
cursor: pointer;
}
/* Change layout for narrow viewport */
@media (max-width: 800px) {
#viewer {
flex-direction: column;
}
#catalog-list {
width: 100%;
border-right: none;
}
#details {
width: 100%;
}
}
details h2 {
font-size: medium;
}

78
stac_server/templates/index.html Normal file
View File

@ -0,0 +1,78 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>ECMWF DestinE STAC Viewer</title>
<link rel="stylesheet" href="/static/styles.css" />
<link rel="stylesheet" href="/static/qube_styles.css" />
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github.min.css">
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/json.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/python.min.js"></script>
<link rel="icon" href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 100 100%22><text y=%22.9em%22 font-size=%2290%22>📚</text></svg>">
</head>
<body>
<div id="viewer">
<div id="catalog-list">
<h2>STAC Items</h2>
<p>{{ config.get('message', '')}}</p>
<p>Select one <strong>or multiple</strong> items and then click next to iteratively build up a full request.</p>
<p>Last database update: <time>{{config.get('last_database_update', '')}}</time></p>
<div class="sidebar-header">
<button id="previous-btn">Previous</button>
<a id="stac-anchor"><button id="stac-btn">Raw STAC</button></a>
<button id="next-btn">Next</button>
</div>
<div id="items">
<!-- Items from the STAC catalog will be rendered here -->
</div>
</div>
<div id="details">
<h2>Current Selection</h2>
This is a <a href="https://github.com/ecmwf/datacube-spec/blob/main/spec/selection.md">MARS Selection</a> object in JSON format. Hover over a key or value for more info.
<!-- Container for the request part, preloaded to prevent layout shift. -->
<pre><code id="request-breakdown" class="language-json">
{
}
</code></pre>
<!-- Container to show the current tree -->
<h2>Currently Selected Tree</h2></summary>
<pre id = "qube"></pre>
<details>
<summary><h2>Example Qubed Code</h2></summary>
See the <a href="https://qubed.readthedocs.io/en/latest/">Qubed documentation</a> for more details.
<pre><code id="example-python" class="language-python">
# pip install qubed requests
import requests
from qubed import Qube
qube = Qube.from_json(requests.get("http://stac-catalogs.ecmwf-development.f.ewcloud.host/api/v1/select/climate-dt?{{request.url.query}}").json())
qube.print()
</code></pre>
</details>
<!-- Container fo the raw STAC response -->
<details>
<summary><h2>Raw STAC Response</h2></summary>
<p>See the <a href="https://github.com/ecmwf-projects/catalogs/blob/main/structured_stac.md">STAC Extension Proposal</a> for more details on the format.</p>
<pre class="json-pre"><code id="raw-stac" class="language-json"></code></pre>
</details>
<!-- Container for the debug response -->
<details>
<summary><h2>Debug Info</h2></summary>
<pre class="json-pre"><code id="debug" class="language-json"></code></pre>
</details>
</div>
</div>
<script>
window.API_URL = "{{ api_url }}";
</script>
<script src="/static/app.js"></script>
</body>
</html>