add web_query_builder

This commit is contained in:
Tom Hodson 2024-11-21 13:57:16 +00:00
parent e00a5ff919
commit 50d86c77ec
6 changed files with 640 additions and 0 deletions

55
web_query_builder/app.py Normal file
View File

@ -0,0 +1,55 @@
from flask import (
Flask,
render_template,
request,
redirect,
Response,
)
import requests
from flask_cors import CORS
from werkzeug.middleware.proxy_fix import ProxyFix
app = Flask(__name__)
CORS(app, resources={r"/api/*": {"origins": "*"}})
# This is required because when running in k8s the flask server sits behind a TLS proxy
# So flask speaks http while the client speaks https
# Client <-- https ---> Proxy <---- http ---> Flask server
# For the Oauth flow, flask needs to provide a callback url and it needs to use the right scheme=https
# This line tells flask to look at HTTP headers set by the TLS proxy to figure out what the original
# Traffic looked like.
# See https://flask.palletsprojects.com/en/3.0.x/deploying/proxy_fix/
app.wsgi_app = ProxyFix(
app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1
)
config = {}
@app.route("/")
def index():
return render_template("index.html", request = request, config = config)
# @app.route('/stac', methods=["GET", "POST"]) # ref. https://medium.com/@zwork101/making-a-flask-proxy-server-online-in-10-lines-of-code-44b8721bca6
# def redirect_to_API_HOST(): #NOTE var :subpath will be unused as all path we need will be read from :request ie from flask import request
# url = f'http://localhost:8124/stac'
# res = requests.request( # ref. https://stackoverflow.com/a/36601467/248616
# method = request.method,
# url = url,
# headers = {k:v for k,v in request.headers if k.lower() != 'host'}, # exclude 'host' header
# data = request.get_data(),
# cookies = request.cookies,
# allow_redirects = False,
# )
# excluded_headers = ['content-encoding', 'content-length', 'transfer-encoding', 'connection'] #NOTE we here exclude all "hop-by-hop headers" defined by RFC 2616 section 13.5.1 ref. https://www.rfc-editor.org/rfc/rfc2616#section-13.5.1
# headers = [
# (k,v) for k,v in res.raw.headers.items()
# if k.lower() not in excluded_headers
# ]
# response = Response(res.content, res.status_code, headers)
# return response

View File

@ -0,0 +1,7 @@
flask==3
pyyaml
flask_dance
python-dotenv
flask-login
flask-cors
cachetools

1
web_query_builder/run.sh Executable file
View File

@ -0,0 +1 @@
flask run --debug --port=5005

View File

@ -0,0 +1,308 @@
// app.js
// Take the query string and stick it on the API URL
function getSTACUrlFromQuery() {
const params = new URLSearchParams(window.location.search);
// get current window url and remove path part
let api_url = new URL(window.location.href);
api_url.pathname = "/stac";
for (const [key, value] of params.entries()) {
api_url.searchParams.set(key, value);
}
console.log(api_url.toString());
return api_url.toString();
}
function get_request_from_url() {
// Extract the query params in order and split any with a , delimiter
// request is an ordered array of [key, [value1, value2, value3, ...]]
const url = new URL(window.location.href);
const params = new URLSearchParams(url.search);
const request = [];
for (const [key, value] of params.entries()) {
request.push([key, value.split(",")]);
}
return request;
}
function make_url_from_request(request) {
const url = new URL(window.location.href);
url.search = ""; // Clear existing params
const params = new URLSearchParams();
for (const [key, values] of request) {
params.set(key, values.join(","));
}
url.search = params.toString();
return url.toString().replace(/%2C/g, ",");
}
function goToPreviousUrl() {
let request = get_request_from_url();
request.pop();
console.log("Request:", request);
const url = make_url_from_request(request);
console.log("URL:", url);
window.location.href = make_url_from_request(request);
}
// Function to generate a new STAC URL based on current selection
function goToNextUrl() {
const request = get_request_from_url();
// Get the currently selected key = value,value2,value3 pairs
const items = Array.from(document.querySelectorAll("div#items > div"));
let any_new_keys = false;
const new_keys = items.map((item) => {
const key = item.dataset.key;
const key_type = item.dataset.keyType;
let values = [];
const datePicker = item.querySelector("input[type='date']");
if (datePicker) {
values.push(datePicker.value.replace(/-/g, ""));
}
const timePicker = item.querySelector("input[type='time']");
if (timePicker) {
values.push(timePicker.value.replace(":", ""));
}
const enum_checkboxes = item.querySelectorAll("input[type='checkbox']:checked");
if (enum_checkboxes.length > 0) {
values.push(...Array.from(enum_checkboxes).map((checkbox) => checkbox.value));
}
const any = item.querySelector("input[type='text']");
if (any && any.value !== "") {
values.push(any.value);
}
// Keep track of whether any new keys are selected
if (values.length > 0) {
any_new_keys = true;
}
return { key, values };
});
// if not new keys are selected, do nothing
if (!any_new_keys) {
return;
}
// Update the request with the new keys
for (const { key, values } of new_keys) {
// Find the index of the existing key in the request array
const existingIndex = request.findIndex(
([existingKey, existingValues]) => existingKey === key
);
if (existingIndex !== -1) {
// If the key already exists,
// and the values aren't already in there,
// append the values
request[existingIndex][1] = [...request[existingIndex][1], ...values];
} else {
// If the key doesn't exist, add a new entry
request.push([key, values]);
}
}
const url = make_url_from_request(request);
window.location.href = url;
}
async function createCatalogItem(link, itemsContainer) {
const itemDiv = document.createElement("div");
itemDiv.className = "item loading";
itemDiv.textContent = "Loading...";
itemsContainer.appendChild(itemDiv);
try {
// Update the item div with real content
itemDiv.classList.remove("loading");
const dimension = link["generalized_datacube:dimension"];
// add data-key attribute to the itemDiv
itemDiv.dataset.key = link.title;
itemDiv.dataset.keyType = dimension.type;
itemDiv.innerHTML = `
<h3 class="item-title">${link.title || "No title available"}</h3>
<p class="item-type">Key Type: ${itemDiv.dataset.keyType || "Unknown"}</p>
<!-- <p class="item-type">Paths: ${dimension.paths}</p> -->
<p class="item-type">Optional: ${dimension.optional ? "Yes" : "No"}</p>
<p class="item-description">${dimension.description ? dimension.description.slice(0, 100) : "No description available"}...</p>
`;
// if (dimension.type === "date" || dimension.type === "time") {
// // Render a date picker for the "date" key
// const picker = `<input type="${link.title}" name="${link.title}">`;
// //convert picker to HTML node
// const pickerNode = document
// .createRange()
// .createContextualFragment(picker);
// itemDiv.appendChild(pickerNode);
// }
// Otherwise create a scrollable list with checkboxes for values if available
if (
// dimension.type === "enum" &&
dimension.values &&
dimension.values.length > 0
) {
const listContainer = renderCheckboxList(link);
itemDiv.appendChild(listContainer);
} else {
const any = `<input type="text" name="${link.title}">`;
const anyNode = document.createRange().createContextualFragment(any);
itemDiv.appendChild(anyNode);
}
} catch (error) {
console.error("Error loading item data:", error);
itemDiv.innerHTML = `<p>Error loading item details: ${error}</p>`;
}
}
function renderCheckboxList(link) {
const dimension = link["generalized_datacube:dimension"];
const value_descriptions = dimension.value_descriptions || [];
const listContainerHTML = `
<div class="item-list-container">
<label class="list-label">Select one or more values:</label>
<div class="scrollable-list">
${dimension.values
.map((value, index) => {
const labelText = value_descriptions[index] ? `${value} - ${value_descriptions[index]}` : value;
return `
<div class="checkbox-container">
<label class="checkbox-label">
<input type="checkbox" class="item-checkbox" value="${value}" ${dimension.values.length === 1? 'checked' : ''}>
${labelText}
</label>
</div>
`;
})
.join("")}
</div>
</div>
`;
return document.createRange().createContextualFragment(listContainerHTML).firstElementChild;
}
// Render catalog items in the sidebar
function renderCatalogItems(links) {
const itemsContainer = document.getElementById("items");
itemsContainer.innerHTML = ""; // Clear previous items
console.log("Number of Links:", links);
const children = links.filter(
(link) => link.rel === "child" || link.rel === "items"
);
console.log("Number of Children:", children.length);
children.forEach((link) => {
createCatalogItem(link, itemsContainer);
});
}
function renderRequestBreakdown(request, descriptions) {
const container = document.getElementById("request-breakdown");
const format_value = (key, value) => {
return `<span class="value" title="${descriptions[key]['value_descriptions'][value]}">"${value}"</span>`;
};
const format_values = (key, values) => {
if (values.length === 1) {
return format_value(key, values[0]);
}
return `[${values.map((v) =>
format_value(key, v)
).join(", ")}]`;
};
let html = `{\n` +
request
.map(
([key, values]) =>
` <span class="key" title="${descriptions[key]['description']}">"${key}"</span>: ${format_values(key, values)},`
)
.join("\n") +
`\n}`;
container.innerHTML = html;
}
function renderRawSTACResponse(catalog) {
const itemDetails = document.getElementById("raw-stac");
// create new object without debug key
let just_stac = Object.assign({}, catalog);
delete just_stac.debug;
itemDetails.textContent = JSON.stringify(just_stac, null, 2);
const debug_container = document.getElementById("debug");
// create new object without debug key
debug_container.textContent = JSON.stringify(catalog.debug, null, 2);
}
// Fetch STAC catalog and display items
async function fetchCatalog(request, stacUrl) {
try {
const response = await fetch(stacUrl);
const catalog = await response.json();
// Render the request breakdown in the sidebar
renderRequestBreakdown(request, catalog.debug.descriptions);
// Show the raw STAC in the sidebar
renderRawSTACResponse(catalog);
// Render the items from the catalog
if (catalog.links) {
console.log("Fetched STAC catalog:", stacUrl, catalog.links);
renderCatalogItems(catalog.links);
}
// Highlight the request and raw STAC
hljs.highlightElement(document.getElementById("raw-stac"));
hljs.highlightElement(document.getElementById("debug"));
} catch (error) {
console.error("Error fetching STAC catalog:", error);
}
}
// Initialize the viewer by fetching the STAC catalog
function initializeViewer() {
const stacUrl = getSTACUrlFromQuery();
const request = get_request_from_url();
if (stacUrl) {
console.log("Fetching STAC catalog from query string URL:", stacUrl);
fetchCatalog(request, stacUrl);
} else {
console.error("No STAC URL provided in the query string.");
}
// Add event listener for the "Generate STAC URL" button
const generateUrlBtn = document.getElementById("next-btn");
generateUrlBtn.addEventListener("click", goToNextUrl);
const previousUrlBtn = document.getElementById("previous-btn");
previousUrlBtn.addEventListener("click", goToPreviousUrl);
// Add event listener for the "Raw STAC" button
const stacAnchor = document.getElementById("stac-anchor");
stacAnchor.href = getSTACUrlFromQuery();
}
// Call initializeViewer on page load
initializeViewer();

View File

@ -0,0 +1,211 @@
html,
body {
min-height: 100vh;
height: 100%;
}
body {
font-family: Arial, sans-serif;
margin: 0;
padding-left: 0.5em;
padding-right: 0.5em;
}
#viewer {
display: flex;
flex-direction: row;
height: fit-content;
min-height: 100vh;
}
#catalog-list {
width: 30%;
padding: 10px;
overflow-y: scroll;
background-color: #f4f4f4;
border-right: 1px solid #ddd;
}
#catalog-list h2 {
margin-top: 0;
}
#details {
width: 70%;
padding: 10px;
}
.sidebar-header {
display: flex;
justify-content: center;
margin-bottom: 10px;
flex-wrap: wrap;
gap: 0.5em;
}
.sidebar-header button {
width: 10em;
}
canvas {
width: 100%;
height: 300px;
border: 1px solid #ccc;
margin-top: 20px;
}
/* Updated CSS for the item elements in the catalog list */
.item {
background-color: white;
border: 1px solid #ddd;
padding: 10px;
margin-bottom: 10px;
border-radius: 5px;
transition: background-color 0.2s ease;
}
.item-title {
font-size: 18px;
margin: 0;
color: #333;
}
.item-type {
font-size: 14px;
margin: 5px 0;
color: #666;
}
.item-id,
.item-key-type {
font-size: 12px;
color: #999;
}
.item-description {
font-size: 13px;
margin: 5px 0;
color: #444;
font-style: italic;
}
.item.selected {
background-color: #d4e9ff;
/* Lighter blue for selection */
border-color: #003399;
/* Keep the original ECMWF blue for the border */
}
summary h2 {
display: inline;
}
.json-pre {
white-space: pre-wrap;
/* background-color: #f9f9f9; */
border: 1px solid #ccc;
border-radius: 5px;
padding: 10px;
}
/* Button styles */
button {
height: 3em;
padding: 10px 20px;
/* Padding around button text */
margin: 0 5px;
/* Margin between buttons */
background-color: #003399;
/* ECMWF blue */
color: white;
/* White text color */
border: none;
/* Remove default button border */
cursor: pointer;
/* Pointer cursor on hover */
border-radius: 5px;
/* Rounded corners */
transition: background-color 0.3s ease;
/* Smooth background color transition */
}
button:hover {
background-color: #001f66;
/* Darker shade of ECMWF blue on hover */
}
.item-list-container {
margin-top: 20px;
margin-bottom: 20px;
}
.scrollable-list {
max-height: 200px;
overflow-y: auto;
padding: 10px;
border: 1px solid #ccc;
border-radius: 4px;
background-color: #fff;
box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1);
}
.checkbox-container {
display: flex;
align-items: center;
margin-bottom: 10px;
}
.item-checkbox {
margin-right: 10px;
cursor: pointer;
}
.checkbox-label {
font-size: 16px;
color: #333;
}
.checkbox-container:hover .checkbox-label {
color: #003399;
}
.list-label {
font-weight: bold;
margin-bottom: 0.5em;
display: block;
color: #003399;
}
span.key,
span.value {
color: #ba2121;
;
}
span.key {
font-weight: bold;
}
span.key:hover,
span.value:hover {
color: #ff2a2a;
cursor: pointer;
}
/* Change layout for narrow viewport */
@media (max-width: 800px) {
#viewer {
flex-direction: column;
}
#catalog-list {
width: 100%;
border-right: none;
}
#details {
width: 100%;
}
}

View File

@ -0,0 +1,58 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>ECMWF DestinE STAC Viewer</title>
<link rel="stylesheet" href="/static/styles.css" />
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github.min.css">
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/json.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/python.min.js"></script>
<link rel="icon" href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 100 100%22><text y=%22.9em%22 font-size=%2290%22>📚</text></svg>">
</head>
<body>
<div id="viewer">
<div id="catalog-list">
<h2>STAC Items</h2>
<p>{{ config.get('message', '')}}</p>
<p>Select one <strong>or multiple</strong> items and then click next to iteratively build up a full request.</p>
<p>Last database update: <time>{{config.get('last_database_update', '')}}</time></p>
<div class="sidebar-header">
<button id="previous-btn">Previous</button>
<a id="stac-anchor"><button id="stac-btn">Raw STAC</button></a>
<button id="next-btn">Next</button>
</div>
<div id="items">
<!-- Items from the STAC catalog will be rendered here -->
</div>
</div>
<div id="details">
<h2>Current Request</h2>
Hover over a key or value for more info.
<!-- Container for the request part, preloaded to prevent layout shift. -->
<pre><code id="request-breakdown" class="language-json">
{
}
</code></pre>
<!-- Container fo the raw STAC response -->
<details open>
<summary><h2>Raw STAC Response</h2></summary>
<p>See the <a href="https://github.com/ecmwf-projects/catalogs/blob/main/structured_stac.md">extension proposal</a> for more details on the format.</p>
<pre class="json-pre"><code id="raw-stac" class="language-json"></code></pre>
</details>
<!-- Container for the debug response -->
<details>
<summary><h2>Debug Info</h2></summary>
<pre class="json-pre"><code id="debug" class="language-json"></code></pre>
</details>
</div>
</div>
<script src="/static/app.js"></script>
</body>
</html>