From 50d86c77ec529692aec8ee63db4c373c1f4a00b5 Mon Sep 17 00:00:00 2001 From: Tom Hodson <thomas.hodson@ecmwf.int> Date: Thu, 21 Nov 2024 13:57:16 +0000 Subject: [PATCH] add web_query_builder --- web_query_builder/app.py | 55 +++++ web_query_builder/requirements.txt | 7 + web_query_builder/run.sh | 1 + web_query_builder/static/app.js | 308 +++++++++++++++++++++++++ web_query_builder/static/styles.css | 211 +++++++++++++++++ web_query_builder/templates/index.html | 58 +++++ 6 files changed, 640 insertions(+) create mode 100644 web_query_builder/app.py create mode 100644 web_query_builder/requirements.txt create mode 100755 web_query_builder/run.sh create mode 100644 web_query_builder/static/app.js create mode 100644 web_query_builder/static/styles.css create mode 100644 web_query_builder/templates/index.html diff --git a/web_query_builder/app.py b/web_query_builder/app.py new file mode 100644 index 0000000..6790e60 --- /dev/null +++ b/web_query_builder/app.py @@ -0,0 +1,55 @@ +from flask import ( + Flask, + render_template, + request, + redirect, + Response, +) +import requests +from flask_cors import CORS + +from werkzeug.middleware.proxy_fix import ProxyFix + +app = Flask(__name__) +CORS(app, resources={r"/api/*": {"origins": "*"}}) + +# This is required because when running in k8s the flask server sits behind a TLS proxy +# So flask speaks http while the client speaks https +# Client <-- https ---> Proxy <---- http ---> Flask server +# For the Oauth flow, flask needs to provide a callback url and it needs to use the right scheme=https +# This line tells flask to look at HTTP headers set by the TLS proxy to figure out what the original +# Traffic looked like. +# See https://flask.palletsprojects.com/en/3.0.x/deploying/proxy_fix/ +app.wsgi_app = ProxyFix( + app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1 +) + +config = {} + +@app.route("/") +def index(): + return render_template("index.html", request = request, config = config) + + + +# @app.route('/stac', methods=["GET", "POST"]) # ref. https://medium.com/@zwork101/making-a-flask-proxy-server-online-in-10-lines-of-code-44b8721bca6 +# def redirect_to_API_HOST(): #NOTE var :subpath will be unused as all path we need will be read from :request ie from flask import request +# url = f'http://localhost:8124/stac' +# res = requests.request( # ref. https://stackoverflow.com/a/36601467/248616 +# method = request.method, +# url = url, +# headers = {k:v for k,v in request.headers if k.lower() != 'host'}, # exclude 'host' header +# data = request.get_data(), +# cookies = request.cookies, +# allow_redirects = False, +# ) + +# excluded_headers = ['content-encoding', 'content-length', 'transfer-encoding', 'connection'] #NOTE we here exclude all "hop-by-hop headers" defined by RFC 2616 section 13.5.1 ref. https://www.rfc-editor.org/rfc/rfc2616#section-13.5.1 +# headers = [ +# (k,v) for k,v in res.raw.headers.items() +# if k.lower() not in excluded_headers +# ] + +# response = Response(res.content, res.status_code, headers) +# return response + diff --git a/web_query_builder/requirements.txt b/web_query_builder/requirements.txt new file mode 100644 index 0000000..b75273f --- /dev/null +++ b/web_query_builder/requirements.txt @@ -0,0 +1,7 @@ +flask==3 +pyyaml +flask_dance +python-dotenv +flask-login +flask-cors +cachetools \ No newline at end of file diff --git a/web_query_builder/run.sh b/web_query_builder/run.sh new file mode 100755 index 0000000..d1a12b8 --- /dev/null +++ b/web_query_builder/run.sh @@ -0,0 +1 @@ +flask run --debug --port=5005 \ No newline at end of file diff --git a/web_query_builder/static/app.js b/web_query_builder/static/app.js new file mode 100644 index 0000000..a89853b --- /dev/null +++ b/web_query_builder/static/app.js @@ -0,0 +1,308 @@ +// app.js + +// Take the query string and stick it on the API URL +function getSTACUrlFromQuery() { + const params = new URLSearchParams(window.location.search); + + // get current window url and remove path part + let api_url = new URL(window.location.href); + api_url.pathname = "/stac"; + + for (const [key, value] of params.entries()) { + api_url.searchParams.set(key, value); + } + + console.log(api_url.toString()); + return api_url.toString(); +} + +function get_request_from_url() { + // Extract the query params in order and split any with a , delimiter + // request is an ordered array of [key, [value1, value2, value3, ...]] + const url = new URL(window.location.href); + const params = new URLSearchParams(url.search); + const request = []; + for (const [key, value] of params.entries()) { + request.push([key, value.split(",")]); + } + return request; +} + +function make_url_from_request(request) { + const url = new URL(window.location.href); + url.search = ""; // Clear existing params + const params = new URLSearchParams(); + + for (const [key, values] of request) { + params.set(key, values.join(",")); + } + url.search = params.toString(); + + return url.toString().replace(/%2C/g, ","); +} + +function goToPreviousUrl() { + let request = get_request_from_url(); + request.pop(); + console.log("Request:", request); + const url = make_url_from_request(request); + console.log("URL:", url); + window.location.href = make_url_from_request(request); +} + +// Function to generate a new STAC URL based on current selection +function goToNextUrl() { + const request = get_request_from_url(); + + // Get the currently selected key = value,value2,value3 pairs + const items = Array.from(document.querySelectorAll("div#items > div")); + + let any_new_keys = false; + const new_keys = items.map((item) => { + const key = item.dataset.key; + const key_type = item.dataset.keyType; + let values = []; + + const datePicker = item.querySelector("input[type='date']"); + if (datePicker) { + values.push(datePicker.value.replace(/-/g, "")); + } + + const timePicker = item.querySelector("input[type='time']"); + if (timePicker) { + values.push(timePicker.value.replace(":", "")); + } + + const enum_checkboxes = item.querySelectorAll("input[type='checkbox']:checked"); + if (enum_checkboxes.length > 0) { + values.push(...Array.from(enum_checkboxes).map((checkbox) => checkbox.value)); + } + + const any = item.querySelector("input[type='text']"); + if (any && any.value !== "") { + values.push(any.value); + } + + // Keep track of whether any new keys are selected + if (values.length > 0) { + any_new_keys = true; + } + + return { key, values }; + }); + + // if not new keys are selected, do nothing + if (!any_new_keys) { + return; + } + + // Update the request with the new keys + for (const { key, values } of new_keys) { + // Find the index of the existing key in the request array + const existingIndex = request.findIndex( + ([existingKey, existingValues]) => existingKey === key + ); + + if (existingIndex !== -1) { + // If the key already exists, + // and the values aren't already in there, + // append the values + request[existingIndex][1] = [...request[existingIndex][1], ...values]; + } else { + // If the key doesn't exist, add a new entry + request.push([key, values]); + } + } + + const url = make_url_from_request(request); + window.location.href = url; +} + +async function createCatalogItem(link, itemsContainer) { + const itemDiv = document.createElement("div"); + itemDiv.className = "item loading"; + itemDiv.textContent = "Loading..."; + itemsContainer.appendChild(itemDiv); + + try { + + // Update the item div with real content + itemDiv.classList.remove("loading"); + + const dimension = link["generalized_datacube:dimension"]; + + // add data-key attribute to the itemDiv + itemDiv.dataset.key = link.title; + itemDiv.dataset.keyType = dimension.type; + + itemDiv.innerHTML = ` + <h3 class="item-title">${link.title || "No title available"}</h3> + <p class="item-type">Key Type: ${itemDiv.dataset.keyType || "Unknown"}</p> + <!-- <p class="item-type">Paths: ${dimension.paths}</p> --> + <p class="item-type">Optional: ${dimension.optional ? "Yes" : "No"}</p> + <p class="item-description">${dimension.description ? dimension.description.slice(0, 100) : "No description available"}...</p> + `; + + + // if (dimension.type === "date" || dimension.type === "time") { + // // Render a date picker for the "date" key + // const picker = `<input type="${link.title}" name="${link.title}">`; + // //convert picker to HTML node + // const pickerNode = document + // .createRange() + // .createContextualFragment(picker); + // itemDiv.appendChild(pickerNode); + // } + // Otherwise create a scrollable list with checkboxes for values if available + if ( + // dimension.type === "enum" && + dimension.values && + dimension.values.length > 0 + ) { + const listContainer = renderCheckboxList(link); + itemDiv.appendChild(listContainer); + } else { + const any = `<input type="text" name="${link.title}">`; + const anyNode = document.createRange().createContextualFragment(any); + itemDiv.appendChild(anyNode); + } + } catch (error) { + console.error("Error loading item data:", error); + itemDiv.innerHTML = `<p>Error loading item details: ${error}</p>`; + } +} + +function renderCheckboxList(link) { + const dimension = link["generalized_datacube:dimension"]; + const value_descriptions = dimension.value_descriptions || []; + + const listContainerHTML = ` + <div class="item-list-container"> + <label class="list-label">Select one or more values:</label> + <div class="scrollable-list"> + ${dimension.values + .map((value, index) => { + const labelText = value_descriptions[index] ? `${value} - ${value_descriptions[index]}` : value; + return ` + <div class="checkbox-container"> + <label class="checkbox-label"> + <input type="checkbox" class="item-checkbox" value="${value}" ${dimension.values.length === 1? 'checked' : ''}> + ${labelText} + </label> + </div> + `; + }) + .join("")} + </div> + </div> + `; + + return document.createRange().createContextualFragment(listContainerHTML).firstElementChild; + } + +// Render catalog items in the sidebar +function renderCatalogItems(links) { + const itemsContainer = document.getElementById("items"); + itemsContainer.innerHTML = ""; // Clear previous items + + console.log("Number of Links:", links); + const children = links.filter( + (link) => link.rel === "child" || link.rel === "items" + ); + console.log("Number of Children:", children.length); + + children.forEach((link) => { + createCatalogItem(link, itemsContainer); + }); +} + +function renderRequestBreakdown(request, descriptions) { + const container = document.getElementById("request-breakdown"); + const format_value = (key, value) => { + return `<span class="value" title="${descriptions[key]['value_descriptions'][value]}">"${value}"</span>`; + }; + + const format_values = (key, values) => { + if (values.length === 1) { + return format_value(key, values[0]); + } + return `[${values.map((v) => + format_value(key, v) + ).join(", ")}]`; + }; + + let html = `{\n` + + request + .map( + ([key, values]) => + ` <span class="key" title="${descriptions[key]['description']}">"${key}"</span>: ${format_values(key, values)},` + ) + .join("\n") + + `\n}`; + container.innerHTML = html; + } + +function renderRawSTACResponse(catalog) { + const itemDetails = document.getElementById("raw-stac"); + // create new object without debug key + let just_stac = Object.assign({}, catalog); + delete just_stac.debug; + itemDetails.textContent = JSON.stringify(just_stac, null, 2); + + const debug_container = document.getElementById("debug"); + // create new object without debug key + debug_container.textContent = JSON.stringify(catalog.debug, null, 2); +} + +// Fetch STAC catalog and display items +async function fetchCatalog(request, stacUrl) { + try { + const response = await fetch(stacUrl); + const catalog = await response.json(); + + // Render the request breakdown in the sidebar + renderRequestBreakdown(request, catalog.debug.descriptions); + + // Show the raw STAC in the sidebar + renderRawSTACResponse(catalog); + + // Render the items from the catalog + if (catalog.links) { + console.log("Fetched STAC catalog:", stacUrl, catalog.links); + renderCatalogItems(catalog.links); + } + + // Highlight the request and raw STAC + hljs.highlightElement(document.getElementById("raw-stac")); + hljs.highlightElement(document.getElementById("debug")); + } catch (error) { + console.error("Error fetching STAC catalog:", error); + } +} + +// Initialize the viewer by fetching the STAC catalog +function initializeViewer() { + const stacUrl = getSTACUrlFromQuery(); + const request = get_request_from_url(); + + if (stacUrl) { + console.log("Fetching STAC catalog from query string URL:", stacUrl); + fetchCatalog(request, stacUrl); + } else { + console.error("No STAC URL provided in the query string."); + } + + // Add event listener for the "Generate STAC URL" button + const generateUrlBtn = document.getElementById("next-btn"); + generateUrlBtn.addEventListener("click", goToNextUrl); + + const previousUrlBtn = document.getElementById("previous-btn"); + previousUrlBtn.addEventListener("click", goToPreviousUrl); + + // Add event listener for the "Raw STAC" button + const stacAnchor = document.getElementById("stac-anchor"); + stacAnchor.href = getSTACUrlFromQuery(); +} + +// Call initializeViewer on page load +initializeViewer(); diff --git a/web_query_builder/static/styles.css b/web_query_builder/static/styles.css new file mode 100644 index 0000000..a90f66f --- /dev/null +++ b/web_query_builder/static/styles.css @@ -0,0 +1,211 @@ +html, +body { + min-height: 100vh; + height: 100%; +} + +body { + font-family: Arial, sans-serif; + margin: 0; + padding-left: 0.5em; + padding-right: 0.5em; + +} + +#viewer { + display: flex; + flex-direction: row; + height: fit-content; + min-height: 100vh; +} + +#catalog-list { + width: 30%; + padding: 10px; + overflow-y: scroll; + background-color: #f4f4f4; + border-right: 1px solid #ddd; +} + +#catalog-list h2 { + margin-top: 0; +} + +#details { + width: 70%; + padding: 10px; +} + +.sidebar-header { + display: flex; + justify-content: center; + margin-bottom: 10px; + flex-wrap: wrap; + gap: 0.5em; +} + +.sidebar-header button { + width: 10em; +} + +canvas { + width: 100%; + height: 300px; + border: 1px solid #ccc; + margin-top: 20px; +} + +/* Updated CSS for the item elements in the catalog list */ +.item { + background-color: white; + border: 1px solid #ddd; + padding: 10px; + margin-bottom: 10px; + border-radius: 5px; + transition: background-color 0.2s ease; +} + +.item-title { + font-size: 18px; + margin: 0; + color: #333; +} + +.item-type { + font-size: 14px; + margin: 5px 0; + color: #666; +} + +.item-id, +.item-key-type { + font-size: 12px; + color: #999; +} + +.item-description { + font-size: 13px; + margin: 5px 0; + color: #444; + font-style: italic; +} + +.item.selected { + background-color: #d4e9ff; + /* Lighter blue for selection */ + border-color: #003399; + /* Keep the original ECMWF blue for the border */ +} + +summary h2 { + display: inline; +} + +.json-pre { + white-space: pre-wrap; + /* background-color: #f9f9f9; */ + border: 1px solid #ccc; + border-radius: 5px; + padding: 10px; +} + + +/* Button styles */ +button { + height: 3em; + padding: 10px 20px; + /* Padding around button text */ + margin: 0 5px; + /* Margin between buttons */ + background-color: #003399; + /* ECMWF blue */ + color: white; + /* White text color */ + border: none; + /* Remove default button border */ + cursor: pointer; + /* Pointer cursor on hover */ + border-radius: 5px; + /* Rounded corners */ + transition: background-color 0.3s ease; + /* Smooth background color transition */ +} + +button:hover { + background-color: #001f66; + /* Darker shade of ECMWF blue on hover */ +} + +.item-list-container { + margin-top: 20px; + margin-bottom: 20px; +} + +.scrollable-list { + max-height: 200px; + overflow-y: auto; + padding: 10px; + border: 1px solid #ccc; + border-radius: 4px; + background-color: #fff; + box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1); +} + +.checkbox-container { + display: flex; + align-items: center; + margin-bottom: 10px; +} + +.item-checkbox { + margin-right: 10px; + cursor: pointer; +} + +.checkbox-label { + font-size: 16px; + color: #333; +} + +.checkbox-container:hover .checkbox-label { + color: #003399; +} + +.list-label { + font-weight: bold; + margin-bottom: 0.5em; + display: block; + color: #003399; +} + +span.key, +span.value { + color: #ba2121; + ; +} + +span.key { + font-weight: bold; +} + +span.key:hover, +span.value:hover { + color: #ff2a2a; + cursor: pointer; +} + +/* Change layout for narrow viewport */ +@media (max-width: 800px) { + #viewer { + flex-direction: column; + } + + #catalog-list { + width: 100%; + border-right: none; + } + + #details { + width: 100%; + } +} \ No newline at end of file diff --git a/web_query_builder/templates/index.html b/web_query_builder/templates/index.html new file mode 100644 index 0000000..6f2c7e1 --- /dev/null +++ b/web_query_builder/templates/index.html @@ -0,0 +1,58 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <title>ECMWF DestinE STAC Viewer</title> + <link rel="stylesheet" href="/static/styles.css" /> + <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github.min.css"> + <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script> + <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/json.min.js"></script> + <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/python.min.js"></script> + <link rel="icon" href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 100 100%22><text y=%22.9em%22 font-size=%2290%22>📚</text></svg>"> + +</head> +<body> + <div id="viewer"> + <div id="catalog-list"> + <h2>STAC Items</h2> + <p>{{ config.get('message', '')}}</p> + <p>Select one <strong>or multiple</strong> items and then click next to iteratively build up a full request.</p> + <p>Last database update: <time>{{config.get('last_database_update', '')}}</time></p> + <div class="sidebar-header"> + <button id="previous-btn">Previous</button> + <a id="stac-anchor"><button id="stac-btn">Raw STAC</button></a> + <button id="next-btn">Next</button> + </div> + + <div id="items"> + <!-- Items from the STAC catalog will be rendered here --> + </div> + </div> + <div id="details"> + <h2>Current Request</h2> + Hover over a key or value for more info. + <!-- Container for the request part, preloaded to prevent layout shift. --> + <pre><code id="request-breakdown" class="language-json"> +{ +} + </code></pre> + + <!-- Container fo the raw STAC response --> + <details open> + <summary><h2>Raw STAC Response</h2></summary> + <p>See the <a href="https://github.com/ecmwf-projects/catalogs/blob/main/structured_stac.md">extension proposal</a> for more details on the format.</p> + <pre class="json-pre"><code id="raw-stac" class="language-json"></code></pre> + </details> + + <!-- Container for the debug response --> + <details> + <summary><h2>Debug Info</h2></summary> + <pre class="json-pre"><code id="debug" class="language-json"></code></pre> + </details> + </div> + </div> + + <script src="/static/app.js"></script> +</body> +</html> \ No newline at end of file