HDA Tutorial - Queryables

# Collection https://hda.data.destination-earth.eu/ui/dataset/EO.ECMWF.DAT.CAMS_EUROPE_AIR_QUALITY_FORECASTS as default
COLLECTION_ID = "EO.ECMWF.DAT.CAMS_EUROPE_AIR_QUALITY_FORECASTS"

# Core API
HDA_API_URL = "https://hda.data.destination-earth.eu"

# STAC API
## Core
STAC_API_URL = f"{HDA_API_URL}/stac"

## Item Search
SEARCH_URL = f"{STAC_API_URL}/search"

## Collections
COLLECTIONS_URL = f"{STAC_API_URL}/collections"

## Queryables
QUERYABLES_URL = f"{STAC_API_URL}/queryables"
QUERYABLES_BY_COLLECTION_ID = f"{COLLECTIONS_URL}/{COLLECTION_ID}/queryables"
HDA_FILTERS =''

## HTTP Success
HTTP_SUCCESS_CODE = 200

import destinelab as deauth

pip install --quiet --upgrade DEDLUtils

from DEDLUtils import dedl_utilities
#import dedl_utilities

import requests
import json
from getpass import getpass

import ipywidgets as widgets
from IPython.display import display, clear_output, HTML
from ipywidgets import Layout, Box
import datetime

from rich.console import Console
import rich.table

from IPython.display import JSON

DESP_USERNAME = input("Please input your DESP username or email: ")
DESP_PASSWORD = getpass("Please input your DESP password: ")

auth = deauth.AuthHandler(DESP_USERNAME, DESP_PASSWORD)
access_token = auth.get_token()
if access_token is not None:
    print("DEDL/DESP Access Token Obtained Successfully")
else:
    print("Failed to Obtain DEDL/DESP Access Token")

auth_headers = {"Authorization": f"Bearer {access_token}"}

prefix = "EO.ECMWF"

dedlUtils=dedl_utilities.DEDLUtilities(COLLECTION_ID )
dedlUtils.create_collections_dropdown(prefix)

# Layout
# Define the layout for the dropdown
dropdown_layout = Layout(display='space-between', justify_content='center', width='90%')
# Create a box to hold the dropdown with the specified layout
box = Box([dedlUtils.dropdown, dedlUtils.outputArea], layout=dropdown_layout)
display( box)  

filters_resp=requests.get(dedlUtils.queryablesByCollectionId)
filters_resp.raise_for_status()

filters = filters_resp.json()["properties"]
dedlQueryablesUtils=dedl_utilities.DEDLQueryablesUtilities(dedlUtils.collectionId)
console = Console()
console.print(dedlQueryablesUtils.create_queryables_table(filters))

# Event listeners
dedlQueryablesUtils=dedl_utilities.DEDLQueryablesUtilities(dedlUtils.collectionId)
dedlQueryablesUtils.update_queryables_dropdowns()
display(dedlQueryablesUtils.dropdownContainer, dedlQueryablesUtils.outputArea)

auth.is_DTaccess_allowed(access_token)

# The JSON objects containing the generic query parameters:
jsonGenericQuery = '{"collections": ["'+dedlUtils.collectionId+'"], "datetime": "2024-04-01T00:00:00Z/2024-04-19T00:00:00Z"}'
# Convert JSON strings to Python dictionaries
dictQuery = json.loads(jsonGenericQuery)

# Include the filters selected in the previous steps inside the JSON containing the generic query parameters:
dictQuery['query'] = dedlQueryablesUtils.hdaFilters
# Convert the merged dictionary back to a JSON string
queryJson = json.dumps(dictQuery, indent=4)

print(queryJson)

response = requests.post("https://hda.data.destination-earth.eu/stac/search", headers=auth_headers, json= json.loads(queryJson) )
#print(response)
# Requests to ADS data always return a single item containing all the requested data
product = response.json()["features"][0]
JSON(product, expanded= False)

download_url = product["assets"]["downloadLink"]["href"]
print(download_url )

from tqdm import tqdm
import time
import re
direct_download_url=''

response = requests.get(download_url, headers=auth_headers)
if (response.status_code == HTTP_SUCCESS_CODE):
    direct_download_url = product['assets']['downloadLink']['href']
else:
    print(response.text)


# we poll as long as the data is not ready
if direct_download_url=='':
    while url := response.headers.get("Location"):
        print(f"order status: {response.json()['status']}")
        response = requests.get(url, headers=auth_headers, stream=True)
        response.raise_for_status()

filename = re.findall('filename=\"?(.+)\"?', response.headers["Content-Disposition"])[0]
total_size = int(response.headers.get("content-length", 0))

print(f"downloading {filename}")

with tqdm(total=total_size, unit="B", unit_scale=True) as progress_bar:
    with open(filename, 'wb') as f:
        for data in response.iter_content(1024):
            progress_bar.update(len(data))
            f.write(data)