Climate DT Parameter Series Plot- Data Access using DEDL HDA

import destinelab as deauth
import json
import datetime
import importlib.metadata

import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
import os
from getpass import getpass
from tqdm import tqdm
import time
from datetime import datetime
from urllib.parse import unquote
from time import sleep
from IPython.display import JSON

DESP_USERNAME = input("Please input your DESP username: ")
DESP_PASSWORD = getpass("Please input your DESP password: ")

auth = deauth.AuthHandler(DESP_USERNAME, DESP_PASSWORD)
access_token = auth.get_token()
if access_token is not None:
    print("DEDL/DESP Access Token Obtained Successfully")
else:
    print("Failed to Obtain DEDL/DESP Access Token")

auth_headers = {"Authorization": f"Bearer {access_token}"}

import importlib
installed_version = importlib.metadata.version("destinelab")
version_number = installed_version.split('.')[1]
if((int(version_number) >= 8 and float(installed_version) < 1) or float(installed_version) >= 1):
    auth.is_DTaccess_allowed(access_token)

HDA_STAC_ENDPOINT="https://hda.data.destination-earth.eu/stac/v2"
COLLECTION_ID="EO.ECMWF.DAT.DT_CLIMATE_ADAPTATION"

datechoice = "20280610"

filter_params = {
        "ecmwf:class": "d1",             # fixed 
        "ecmwf:dataset": "climate-dt",   # fixed climate-dt access
        "ecmwf:activity": "ScenarioMIP", # activity + experiment + model (go together)
        "ecmwf:experiment": "SSP3-7.0",  # activity + experiment + model (go together)
        "ecmwf:model": "IFS-NEMO",       # activity + experiment + model (go together)
        "ecmwf:generation": "1",         # fixed Specifies the generation of the dataset, which can be incremented as required (latest is 1)
        "ecmwf:realization": "1",        # fixed Specifies the climate realization. Default 1. Based on perturbations of initial conditions
        "ecmwf:resolution": "high",      # standard/ high 
        "ecmwf:expver": "0001",          # fixed experiment version 
        "ecmwf:stream": "clte",          # fixed climate
        "ecmwf:time": "0000",            # choose the hourly slot(s)
        "ecmwf:type": "fc",              # fixed forecasted fields
        "ecmwf:levtype": "sfc",          # Surface fields (levtype=sfc), Height level fields (levtype=hl), Pressure level fields (levtype=pl), Model Level (Levtype=ml)
#        "levelist": "1/2/3/...",  # for ml/pl/sol type data
        "ecmwf:param": "167",             # 2m Temperature parameter
        "ecmwf:date":datechoice+"/to/"+datechoice
    }

# Define date choice and filters if needed
datechoice = "2024-07-01"

# Initialize a list to store filenames
filenames = []

# Define start and end years
start_year = 2024
start_month = 7
end_year = 2028

#timeout and step for polling (sec)
TIMEOUT = 300
STEP = 1
ONLINE_STATUS = "online"

# Loop 
for year in range(start_year, end_year + 1):
    # Create a datetime object 
    obsdate = datetime(year, start_month, 1)
    datechoice = obsdate.strftime("%Y%m%d")
    filter_params["ecmwf:date"]=datechoice+"/to/"+datechoice
    response = requests.post(f"{HDA_STAC_ENDPOINT}/collections/{COLLECTION_ID}/order", json=filter_params, headers=auth_headers)

    if response.status_code != 200:
        print(response.content)
    response.raise_for_status()

    ordered_item = response.json()

    product_id = ordered_item["id"]
    storage_tier = ordered_item["properties"].get("storage:tier", "online")
    order_status = ordered_item["properties"].get("order:status", "unknown")
    federation_backend = ordered_item["properties"].get("federation:backends", [None])[0]

    print(f"Product ordered: {product_id}")
    print(f"Provider: {federation_backend}")
    print(f"Storage tier: {storage_tier} (product must have storage tier \"online\" to be downloadable)")
    print(f"Order status: {order_status}")      

    self_url = f"{HDA_STAC_ENDPOINT}/collections/{COLLECTION_ID}/items/{product_id}"
    item = {}

    for i in range(0, TIMEOUT, STEP):
        print(f"Polling {i + 1}/{TIMEOUT // STEP}")

        response = requests.get(self_url, headers=auth_headers)
        if response.status_code != 200:
            print(response.content)
        response.raise_for_status()
        item = response.json()

        storage_tier = item["properties"].get("storage:tier", ONLINE_STATUS)

        if storage_tier == ONLINE_STATUS:
            download_url = item["assets"]["downloadLink"]["href"]
            print("Product is ready to be downloaded.")
            print(f"Asset URL: {download_url}")
            break
        sleep(STEP)
    else:
        order_status = item["properties"].get("order:status", "unknown")
        print(f"We could not download the product after {TIMEOUT // STEP} tries. Current order status is {order_status}")
    
    response = requests.get(download_url, stream=True, headers=auth_headers)
    response.raise_for_status()

    content_disposition = response.headers.get('Content-Disposition')
    total_size = int(response.headers.get("content-length", 0))
    if content_disposition:
        filename = content_disposition.split('filename=')[1].strip('"')
        filename = unquote(filename)
    else:
        filename = os.path.basename(url)

    # Open a local file in binary write mode and write the content
    print(f"downloading {filename}")

    with tqdm(total=total_size, unit="B", unit_scale=True) as progress_bar:
        with open(filename, 'wb') as f:
            for data in response.iter_content(1024):
                progress_bar.update(len(data))
                f.write(data)
               
    # Add the filename to the list
    filenames.append(filename)

import earthkit.data
import earthkit.plots
import earthkit.regrid

# Iterate over filenames
for filename in filenames:
    print(filename)  # For example, print each filename
    data = earthkit.data.from_source("file", filename)
    earthkit.plots.quickplot(data)