{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Loading data directly from ESGF nodes" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* This notebook demonstrates searching for and loading CMIP simulations from ESGF, using the intake_esgf package.\n", "* This code is taken from the intake-esgf tutorial. Please refer there for a fuller explanation of each step. \n", "* The data search uses the standard CMIP directory structure, see here for an explanation. We make use of xarray; see this turorial for an introduction. " ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": true, "jupyter": { "outputs_hidden": true } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting intake_esgf\n", " Downloading intake_esgf-2024.12.7-py3-none-any.whl.metadata (4.0 kB)\n", "Requirement already satisfied: pandas in /srv/conda/envs/notebook/lib/python3.12/site-packages (from intake_esgf) (2.2.3)\n", "Requirement already satisfied: dask in /srv/conda/envs/notebook/lib/python3.12/site-packages (from intake_esgf) (2025.1.0)\n", "Requirement already satisfied: xarray in /srv/conda/envs/notebook/lib/python3.12/site-packages (from intake_esgf) (2025.1.1)\n", "Requirement already satisfied: netCDF4 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from intake_esgf) (1.7.2)\n", "Collecting globus-sdk (from intake_esgf)\n", " Downloading globus_sdk-3.55.0-py3-none-any.whl.metadata (3.3 kB)\n", "Requirement already satisfied: requests in /srv/conda/envs/notebook/lib/python3.12/site-packages (from intake_esgf) (2.32.3)\n", "Requirement already satisfied: tqdm[notebook] in /srv/conda/envs/notebook/lib/python3.12/site-packages (from intake_esgf) (4.67.1)\n", "Requirement already satisfied: pyyaml in /srv/conda/envs/notebook/lib/python3.12/site-packages (from intake_esgf) (6.0.2)\n", "Requirement already satisfied: click>=8.1 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from dask->intake_esgf) (8.1.8)\n", "Requirement already satisfied: cloudpickle>=3.0.0 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from dask->intake_esgf) (3.1.1)\n", "Requirement already satisfied: fsspec>=2021.09.0 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from dask->intake_esgf) (2024.12.0)\n", "Requirement already satisfied: packaging>=20.0 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from dask->intake_esgf) (24.2)\n", "Requirement already satisfied: partd>=1.4.0 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from dask->intake_esgf) (1.4.2)\n", "Requirement already satisfied: toolz>=0.10.0 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from dask->intake_esgf) (1.0.0)\n", "Requirement already satisfied: pyjwt<3.0.0,>=2.0.0 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from pyjwt[crypto]<3.0.0,>=2.0.0->globus-sdk->intake_esgf) (2.10.1)\n", "Requirement already satisfied: cryptography!=3.4.0,>=3.3.1 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from globus-sdk->intake_esgf) (43.0.1)\n", "Requirement already satisfied: charset_normalizer<4,>=2 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from requests->intake_esgf) (3.4.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from requests->intake_esgf) (3.10)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from requests->intake_esgf) (1.26.19)\n", "Requirement already satisfied: certifi>=2017.4.17 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from requests->intake_esgf) (2024.12.14)\n", "Requirement already satisfied: cftime in /srv/conda/envs/notebook/lib/python3.12/site-packages (from netCDF4->intake_esgf) (1.6.4)\n", "Requirement already satisfied: numpy in /srv/conda/envs/notebook/lib/python3.12/site-packages (from netCDF4->intake_esgf) (2.0.2)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from pandas->intake_esgf) (2.9.0)\n", "Requirement already satisfied: pytz>=2020.1 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from pandas->intake_esgf) (2024.1)\n", "Requirement already satisfied: tzdata>=2022.7 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from pandas->intake_esgf) (2025.1)\n", "Requirement already satisfied: ipywidgets>=6 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from tqdm[notebook]->intake_esgf) (8.1.5)\n", "Requirement already satisfied: cffi>=1.12 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from cryptography!=3.4.0,>=3.3.1->globus-sdk->intake_esgf) (1.17.1)\n", "Requirement already satisfied: comm>=0.1.3 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from ipywidgets>=6->tqdm[notebook]->intake_esgf) (0.2.2)\n", "Requirement already satisfied: ipython>=6.1.0 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from ipywidgets>=6->tqdm[notebook]->intake_esgf) (8.17.2)\n", "Requirement already satisfied: traitlets>=4.3.1 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from ipywidgets>=6->tqdm[notebook]->intake_esgf) (5.14.3)\n", "Requirement already satisfied: widgetsnbextension~=4.0.12 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from ipywidgets>=6->tqdm[notebook]->intake_esgf) (4.0.13)\n", "Requirement already satisfied: jupyterlab_widgets~=3.0.12 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from ipywidgets>=6->tqdm[notebook]->intake_esgf) (3.0.13)\n", "Requirement already satisfied: locket in /srv/conda/envs/notebook/lib/python3.12/site-packages (from partd>=1.4.0->dask->intake_esgf) (1.0.0)\n", "Requirement already satisfied: six>=1.5 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas->intake_esgf) (1.17.0)\n", "Requirement already satisfied: pycparser in /srv/conda/envs/notebook/lib/python3.12/site-packages (from cffi>=1.12->cryptography!=3.4.0,>=3.3.1->globus-sdk->intake_esgf) (2.22)\n", "Requirement already satisfied: decorator in /srv/conda/envs/notebook/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets>=6->tqdm[notebook]->intake_esgf) (5.1.1)\n", "Requirement already satisfied: jedi>=0.16 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets>=6->tqdm[notebook]->intake_esgf) (0.19.2)\n", "Requirement already satisfied: matplotlib-inline in /srv/conda/envs/notebook/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets>=6->tqdm[notebook]->intake_esgf) (0.1.7)\n", "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets>=6->tqdm[notebook]->intake_esgf) (3.0.38)\n", "Requirement already satisfied: pygments>=2.4.0 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets>=6->tqdm[notebook]->intake_esgf) (2.19.1)\n", "Requirement already satisfied: stack-data in /srv/conda/envs/notebook/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets>=6->tqdm[notebook]->intake_esgf) (0.6.3)\n", "Requirement already satisfied: pexpect>4.3 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets>=6->tqdm[notebook]->intake_esgf) (4.9.0)\n", "Requirement already satisfied: parso<0.9.0,>=0.8.4 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets>=6->tqdm[notebook]->intake_esgf) (0.8.4)\n", "Requirement already satisfied: ptyprocess>=0.5 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets>=6->tqdm[notebook]->intake_esgf) (0.7.0)\n", "Requirement already satisfied: wcwidth in /srv/conda/envs/notebook/lib/python3.12/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets>=6->tqdm[notebook]->intake_esgf) (0.2.13)\n", "Requirement already satisfied: executing>=1.2.0 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=6->tqdm[notebook]->intake_esgf) (2.1.0)\n", "Requirement already satisfied: asttokens>=2.1.0 in /srv/conda/envs/notebook/lib/python3.12/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=6->tqdm[notebook]->intake_esgf) (3.0.0)\n", "Requirement already satisfied: pure_eval in /srv/conda/envs/notebook/lib/python3.12/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=6->tqdm[notebook]->intake_esgf) (0.2.3)\n", "Downloading intake_esgf-2024.12.7-py3-none-any.whl (36 kB)\n", "Downloading globus_sdk-3.55.0-py3-none-any.whl (408 kB)\n", "Installing collected packages: globus-sdk, intake_esgf\n", "Successfully installed globus-sdk-3.55.0 intake_esgf-2024.12.7\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ "# on starting the server, we need to run the line below once, as intake_esgf is not yet in our standard pangeo environment\n", "%pip install intake_esgf" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "import intake_esgf\n", "cat = ESGFCatalog()" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", " | cf_standard_name | \n", "variable_units | \n", "variable_long_name | \n", "
---|---|---|---|
variable_id | \n", "\n", " | \n", " | \n", " |
hfls | \n", "surface_upward_latent_heat_flux | \n", "W m-2 | \n", "Surface Upward Latent Heat Flux | \n", "
hfss | \n", "surface_upward_sensible_heat_flux | \n", "W m-2 | \n", "Surface Upward Sensible Heat Flux | \n", "
rlds | \n", "surface_downwelling_longwave_flux_in_air | \n", "W m-2 | \n", "Surface Downwelling Longwave Radiation | \n", "
rsds | \n", "surface_downwelling_shortwave_flux_in_air | \n", "W m-2 | \n", "Surface Downwelling Shortwave Radiation | \n", "
sfcWind | \n", "wind_speed | \n", "m s-1 | \n", "Near-Surface Wind Speed | \n", "
ta | \n", "air_temperature | \n", "K | \n", "Air Temperature | \n", "
tas | \n", "air_temperature | \n", "K | \n", "Near-Surface Air Temperature | \n", "
tasmax | \n", "air_temperature | \n", "K | \n", "Daily Maximum Near-Surface Air Temperature | \n", "
tasmin | \n", "air_temperature | \n", "K | \n", "Daily Minimum Near-Surface Air Temperature | \n", "
vas | \n", "northward_wind | \n", "m s-1 | \n", "Northward Near-Surface Wind | \n", "