{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Make 4D fields (to netCDF)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## with integrated functionalities: meta-resources" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "collapsed": false }, "outputs": [], "source": [ "import os\n", "import epygram\n", "epygram.init_env()\n", "workdir = epygram.config.userlocaldir + '/notebooks_data'\n", "os.chdir(workdir)\n", "import numpy" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "collapsed": false }, "outputs": [], "source": [ "files = ['advanced_examples/ICMSHAROM+{:0>4}'.format(i) for i in range(0,2+1)]" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# meta-resource are a wrapper over raw resources, enabling them to combine H2D fields on a vertical discretization\n", "# as 3D fields, or combine fields from a temporal series of resources as a field with a temporal dimension\n", "r = epygram.resources.meta_resource(files, 'r', 'MV+CL') # MV = Multi-Validities, CL = Combine-Levels\n", "\n", "# such meta-resources (CL) need a 'generic' or GRIB2 kind of fid to read in, for recombining coherent levels\n", "# of course it underneath needs a fid conversion, which is implemented basically only for FA and LFI\n", "t = r.readfield({'discipline':0, 'parameterCategory':0, 'parameterNumber':0, # temperature, as GRIB2\n", " 'typeOfFirstFixedSurface':119}) # on hybrid-pressure (model) levels" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(3, 60, 423256)\n", "(3, 60, 720, 750)\n" ] } ], "source": [ "print(t.data.shape)\n", "t.sp2gp()\n", "print(t.data.shape)" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "collapsed": false }, "outputs": [], "source": [ "rh = r.readfield({'discipline':0, 'parameterCategory':1, 'parameterNumber':1,\n", " 'typeOfFirstFixedSurface':103, 'level':2}) # relative humidity at 2m, as GRIB2" ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "collapsed": false }, "outputs": [], "source": [ "out = epygram.formats.resource('out4D.nc', 'w', fmt='netCDF') # open the output netCDF" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "collapsed": false }, "outputs": [], "source": [ "t.fid['netCDF'] = 'temperature'\n", "out.writefield(t)" ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "collapsed": false }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "# [2017/08/25-10:52:38][epygram.formats.netCDF][writefield:1039][INFO]: assume lons/lats match.\n", "# [2017/08/25-10:52:38][epygram.formats.netCDF][writefield:1144][INFO]: assume projection parameters match.\n" ] } ], "source": [ "rh.fid['netCDF'] = 'relative_humidity'\n", "out.writefield(rh)" ] }, { "cell_type": "code", "execution_count": 38, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "### FORMAT: netCDF\n", "\n", "NetCDF Global Attributes:\n", "\tmade_with: u'epygram-1.2.10'\n", "\tConventions: u'CF-1.6'\n", "NetCDF dimension information:\n", "\tName: time\n", "\t\tsize: 3\n", "\tName: Z\n", "\t\tsize: 60\n", "\tName: Y\n", "\t\tsize: 720\n", "\tName: X\n", "\t\tsize: 750\n", "\tName: Z+1\n", "\t\tsize: 61\n", "NetCDF variable information:\n", "\tName: time\n", "\t\tdimensions: (u'time',)\n", "\t\tsize: 3\n", "\t\ttype: dtype('float64')\n", "\t\tunits: u'seconds since 2015-02-16 00:00:00'\n", "\tName: Z\n", "\t\tdimensions: ()\n", "\t\tsize: 1.0\n", "\t\ttype: dtype('int64')\n", "\t\tstandard_name: u'atmosphere_hybrid_sigma_pressure_coordinate'\n", "\t\tpositive: u'down'\n", "\t\tformula_terms: u'ap: hybrid_coef_A b: hybrid_coef_B ps: surface_air_pressure'\n", "\t\tshort_name: u'hybrid-pressure'\n", "\tName: hybrid_coef_A\n", "\t\tdimensions: (u'Z+1',)\n", "\t\tsize: 61\n", "\t\ttype: dtype('float64')\n", "\tName: hybrid_coef_B\n", "\t\tdimensions: (u'Z+1',)\n", "\t\tsize: 61\n", "\t\ttype: dtype('float64')\n", "\tName: longitude\n", "\t\tdimensions: (u'Y', u'X')\n", "\t\tsize: 540000\n", "\t\ttype: dtype('float64')\n", "\t\tunits: u'degrees'\n", "\tName: latitude\n", "\t\tdimensions: (u'Y', u'X')\n", "\t\tsize: 540000\n", "\t\ttype: dtype('float64')\n", "\t\tunits: u'degrees'\n", "\tName: Projection_parameters\n", "\t\tdimensions: ()\n", "\t\tsize: 1.0\n", "\t\ttype: dtype('int64')\n", "\t\tgrid_mapping_name: u'lambert_conformal_conic'\n", "\t\tearth_radius: 6371229.0\n", "\t\tx_resolution: 2500.0\n", "\t\ty_resolution: 2500.0\n", "\t\tlongitude_of_central_meridian: 2.0\n", "\t\tlatitude_of_projection_origin: 45.800000000000004\n", "\t\tstandard_parallel: 45.800000000000004\n", "\t\tfalse_easting: 922500.0\n", "\t\tfalse_northing: 885000.0000025304\n", "\tName: temperature\n", "\t\tdimensions: (u'time', u'Z', u'Y', u'X')\n", "\t\tsize: 97200000\n", "\t\ttype: dtype('float64')\n", "\t\tgrid_mapping: u'Projection_parameters'\n", "\t\tvertical_grid: u'Z'\n", "\tName: relative_humidity\n", "\t\tdimensions: (u'time', u'Y', u'X')\n", "\t\tsize: 1620000\n", "\t\ttype: dtype('float64')\n", "\t\tgrid_mapping: u'Projection_parameters'\n" ] } ], "source": [ "out.close()\n", "out.open(openmode='r')\n", "out.what()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## or do it manually" ] }, { "cell_type": "code", "execution_count": 39, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['advanced_examples/GRIDFRANGP0025r0_0000', 'advanced_examples/GRIDFRANGP0025r0_0001', 'advanced_examples/GRIDFRANGP0025r0_0002']\n" ] } ], "source": [ "files = ['advanced_examples/GRIDFRANGP0025r0_{:0>4}'.format(i) for i in range(0,2+1)] # these are GRIB1 files\n", "print(files)" ] }, { "cell_type": "code", "execution_count": 40, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# try with Combine-Levels meta-resource\n", "r = epygram.resources.meta_resource(files[0], 'r', 'CL')" ] }, { "cell_type": "code", "execution_count": 41, "metadata": { "collapsed": false }, "outputs": [ { "ename": "AssertionError", "evalue": "Not able to combine levels if fields do not have 'generic' fids", "output_type": "error", "traceback": [ "\u001b[0;31m\u001b[0m", "\u001b[0;31mAssertionError\u001b[0mTraceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m t = r.readfield({'indicatorOfParameter':11,\n\u001b[0;32m----> 2\u001b[0;31m 'indicatorOfTypeOfLevel':100}) # temperature on Pressure levels, as GRIB1\n\u001b[0m", "\u001b[0;32m/home/mary/EPyGrAM/src/epygram/resources/CombineLevelsResource.pyc\u001b[0m in \u001b[0;36mreadfield\u001b[0;34m(self, handgrip, getdata)\u001b[0m\n\u001b[1;32m 158\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0mparam\u001b[0m \u001b[0mgetdata\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdo\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mread\u001b[0m \u001b[0mdata\u001b[0m \u001b[0mbut\u001b[0m \u001b[0monly\u001b[0m \u001b[0mmetadata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 159\u001b[0m \"\"\"\n\u001b[0;32m--> 160\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadfields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandgrip\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhandgrip\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgetdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgetdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 161\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 162\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mepygramError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"field(s) have been found, only one expected.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/mary/EPyGrAM/src/epygram/resources/CombineLevelsResource.pyc\u001b[0m in \u001b[0;36mreadfields\u001b[0;34m(self, handgrip, getdata)\u001b[0m\n\u001b[1;32m 171\u001b[0m \"\"\"\n\u001b[1;32m 172\u001b[0m \u001b[0mfieldset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFieldSet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 173\u001b[0;31m \u001b[0mcont\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 174\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfid\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistfields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselect\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhandgrip\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 175\u001b[0m \u001b[0mfound\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/mary/EPyGrAM/src/epygram/resources/CombineLevelsResource.pyc\u001b[0m in \u001b[0;36m_create_list\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 97\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfid\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresource\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistfields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcomplete\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0;34m'generic'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfid\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 99\u001b[0;31m \u001b[0;34m\"Not able to combine levels if fields do not have 'generic' fids\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 100\u001b[0m \u001b[0moriginal_fid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfid\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfmtfid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresource\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 101\u001b[0m \u001b[0mgeneric_fid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfid\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'generic'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mAssertionError\u001b[0m: Not able to combine levels if fields do not have 'generic' fids" ] } ], "source": [ "t = r.readfield({'indicatorOfParameter':11,\n", " 'indicatorOfTypeOfLevel':100}) # temperature on Pressure levels, as GRIB1" ] }, { "cell_type": "code", "execution_count": 42, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# This did not worked because 'CL' meta-resource need generic/GRIB2 => let's do it manually" ] }, { "cell_type": "code", "execution_count": 43, "metadata": { "collapsed": false }, "outputs": [], "source": [ "r = epygram.resources.meta_resource(files, 'r', 'MV') # this works anyway" ] }, { "cell_type": "code", "execution_count": 44, "metadata": { "collapsed": false }, "outputs": [], "source": [ "t = r.readfield({'indicatorOfParameter':11,\n", " 'indicatorOfTypeOfLevel':100,\n", " 'level':850}) # temperature, as GRIB1" ] }, { "cell_type": "code", "execution_count": 45, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "(3, 601, 801)" ] }, "execution_count": 45, "metadata": {}, "output_type": "execute_result" } ], "source": [ "t.data.shape" ] }, { "cell_type": "code", "execution_count": 46, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(3, 601, 801)\n" ] } ], "source": [ "# either, we could have aggregated time dimension manually:\n", "temp_time = [epygram.formats.resource(f, 'r').readfield({'indicatorOfParameter':11,\n", " 'indicatorOfTypeOfLevel':100,\n", " 'level':850})\n", " for f in files]\n", "fld_t = temp_time[0]\n", "for fld in temp_time[1:]:\n", " fld_t.extend(fld)\n", "print(fld_t.data.shape)" ] }, { "cell_type": "code", "execution_count": 47, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[1000, 500, 600, 700, 800, 100, 150, 850, 200, 250, 900, 925, 950, 300, 400]\n" ] } ], "source": [ "# now let's gather vertical levels\n", "levels = r.listfields(select={'indicatorOfParameter':11,\n", " 'indicatorOfTypeOfLevel':100},\n", " onlykey='level')\n", "print(levels)" ] }, { "cell_type": "code", "execution_count": 48, "metadata": { "collapsed": false }, "outputs": [], "source": [ "temp_levels = [] # we can read levels one by one as 2D+T; vertical aggregation will have to be done manually\n", "for l in sorted(levels, reverse=True):\n", " t = r.readfield({'indicatorOfParameter':11,\n", " 'indicatorOfTypeOfLevel':100,\n", " 'level':l})\n", " temp_levels.append(t)" ] }, { "cell_type": "code", "execution_count": 49, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "[,\n", " ,\n", " ,\n", " ,\n", " ,\n", " ,\n", " ,\n", " ,\n", " ,\n", " ,\n", " ,\n", " ,\n", " ,\n", " ,\n", " ]" ] }, "execution_count": 49, "metadata": {}, "output_type": "execute_result" } ], "source": [ "temp_levels" ] }, { "cell_type": "code", "execution_count": 50, "metadata": { "collapsed": false }, "outputs": [], "source": [ "temp_4D_data = numpy.concatenate([f.getdata(d4=True) for f in temp_levels], axis=1) \n", "# axis = 1 because data in epygram fields is stored with order: (t,z,y,x)" ] }, { "cell_type": "code", "execution_count": 51, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "(3, 15, 601, 801)" ] }, "execution_count": 51, "metadata": {}, "output_type": "execute_result" } ], "source": [ "temp_4D_data.shape" ] }, { "cell_type": "code", "execution_count": 52, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# then create a 4Dfield from a 2D+T one\n", "temp_4d = temp_levels[0].deepcopy() # now still 2D+T\n", "# create an ad hoc vertical geometry, making a copy where we just change the list of levels\n", "vgeom_4d = temp_4d.geometry.vcoordinate.footprint_clone(extra={'levels':sorted(levels, reverse=True)})\n", "temp_4d.geometry.vcoordinate = vgeom_4d" ] }, { "cell_type": "code", "execution_count": 53, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# finally, set 4D data inside\n", "temp_4d.setdata(temp_4D_data)" ] }, { "cell_type": "code", "execution_count": 54, "metadata": { "collapsed": false }, "outputs": [], "source": [ "temp_4d.fid['netCDF'] = 'temperature'\n", "out = epygram.formats.resource('out4D_manual.nc', 'w', fmt='netCDF')\n", "out.writefield(temp_4d)" ] }, { "cell_type": "code", "execution_count": 55, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "### FORMAT: netCDF\n", "\n", "NetCDF Global Attributes:\n", "\tmade_with: u'epygram-1.2.10'\n", "\tConventions: u'CF-1.6'\n", "NetCDF dimension information:\n", "\tName: time\n", "\t\tsize: 3\n", "\tName: Z\n", "\t\tsize: 15\n", "\tName: Y\n", "\t\tsize: 601\n", "\tName: X\n", "\t\tsize: 801\n", "NetCDF variable information:\n", "\tName: time\n", "\t\tdimensions: (u'time',)\n", "\t\tsize: 3\n", "\t\ttype: dtype('float64')\n", "\t\tunits: u'seconds since 2015-02-16 00:00:00'\n", "\tName: Z\n", "\t\tdimensions: (u'Z',)\n", "\t\tsize: 15\n", "\t\ttype: dtype('float64')\n", "\t\tunits: u'hPa'\n", "\t\tshort_name: u'pressure'\n", "\tName: longitude\n", "\t\tdimensions: (u'Y', u'X')\n", "\t\tsize: 481401\n", "\t\ttype: dtype('float64')\n", "\t\tunits: u'degrees'\n", "\tName: latitude\n", "\t\tdimensions: (u'Y', u'X')\n", "\t\tsize: 481401\n", "\t\ttype: dtype('float64')\n", "\t\tunits: u'degrees'\n", "\tName: temperature\n", "\t\tdimensions: (u'time', u'Z', u'Y', u'X')\n", "\t\tsize: 21663045\n", "\t\ttype: dtype('float64')\n", "\t\tunits: u'K'\n" ] } ], "source": [ "out.close()\n", "out.open(openmode='r')\n", "out.what()" ] } ], "metadata": { "kernelspec": { "display_name": "Python 2", "language": "python", "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.12" } }, "nbformat": 4, "nbformat_minor": 0 }