From 149ad1adc27a8d2b1c14270fcb50c72bc0c69776 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Sun, 25 May 2025 20:50:44 -0600 Subject: [PATCH 1/7] Add inputs to metadata for get_pvgis_hourly --- pvlib/iotools/pvgis.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 425cceb84c..9594aa9a94 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -130,8 +130,6 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, ------- data : pandas.DataFrame Time-series of hourly data, see Notes for fields - inputs : dict - Dictionary of the request input parameters metadata : dict Dictionary containing metadata @@ -189,7 +187,7 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, Examples -------- >>> # Retrieve two years of irradiance data from PVGIS: - >>> data, meta, inputs = pvlib.iotools.get_pvgis_hourly( # doctest: +SKIP + >>> data, meta = pvlib.iotools.get_pvgis_hourly( # doctest: +SKIP >>> latitude=45, longitude=8, start=2015, end=2016) # doctest: +SKIP References @@ -241,28 +239,32 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, def _parse_pvgis_hourly_json(src, map_variables): - inputs = src['inputs'] - metadata = src['meta'] + metadata = src['meta'].copy() + # Ovrride the "inputs" in metadata + metadata['inputs'] = src['inputs'] + # Re-add the inputs in metadata one-layer down + metadata['inputs']['descriptions'] = src['meta']['inputs'] data = pd.DataFrame(src['outputs']['hourly']) data.index = pd.to_datetime(data['time'], format='%Y%m%d:%H%M', utc=True) data = data.drop('time', axis=1) data = data.astype(dtype={'Int': 'int'}) # The 'Int' column to be integer if map_variables: data = data.rename(columns=VARIABLE_MAP) - return data, inputs, metadata + return data, metadata def _parse_pvgis_hourly_csv(src, map_variables): # The first 4 rows are latitude, longitude, elevation, radiation database - inputs = {} + metadata = {'inputs': {}} + # 'location' metadata # 'Latitude (decimal degrees): 45.000\r\n' - inputs['latitude'] = float(src.readline().split(':')[1]) + metadata['inputs']['latitude'] = float(src.readline().split(':')[1]) # 'Longitude (decimal degrees): 8.000\r\n' - inputs['longitude'] = float(src.readline().split(':')[1]) + metadata['inputs']['longitude'] = float(src.readline().split(':')[1]) # Elevation (m): 1389.0\r\n - inputs['elevation'] = float(src.readline().split(':')[1]) + metadata['inputs']['elevation'] = float(src.readline().split(':')[1]) # 'Radiation database: \tPVGIS-SARAH\r\n' - inputs['radiation_database'] = src.readline().split(':')[1].strip() + metadata['inputs']['radiation_database'] = src.readline().split(':')[1].strip() # Parse through the remaining metadata section (the number of lines for # this section depends on the requested parameters) while True: @@ -273,7 +275,7 @@ def _parse_pvgis_hourly_csv(src, map_variables): break # Only retrieve metadata from non-empty lines elif line.strip() != '': - inputs[line.split(':')[0]] = line.split(':')[1].strip() + metadata['inputs'][line.split(':')[0]] = line.split(':')[1].strip() elif line == '': # If end of file is reached raise ValueError('No data section was detected. File has probably ' 'been modified since being downloaded from PVGIS') @@ -295,11 +297,11 @@ def _parse_pvgis_hourly_csv(src, map_variables): # integer. It is necessary to convert to float, before converting to int data = data.astype(float).astype(dtype={'Int': 'int'}) # Generate metadata dictionary containing description of parameters - metadata = {} + metadata['inputs']['descriptions'] = {} for line in src.readlines(): if ':' in line: - metadata[line.split(':')[0]] = line.split(':')[1].strip() - return data, inputs, metadata + metadata['inputs']['descriptions'][line.split(':')[0]] = line.split(':')[1].strip() + return data, metadata def read_pvgis_hourly(filename, pvgis_format=None, map_variables=True): From f4cf5db4eac2ca63b85485caa73e815706b24517 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Sun, 25 May 2025 20:54:02 -0600 Subject: [PATCH 2/7] Update doc string for read_pvgis_hourly --- pvlib/iotools/pvgis.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 9594aa9a94..9d22c4057d 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -325,8 +325,6 @@ def read_pvgis_hourly(filename, pvgis_format=None, map_variables=True): ------- data : pandas.DataFrame the time series data - inputs : dict - the inputs metadata : dict metadata From 64f31a6b0e0aa57375114e9774f43e3015b38042 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Sun, 25 May 2025 21:16:11 -0600 Subject: [PATCH 3/7] Update tests --- pvlib/iotools/pvgis.py | 4 ++-- tests/iotools/test_pvgis.py | 26 +++++++++++++++++--------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 9d22c4057d..5d62c40059 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -297,10 +297,10 @@ def _parse_pvgis_hourly_csv(src, map_variables): # integer. It is necessary to convert to float, before converting to int data = data.astype(float).astype(dtype={'Int': 'int'}) # Generate metadata dictionary containing description of parameters - metadata['inputs']['descriptions'] = {} + metadata['descriptions'] = {} for line in src.readlines(): if ':' in line: - metadata['inputs']['descriptions'][line.split(':')[0]] = line.split(':')[1].strip() + metadata['descriptions'][line.split(':')[0]] = line.split(':')[1].strip() return data, metadata diff --git a/tests/iotools/test_pvgis.py b/tests/iotools/test_pvgis.py index 7c025f4113..cde7de3fb0 100644 --- a/tests/iotools/test_pvgis.py +++ b/tests/iotools/test_pvgis.py @@ -132,6 +132,15 @@ 'WS10m': {'description': '10-m total wind speed', 'units': 'm/s'}, # noqa: E501 'Int': {'description': '1 means solar radiation values are reconstructed'}}}}} # noqa: E501 +descriptions_csv = metadata_radiation_csv.copy() +metadata_radiation_csv = {} +metadata_radiation_csv['descriptions'] = descriptions_csv +metadata_radiation_csv['inputs'] = inputs_radiation_csv + +descriptions_json = metadata_pv_json['inputs'] +metadata_pv_json['inputs'] = inputs_pv_json +metadata_pv_json['inputs']['descriptions'] = descriptions_json + def generate_expected_dataframe(values, columns, index): """Create dataframe from arrays of values, columns and index, in order to @@ -175,25 +184,24 @@ def expected_pv_json_mapped(): # Test read_pvgis_hourly function using two different files with different # input arguments (to test variable mapping and pvgis_format) # pytest request.getfixturevalue is used to simplify the input arguments -@pytest.mark.parametrize('testfile,expected_name,metadata_exp,inputs_exp,map_variables,pvgis_format', [ # noqa: E501 +@pytest.mark.parametrize('testfile,expected_name,metadata_exp,map_variables,pvgis_format', [ # noqa: E501 (testfile_radiation_csv, 'expected_radiation_csv', metadata_radiation_csv, - inputs_radiation_csv, False, None), + False, None), (testfile_radiation_csv, 'expected_radiation_csv_mapped', - metadata_radiation_csv, inputs_radiation_csv, True, 'csv'), - (testfile_pv_json, 'expected_pv_json', metadata_pv_json, inputs_pv_json, + metadata_radiation_csv, True, 'csv'), + (testfile_pv_json, 'expected_pv_json', metadata_pv_json, False, None), (testfile_pv_json, 'expected_pv_json_mapped', metadata_pv_json, - inputs_pv_json, True, 'json')]) + True, 'json')]) def test_read_pvgis_hourly(testfile, expected_name, metadata_exp, - inputs_exp, map_variables, pvgis_format, request): + map_variables, pvgis_format, request): # Get expected dataframe from fixture expected = request.getfixturevalue(expected_name) # Read data from file - out, inputs, metadata = read_pvgis_hourly( + out, metadata = read_pvgis_hourly( testfile, map_variables=map_variables, pvgis_format=pvgis_format) # Assert whether dataframe, metadata, and inputs are as expected assert_frame_equal(out, expected) - assert inputs == inputs_exp assert metadata == metadata_exp @@ -248,7 +256,7 @@ def test_get_pvgis_hourly(requests_mock, testfile, expected_name, args, # inputs are passing on correctly requests_mock.get(url_test, text=mock_response) # Make API call - an error is raised if requested URI does not match - out, inputs, metadata = get_pvgis_hourly( + out, metadata = get_pvgis_hourly( latitude=45, longitude=8, map_variables=map_variables, **args) # Get expected dataframe from fixture expected = request.getfixturevalue(expected_name) From 63b66892e5fdd32012ef928f133da6d0ff8bea27 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Sun, 25 May 2025 21:19:05 -0600 Subject: [PATCH 4/7] Fix linter --- pvlib/iotools/pvgis.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 5d62c40059..95dd2f96e7 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -264,7 +264,8 @@ def _parse_pvgis_hourly_csv(src, map_variables): # Elevation (m): 1389.0\r\n metadata['inputs']['elevation'] = float(src.readline().split(':')[1]) # 'Radiation database: \tPVGIS-SARAH\r\n' - metadata['inputs']['radiation_database'] = src.readline().split(':')[1].strip() + metadata['inputs']['radiation_database'] = \ + src.readline().split(':')[1].strip() # Parse through the remaining metadata section (the number of lines for # this section depends on the requested parameters) while True: @@ -300,7 +301,8 @@ def _parse_pvgis_hourly_csv(src, map_variables): metadata['descriptions'] = {} for line in src.readlines(): if ':' in line: - metadata['descriptions'][line.split(':')[0]] = line.split(':')[1].strip() + metadata['descriptions'][line.split(':')[0]] = \ + line.split(':')[1].strip() return data, metadata From bc8967d9728725645143990b6de9e877c144801d Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Sun, 25 May 2025 21:24:58 -0600 Subject: [PATCH 5/7] Add whatsnew entry --- docs/sphinx/source/whatsnew/v0.12.1.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/sphinx/source/whatsnew/v0.12.1.rst b/docs/sphinx/source/whatsnew/v0.12.1.rst index 223c037f75..27f4947246 100644 --- a/docs/sphinx/source/whatsnew/v0.12.1.rst +++ b/docs/sphinx/source/whatsnew/v0.12.1.rst @@ -6,7 +6,10 @@ v0.12.1 (XXXX, 2025) Breaking Changes ~~~~~~~~~~~~~~~~ - +* The functions :py:func:`~pvlib.iotools.read_pvgis_hourly` and + :py:func:`~pvlib.iotools.get_pvgis_hourly` now returns ``(data,meta)`` + instead of ``(data,inputs,meta)`` following the iotools convention. + (:pull:`2462`) Deprecations ~~~~~~~~~~~~ From 206a8b3563ae1efc753182c176466b944e92a9e6 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Tue, 27 May 2025 11:30:05 -0600 Subject: [PATCH 6/7] Apply suggestions from code review Co-authored-by: Kevin Anderson --- docs/sphinx/source/whatsnew/v0.12.1.rst | 7 ++++--- pvlib/iotools/pvgis.py | 2 +- tests/iotools/test_pvgis.py | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.12.1.rst b/docs/sphinx/source/whatsnew/v0.12.1.rst index 27f4947246..b247da1961 100644 --- a/docs/sphinx/source/whatsnew/v0.12.1.rst +++ b/docs/sphinx/source/whatsnew/v0.12.1.rst @@ -7,9 +7,10 @@ v0.12.1 (XXXX, 2025) Breaking Changes ~~~~~~~~~~~~~~~~ * The functions :py:func:`~pvlib.iotools.read_pvgis_hourly` and - :py:func:`~pvlib.iotools.get_pvgis_hourly` now returns ``(data,meta)`` - instead of ``(data,inputs,meta)`` following the iotools convention. - (:pull:`2462`) + :py:func:`~pvlib.iotools.get_pvgis_hourly` now return ``(data,meta)`` + following the iotools convention instead of ``(data,inputs,meta)``. + The ``inputs`` dictionary is now included in ``meta``, which + has changed structure to accommodate it. (:pull:`2462`) Deprecations ~~~~~~~~~~~~ diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 95dd2f96e7..2931de8442 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -240,7 +240,7 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, def _parse_pvgis_hourly_json(src, map_variables): metadata = src['meta'].copy() - # Ovrride the "inputs" in metadata + # Override the "inputs" in metadata metadata['inputs'] = src['inputs'] # Re-add the inputs in metadata one-layer down metadata['inputs']['descriptions'] = src['meta']['inputs'] diff --git a/tests/iotools/test_pvgis.py b/tests/iotools/test_pvgis.py index cde7de3fb0..1019aca599 100644 --- a/tests/iotools/test_pvgis.py +++ b/tests/iotools/test_pvgis.py @@ -132,6 +132,7 @@ 'WS10m': {'description': '10-m total wind speed', 'units': 'm/s'}, # noqa: E501 'Int': {'description': '1 means solar radiation values are reconstructed'}}}}} # noqa: E501 +# Reformat the metadata as implemented in #2462 descriptions_csv = metadata_radiation_csv.copy() metadata_radiation_csv = {} metadata_radiation_csv['descriptions'] = descriptions_csv From 9a1f9612451e6e56a49b67277251442b0e3ae756 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Tue, 27 May 2025 11:44:45 -0600 Subject: [PATCH 7/7] Add versionchanged entries --- pvlib/iotools/pvgis.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 2931de8442..cd2207bbb1 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -57,6 +57,12 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, PVGIS data is freely available at [1]_. + .. versionchanged:: 0.13.0 + The function now returns two items ``(data,meta)``. Previous + versions of this function returned three elements + ``(data,inputs,meta)``. The ``inputs`` dictionary is now included in + ``meta``, which has changed structure to accommodate it. + Parameters ---------- latitude: float @@ -309,6 +315,12 @@ def _parse_pvgis_hourly_csv(src, map_variables): def read_pvgis_hourly(filename, pvgis_format=None, map_variables=True): """Read a PVGIS hourly file. + .. versionchanged:: 0.13.0 + The function now returns two items ``(data,meta)``. Previous + versions of this function returned three elements + ``(data,inputs,meta)``. The ``inputs`` dictionary is now included in + ``meta``, which has changed structure to accommodate it. + Parameters ---------- filename : str, pathlib.Path, or file-like buffer