From f4e4ad3bbe335fa49bf300ce53784d69d719ca98 Mon Sep 17 00:00:00 2001 From: bhellis725 Date: Mon, 16 Jul 2018 09:33:42 -0700 Subject: [PATCH 1/8] Fixed inconsistencies between the detect_clearsky function and the original reference. --- pvlib/clearsky.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index 75f598baeb..94fc43cf5b 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -687,7 +687,10 @@ def detect_clearsky(measured, clearsky, times, window_length, raise NotImplementedError('algorithm does not yet support unequal ' \ 'times. consider resampling your data.') - samples_per_window = int(window_length / sample_interval) + samples_per_window = int(window_length / sample_interval) + 1 + # print(window_length) + # print(sample_interval) + # print(samples_per_window) # generate matrix of integers for creating windows with indexing from scipy.linalg import hankel @@ -697,25 +700,27 @@ def detect_clearsky(measured, clearsky, times, window_length, # calculate measurement statistics meas_mean = np.mean(measured[H], axis=0) meas_max = np.max(measured[H], axis=0) - meas_slope = np.diff(measured[H], n=1, axis=0) + meas_ghi_diff = np.diff(measured[H], n=1, axis=0) + meas_slope = np.diff(measured[H], n=1, axis=0) / sample_interval # matlab std function normalizes by N-1, so set ddof=1 here meas_slope_nstd = np.std(meas_slope, axis=0, ddof=1) / meas_mean - meas_slope_max = np.max(np.abs(meas_slope), axis=0) + # meas_slope_max = np.max(np.abs(meas_slope), axis=0) meas_line_length = np.sum(np.sqrt( - meas_slope*meas_slope + sample_interval*sample_interval), axis=0) + meas_ghi_diff*meas_ghi_diff + sample_interval*sample_interval), axis=0) # calculate clear sky statistics clear_mean = np.mean(clearsky[H], axis=0) clear_max = np.max(clearsky[H], axis=0) - clear_slope = np.diff(clearsky[H], n=1, axis=0) - clear_slope_max = np.max(np.abs(clear_slope), axis=0) + clear_ghi_diff = np.diff(clearsky[H], n=1, axis=0) + clear_slope = np.diff(clearsky[H], n=1, axis=0) / sample_interval + # clear_slope_max = np.max(np.abs(clear_slope), axis=0) from scipy.optimize import minimize_scalar alpha = 1 for iteration in range(max_iterations): clear_line_length = np.sum(np.sqrt( - alpha*alpha*clear_slope*clear_slope + + alpha*alpha*clear_ghi_diff*clear_ghi_diff + sample_interval*sample_interval), axis=0) line_diff = meas_line_length - clear_line_length @@ -725,7 +730,8 @@ def detect_clearsky(measured, clearsky, times, window_length, c2 = np.abs(meas_max - alpha*clear_max) < max_diff c3 = (line_diff > lower_line_length) & (line_diff < upper_line_length) c4 = meas_slope_nstd < var_diff - c5 = (meas_slope_max - alpha*clear_slope_max) < slope_dev + # c5 = (meas_slope_max - alpha*clear_slope_max) < slope_dev + c5 = np.max(meas_slope - alpha*clear_slope, axis=0) < slope_dev c6 = (clear_mean != 0) & ~np.isnan(clear_mean) clear_windows = c1 & c2 & c3 & c4 & c5 & c6 @@ -761,6 +767,13 @@ def rmse(alpha): components['slope_max'] = c5 components['mean_nan'] = c6 components['windows'] = clear_windows + + components['mean_diff_array'] = np.abs(meas_mean - alpha*clear_mean) + components['max_diff_array'] = np.abs(meas_max - alpha*clear_max) + components['line_length_array'] = meas_line_length - clear_line_length + components['slope_nstd_array'] = meas_slope_nstd + components['slope_max_array'] = (np.max(meas_slope - alpha*clear_slope, axis=0)) + return clear_samples, components, alpha else: return clear_samples From 3f9dc9d17de244fe74eb015f0e0cbe2b688a3685 Mon Sep 17 00:00:00 2001 From: bhellis725 Date: Thu, 26 Jul 2018 14:10:23 -0700 Subject: [PATCH 2/8] Minor cleanup. --- pvlib/clearsky.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index 94fc43cf5b..8c57214800 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -688,9 +688,6 @@ def detect_clearsky(measured, clearsky, times, window_length, 'times. consider resampling your data.') samples_per_window = int(window_length / sample_interval) + 1 - # print(window_length) - # print(sample_interval) - # print(samples_per_window) # generate matrix of integers for creating windows with indexing from scipy.linalg import hankel @@ -730,8 +727,7 @@ def detect_clearsky(measured, clearsky, times, window_length, c2 = np.abs(meas_max - alpha*clear_max) < max_diff c3 = (line_diff > lower_line_length) & (line_diff < upper_line_length) c4 = meas_slope_nstd < var_diff - # c5 = (meas_slope_max - alpha*clear_slope_max) < slope_dev - c5 = np.max(meas_slope - alpha*clear_slope, axis=0) < slope_dev + c5 = np.max(np.abs(meas_slope - alpha*clear_slope), axis=0) < slope_dev c6 = (clear_mean != 0) & ~np.isnan(clear_mean) clear_windows = c1 & c2 & c3 & c4 & c5 & c6 From 389720a75d1fe9e2c5630add53d0ac43ddbf04c4 Mon Sep 17 00:00:00 2001 From: Cliff Hansen Date: Wed, 3 Oct 2018 11:03:53 -0600 Subject: [PATCH 3/8] Revert samples_per_interval change, rename to clarify --- pvlib/clearsky.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index b8a183b02d..2676901fd7 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -680,12 +680,12 @@ def detect_clearsky(measured, clearsky, times, window_length, raise NotImplementedError('algorithm does not yet support unequal ' 'times. consider resampling your data.') - samples_per_window = int(window_length / sample_interval) + 1 + intervals_per_window = int(window_length / sample_interval) # generate matrix of integers for creating windows with indexing from scipy.linalg import hankel - H = hankel(np.arange(samples_per_window), # noqa: N806 - np.arange(samples_per_window-1, len(times))) + H = hankel(np.arange(intervals_per_window), # noqa: N806 + np.arange(intervals_per_window-1, len(times))) # calculate measurement statistics meas_mean = np.mean(measured[H], axis=0) From de84c3fd6e34693817e51854198e2902a0b88fb8 Mon Sep 17 00:00:00 2001 From: Cliff Hansen Date: Wed, 3 Oct 2018 11:14:07 -0600 Subject: [PATCH 4/8] Style fixes --- pvlib/clearsky.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index 2676901fd7..5666c533f2 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -685,7 +685,7 @@ def detect_clearsky(measured, clearsky, times, window_length, # generate matrix of integers for creating windows with indexing from scipy.linalg import hankel H = hankel(np.arange(intervals_per_window), # noqa: N806 - np.arange(intervals_per_window-1, len(times))) + np.arange(intervals_per_window - 1, len(times))) # calculate measurement statistics meas_mean = np.mean(measured[H], axis=0) @@ -696,7 +696,8 @@ def detect_clearsky(measured, clearsky, times, window_length, meas_slope_nstd = np.std(meas_slope, axis=0, ddof=1) / meas_mean # meas_slope_max = np.max(np.abs(meas_slope), axis=0) meas_line_length = np.sum(np.sqrt( - meas_ghi_diff*meas_ghi_diff + sample_interval*sample_interval), axis=0) + meas_ghi_diff * meas_ghi_diff + + sample_interval * sample_interval), axis=0) # calculate clear sky statistics clear_mean = np.mean(clearsky[H], axis=0) @@ -710,8 +711,8 @@ def detect_clearsky(measured, clearsky, times, window_length, alpha = 1 for iteration in range(max_iterations): clear_line_length = np.sum(np.sqrt( - alpha*alpha*clear_ghi_diff*clear_ghi_diff + - sample_interval*sample_interval), axis=0) + alpha * alpha * clear_ghi_diff * clear_ghi_diff + + sample_interval * sample_interval), axis=0) line_diff = meas_line_length - clear_line_length @@ -720,7 +721,8 @@ def detect_clearsky(measured, clearsky, times, window_length, c2 = np.abs(meas_max - alpha*clear_max) < max_diff c3 = (line_diff > lower_line_length) & (line_diff < upper_line_length) c4 = meas_slope_nstd < var_diff - c5 = np.max(np.abs(meas_slope - alpha*clear_slope), axis=0) < slope_dev + c5 = np.max(np.abs(meas_slope - + alpha * clear_slope), axis=0) < slope_dev c6 = (clear_mean != 0) & ~np.isnan(clear_mean) clear_windows = c1 & c2 & c3 & c4 & c5 & c6 @@ -759,11 +761,12 @@ def rmse(alpha): components['mean_nan'] = c6 components['windows'] = clear_windows - components['mean_diff_array'] = np.abs(meas_mean - alpha*clear_mean) - components['max_diff_array'] = np.abs(meas_max - alpha*clear_max) + components['mean_diff_array'] = np.abs(meas_mean - alpha * clear_mean) + components['max_diff_array'] = np.abs(meas_max - alpha * clear_max) components['line_length_array'] = meas_line_length - clear_line_length components['slope_nstd_array'] = meas_slope_nstd - components['slope_max_array'] = (np.max(meas_slope - alpha*clear_slope, axis=0)) + components['slope_max_array'] = (np.max( + meas_slope - alpha * clear_slope, axis=0)) return clear_samples, components, alpha else: From a8bd655d0a37a25ed05bb5b8d898fb309d653195 Mon Sep 17 00:00:00 2001 From: Cliff Hansen Date: Wed, 3 Oct 2018 11:18:01 -0600 Subject: [PATCH 5/8] Style fixes --- pvlib/clearsky.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index 5666c533f2..6d3d1c18e8 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -696,7 +696,7 @@ def detect_clearsky(measured, clearsky, times, window_length, meas_slope_nstd = np.std(meas_slope, axis=0, ddof=1) / meas_mean # meas_slope_max = np.max(np.abs(meas_slope), axis=0) meas_line_length = np.sum(np.sqrt( - meas_ghi_diff * meas_ghi_diff + + meas_ghi_diff * meas_ghi_diff + sample_interval * sample_interval), axis=0) # calculate clear sky statistics @@ -721,7 +721,7 @@ def detect_clearsky(measured, clearsky, times, window_length, c2 = np.abs(meas_max - alpha*clear_max) < max_diff c3 = (line_diff > lower_line_length) & (line_diff < upper_line_length) c4 = meas_slope_nstd < var_diff - c5 = np.max(np.abs(meas_slope - + c5 = np.max(np.abs(meas_slope - alpha * clear_slope), axis=0) < slope_dev c6 = (clear_mean != 0) & ~np.isnan(clear_mean) clear_windows = c1 & c2 & c3 & c4 & c5 & c6 @@ -766,7 +766,7 @@ def rmse(alpha): components['line_length_array'] = meas_line_length - clear_line_length components['slope_nstd_array'] = meas_slope_nstd components['slope_max_array'] = (np.max( - meas_slope - alpha * clear_slope, axis=0)) + meas_slope - alpha * clear_slope, axis=0)) return clear_samples, components, alpha else: From 1614971ce13dd2195445df59b62b8ea43c5fe723 Mon Sep 17 00:00:00 2001 From: Cliff Hansen Date: Wed, 3 Oct 2018 13:58:05 -0600 Subject: [PATCH 6/8] whatsnew --- docs/sphinx/source/whatsnew/v0.6.1.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/sphinx/source/whatsnew/v0.6.1.rst b/docs/sphinx/source/whatsnew/v0.6.1.rst index 37f8ee312a..bebef39f30 100644 --- a/docs/sphinx/source/whatsnew/v0.6.1.rst +++ b/docs/sphinx/source/whatsnew/v0.6.1.rst @@ -21,6 +21,7 @@ Enhancements Bug fixes ~~~~~~~~~ * Fix when building documentation using Matplotlib 3.0 or greater. +* Fix error in :func:`pvlib.clearsky.detect_clearsky` (:issue:`506`) Testing @@ -29,4 +30,6 @@ Testing Contributors ~~~~~~~~~~~~ -* Will Holmgren (:ghuser:`wholmgren`) \ No newline at end of file +* Will Holmgren (:ghuser:`wholmgren`) +* Ben Ellis (:ghuser:`bhellis725`) +* Cliff Hansen (:ghuser:`cwhanse`) From bde27b739a89a55c2924a0d83c7cbee1611119c6 Mon Sep 17 00:00:00 2001 From: Cliff Hansen Date: Tue, 9 Oct 2018 10:37:00 -0600 Subject: [PATCH 7/8] Variable name edits, delete extinct commented code lines --- pvlib/clearsky.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index 6d3d1c18e8..22248ebd1f 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -690,28 +690,26 @@ def detect_clearsky(measured, clearsky, times, window_length, # calculate measurement statistics meas_mean = np.mean(measured[H], axis=0) meas_max = np.max(measured[H], axis=0) - meas_ghi_diff = np.diff(measured[H], n=1, axis=0) + meas_diff = np.diff(measured[H], n=1, axis=0) meas_slope = np.diff(measured[H], n=1, axis=0) / sample_interval # matlab std function normalizes by N-1, so set ddof=1 here meas_slope_nstd = np.std(meas_slope, axis=0, ddof=1) / meas_mean - # meas_slope_max = np.max(np.abs(meas_slope), axis=0) meas_line_length = np.sum(np.sqrt( - meas_ghi_diff * meas_ghi_diff + + meas_diff * meas_diff + sample_interval * sample_interval), axis=0) # calculate clear sky statistics clear_mean = np.mean(clearsky[H], axis=0) clear_max = np.max(clearsky[H], axis=0) - clear_ghi_diff = np.diff(clearsky[H], n=1, axis=0) + clear_diff = np.diff(clearsky[H], n=1, axis=0) clear_slope = np.diff(clearsky[H], n=1, axis=0) / sample_interval - # clear_slope_max = np.max(np.abs(clear_slope), axis=0) from scipy.optimize import minimize_scalar alpha = 1 for iteration in range(max_iterations): clear_line_length = np.sum(np.sqrt( - alpha * alpha * clear_ghi_diff * clear_ghi_diff + + alpha * alpha * clear_diff * clear_diff + sample_interval * sample_interval), axis=0) line_diff = meas_line_length - clear_line_length @@ -753,19 +751,19 @@ def rmse(alpha): if return_components: components = OrderedDict() - components['mean_diff'] = c1 - components['max_diff'] = c2 - components['line_length'] = c3 - components['slope_nstd'] = c4 - components['slope_max'] = c5 - components['mean_nan'] = c6 + components['mean_diff_flag'] = c1 + components['max_diff_flag'] = c2 + components['line_length_flag'] = c3 + components['slope_nstd_flag'] = c4 + components['slope_max_flag'] = c5 + components['mean_nan_flag'] = c6 components['windows'] = clear_windows - components['mean_diff_array'] = np.abs(meas_mean - alpha * clear_mean) - components['max_diff_array'] = np.abs(meas_max - alpha * clear_max) - components['line_length_array'] = meas_line_length - clear_line_length - components['slope_nstd_array'] = meas_slope_nstd - components['slope_max_array'] = (np.max( + components['mean_diff'] = np.abs(meas_mean - alpha * clear_mean) + components['max_diff'] = np.abs(meas_max - alpha * clear_max) + components['line_length'] = meas_line_length - clear_line_length + components['slope_nstd'] = meas_slope_nstd + components['slope_max'] = (np.max( meas_slope - alpha * clear_slope, axis=0)) return clear_samples, components, alpha From 252cf7b6f0da4433fc6e1e0568bfc5c23ee9434d Mon Sep 17 00:00:00 2001 From: Cliff Hansen Date: Tue, 9 Oct 2018 15:10:19 -0600 Subject: [PATCH 8/8] Document change to API of detect_clearsky --- docs/sphinx/source/whatsnew/v0.6.1.rst | 1 + pvlib/clearsky.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/sphinx/source/whatsnew/v0.6.1.rst b/docs/sphinx/source/whatsnew/v0.6.1.rst index 2e47d15e63..dcd4a52ed4 100644 --- a/docs/sphinx/source/whatsnew/v0.6.1.rst +++ b/docs/sphinx/source/whatsnew/v0.6.1.rst @@ -16,6 +16,7 @@ API Changes * Deprecated ``tmy``, ``tmy.readtmy2`` and ``tmy.readtmy3``; they will be removed in v0.7. Use the new :py:func:`pvlib.iotools.read_tmy2` and :py:func:`pvlib.iotools.read_tmy3` instead. (:issue:`261`) +* Changed key names for `components` returned from :py:func:`pvlib.clearsky.detect_clearsky` Enhancements diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index 22248ebd1f..be8ca8f46c 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -764,7 +764,7 @@ def rmse(alpha): components['line_length'] = meas_line_length - clear_line_length components['slope_nstd'] = meas_slope_nstd components['slope_max'] = (np.max( - meas_slope - alpha * clear_slope, axis=0)) + meas_slope - alpha * clear_slope, axis=0)) return clear_samples, components, alpha else: