Support legacy _raise and get_ordered use cases

[scjody]

plotly.py raises a ValueError when a blank color property is found in the legend font (and possibly other places), rather than a PlotlyGraphObjectError. We depend on PlotlyGraphObjectError being raised in the streambed backend so that we can retry with _raise=False if the user tries to operate on an invalid figure.

Minimal code to reproduce the issue (Python 2.7.6, plotly.py 3.2.0):

import plotly.graph_objs as go
go.Figure(data=[], layout={'legend': {'font': {'color': u''}}})

Expected results: PlotlyGraphObjectError is raised.

Actual results:

ValueError                                Traceback (most recent call last)
<ipython-input-4-ff88e2cee070> in <module>()
----> 1 go.Figure(data=[], layout={'legend': {'font': {'color': u''}}})

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/graph_objs/_figure.pyc in __init__(self, data, layout, frames)
    334                         respective traces in the data attribute
    335         """
--> 336         super(Figure, self).__init__(data, layout, frames)
    337
    338     def add_area(

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/basedatatypes.pyc in __init__(self, data, layout_plotly, frames)
    155
    156         # ### Import Layout ###
--> 157         self._layout_obj = self._layout_validator.validate_coerce(layout)
    158
    159         # ### Import clone of layout properties ###

/tmp/ppytmp/venv/lib/python2.7/site-packages/_plotly_utils/basevalidators.pyc in validate_coerce(self, v)
   1875
   1876         elif isinstance(v, dict):
-> 1877             v = self.data_class(**v)
   1878
   1879         elif isinstance(v, self.data_class):

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/graph_objs/_layout.pyc in __init__(self, arg, angularaxis, annotations, autosize, bargap, bargroupgap, barmode, barnorm, boxgap, boxgroupgap, boxmode, calendar, colorway, datarevision, direction, dragmode, extendpiecolors, font, geo, grid, height, hiddenlabels, hiddenlabelssrc, hidesources, hoverdistance, hoverlabel, hovermode, images, legend, mapbox, margin, orientation, paper_bgcolor, piecolorway, plot_bgcolor, polar, radialaxis, scene, selectdirection, separators, shapes, showlegend, sliders, spikedistance, template, ternary, title, titlefont, updatemenus, violingap, violingroupgap, violinmode, width, xaxis, yaxis, **kwargs)
   3832         self.images = images if images is not None else _v
   3833         _v = arg.pop('legend', None)
-> 3834         self.legend = legend if legend is not None else _v
   3835         _v = arg.pop('mapbox', None)
   3836         self.mapbox = mapbox if mapbox is not None else _v

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/basedatatypes.pyc in __setattr__(self, prop, value)
   3601         if match is None:
   3602             # Set as ordinary property
-> 3603             super(BaseLayoutHierarchyType, self).__setattr__(prop, value)
   3604         else:
   3605             # Set as subplotid property

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/basedatatypes.pyc in __setattr__(self, prop, value)
   2702                 prop in self._validators):
   2703             # Let known properties and private properties through
-> 2704             super(BasePlotlyType, self).__setattr__(prop, value)
   2705         else:
   2706             # Raise error on unknown public properties

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/graph_objs/_layout.pyc in legend(self, val)
   1250     @legend.setter
   1251     def legend(self, val):
-> 1252         self['legend'] = val
   1253
   1254     # mapbox

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/basedatatypes.pyc in __setitem__(self, prop, value)
   3587         if match is None:
   3588             # Set as ordinary property
-> 3589             super(BaseLayoutHierarchyType, self).__setitem__(prop, value)
   3590         else:
   3591             # Set as subplotid property

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/basedatatypes.pyc in __setitem__(self, prop, value)
   2665             # ### Handle compound property ###
   2666             if isinstance(validator, CompoundValidator):
-> 2667                 self._set_compound_prop(prop, value)
   2668
   2669             # ### Handle compound array property ###

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/basedatatypes.pyc in _set_compound_prop(self, prop, val)
   2963         validator = self._validators.get(prop)
   2964         # type: BasePlotlyType
-> 2965         val = validator.validate_coerce(val)
   2966
   2967         # Save deep copies of current and new states

/tmp/ppytmp/venv/lib/python2.7/site-packages/_plotly_utils/basevalidators.pyc in validate_coerce(self, v)
   1875
   1876         elif isinstance(v, dict):
-> 1877             v = self.data_class(**v)
   1878
   1879         elif isinstance(v, self.data_class):

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/graph_objs/layout/_legend.pyc in __init__(self, arg, bgcolor, bordercolor, borderwidth, font, orientation, tracegroupgap, traceorder, x, xanchor, y, yanchor, **kwargs)
    506         self.borderwidth = borderwidth if borderwidth is not None else _v
    507         _v = arg.pop('font', None)
--> 508         self.font = font if font is not None else _v
    509         _v = arg.pop('orientation', None)
    510         self.orientation = orientation if orientation is not None else _v

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/basedatatypes.pyc in __setattr__(self, prop, value)
   2702                 prop in self._validators):
   2703             # Let known properties and private properties through
-> 2704             super(BasePlotlyType, self).__setattr__(prop, value)
   2705         else:
   2706             # Raise error on unknown public properties

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/graph_objs/layout/_legend.pyc in font(self, val)
    186     @font.setter
    187     def font(self, val):
--> 188         self['font'] = val
    189
    190     # orientation

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/basedatatypes.pyc in __setitem__(self, prop, value)
   2665             # ### Handle compound property ###
   2666             if isinstance(validator, CompoundValidator):
-> 2667                 self._set_compound_prop(prop, value)
   2668
   2669             # ### Handle compound array property ###

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/basedatatypes.pyc in _set_compound_prop(self, prop, val)
   2963         validator = self._validators.get(prop)
   2964         # type: BasePlotlyType
-> 2965         val = validator.validate_coerce(val)
   2966
   2967         # Save deep copies of current and new states

/tmp/ppytmp/venv/lib/python2.7/site-packages/_plotly_utils/basevalidators.pyc in validate_coerce(self, v)
   1875
   1876         elif isinstance(v, dict):
-> 1877             v = self.data_class(**v)
   1878
   1879         elif isinstance(v, self.data_class):

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/graph_objs/layout/legend/_font.pyc in __init__(self, arg, color, family, size, **kwargs)
    207         # ----------------------------------
    208         _v = arg.pop('color', None)
--> 209         self.color = color if color is not None else _v
    210         _v = arg.pop('family', None)
    211         self.family = family if family is not None else _v

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/basedatatypes.pyc in __setattr__(self, prop, value)
   2702                 prop in self._validators):
   2703             # Let known properties and private properties through
-> 2704             super(BasePlotlyType, self).__setattr__(prop, value)
   2705         else:
   2706             # Raise error on unknown public properties

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/graph_objs/layout/legend/_font.pyc in color(self, val)
     60     @color.setter
     61     def color(self, val):
---> 62         self['color'] = val
     63
     64     # family

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/basedatatypes.pyc in __setitem__(self, prop, value)
   2674             # ### Handle simple property ###
   2675             else:
-> 2676                 self._set_prop(prop, value)
   2677
   2678         # Handle non-scalar case

/tmp/ppytmp/venv/lib/python2.7/site-packages/plotly/basedatatypes.pyc in _set_prop(self, prop, val)
   2904         # ------------
   2905         validator = self._validators.get(prop)
-> 2906         val = validator.validate_coerce(val)
   2907
   2908         # val is None

/tmp/ppytmp/venv/lib/python2.7/site-packages/_plotly_utils/basevalidators.pyc in validate_coerce(self, v, should_raise)
   1077             validated_v = self.vc_scalar(v)
   1078             if validated_v is None and should_raise:
-> 1079                 self.raise_invalid_val(v)
   1080
   1081             v = validated_v

/tmp/ppytmp/venv/lib/python2.7/site-packages/_plotly_utils/basevalidators.pyc in raise_invalid_val(self, v)
    223             typ=type_str(v),
    224             v=repr(v),
--> 225             valid_clr_desc=self.description()))
    226
    227     def raise_invalid_elements(self, invalid_els):

ValueError:
    Invalid value of type '__builtin__.unicode' received for the 'color' property of layout.legend.font
        Received value: u''

    The 'color' property is a color and may be specified as:
      - A hex string (e.g. '#ff0000')
      - An rgb/rgba string (e.g. 'rgb(255,0,0)')
      - An hsl/hsla string (e.g. 'hsl(0,100%,50%)')
      - An hsv/hsva string (e.g. 'hsv(0,100%,100%)')
      - A named CSS color:
            aliceblue, antiquewhite, aqua, aquamarine, azure,
            beige, bisque, black, blanchedalmond, blue,
            blueviolet, brown, burlywood, cadetblue,
            chartreuse, chocolate, coral, cornflowerblue,
            cornsilk, crimson, cyan, darkblue, darkcyan,
            darkgoldenrod, darkgray, darkgrey, darkgreen,
            darkkhaki, darkmagenta, darkolivegreen, darkorange,
            darkorchid, darkred, darksalmon, darkseagreen,
            darkslateblue, darkslategray, darkslategrey,
            darkturquoise, darkviolet, deeppink, deepskyblue,
            dimgray, dimgrey, dodgerblue, firebrick,
            floralwhite, forestgreen, fuchsia, gainsboro,
            ghostwhite, gold, goldenrod, gray, grey, green,
            greenyellow, honeydew, hotpink, indianred, indigo,
            ivory, khaki, lavender, lavenderblush, lawngreen,
            lemonchiffon, lightblue, lightcoral, lightcyan,
            lightgoldenrodyellow, lightgray, lightgrey,
            lightgreen, lightpink, lightsalmon, lightseagreen,
            lightskyblue, lightslategray, lightslategrey,
            lightsteelblue, lightyellow, lime, limegreen,
            linen, magenta, maroon, mediumaquamarine,
            mediumblue, mediumorchid, mediumpurple,
            mediumseagreen, mediumslateblue, mediumspringgreen,
            mediumturquoise, mediumvioletred, midnightblue,
            mintcream, mistyrose, moccasin, navajowhite, navy,
            oldlace, olive, olivedrab, orange, orangered,
            orchid, palegoldenrod, palegreen, paleturquoise,
            palevioletred, papayawhip, peachpuff, peru, pink,
            plum, powderblue, purple, red, rosybrown,
            royalblue, saddlebrown, salmon, sandybrown,
            seagreen, seashell, sienna, silver, skyblue,
            slateblue, slategray, slategrey, snow, springgreen,
            steelblue, tan, teal, thistle, tomato, turquoise,
            violet, wheat, white, whitesmoke, yellow,
            yellowgreen

@jonmmease FYI

[jonmmease]

@scjody We can definitely bring back PlotlyGraphObjectError for validation exceptions, but it sounds like the bigger issue might be that in 3.0.0+ there is no more _raise arg in the graph_objs constructors. Properties are always validated against the schema of the version of Plotly.js that ships with that version of plotly.py.

That said, every interface between plotly.py and Plotly.js (plotly.plotly.plot, plotly.plotly.iplot, plotly.offline.plot, etc) should accept a dict in place of a Figure and should accept a validate=False argument to bypass validation. What operation are you retrying if PlotlyGraphObjectError is raised?

[scjody]

@jonmmease It sounds like the missing _raise argument will be a problem. Would it be possible to bring it back?

What we're doing is roughly:

try:
    fig = go.Figure(json_fig)
except plotly.exceptions.PlotlyGraphObjectError as e:
    fig = go.Figure(json_fig, _raise=False)

We then call force_clean() and get_ordered() on the figure before passing it to our own code. (fig_to_lang()).

If you have access to the streambed repo you can see the whole thing here: https://github.com/plotly/streambed/blob/570fe40e83c393e3f751f142651e341f6d0f4e1c/shelly/code_translator/translator.py#L40-L52 . This is part of the code translator, which provides code to create plotly.js figures in various languages. This is an important Chart Studio feature and we need to support existing plots in our system, many of which are full of errors; how hard would it be to bring back the plotly.py capabilities we're using?

[jonmmease]

We can definitely add something back to handle this case, but I don't think it can exactly match the _raise=False behavior.

In v3 a Figure can never store/represent an invalid figure state (this is an assumption relied upon throughout the code) so we can't just disable the exceptions. Also, in v3 we don't actually store None values, so I don't think there would be a need for force_clean (which doens't currently exist).

We could add a _raise or _strip constructor argument that would ignore all properties that are not in the schema and all properties with values that don't conform to the schema. For the streambed scenario above, this argument would just always be set to True (no need for the try except block or the force_clean call).

It looks like get_ordered returned an OrderedDict not a graph object, so we should be able to bring this back without much issue. Though I would like to rename it Figure.to_ordered_dict to conform with the new conventions described in #1098.

How does that sound?

[jonmmease]

Implementation outline

In _plotly_utils/basevalidators.py, add a skip_invalid=False argument to the validate_coerce method of CompoundValidator, CompoundArrayValidator, and BaseDataValidator. This argument gets passed along to the constructor of the graph object that each validator constructs.

Add a skip_invalid=False argument to the constructor of each graph_objs class, and a _skip_invalid property to each class. The start of the constructor sets self._skip_invalid=skip_invalid and the end of the constructor sets self._skip_invalid=False

The BasePlotlyType._set_prop method traps exceptions raised by validator.validate_coerce(val). If self._skip_invalid=False the exception is re-raised. If self._skip_invalid=True, the exception is discarded and the _set_prop method returns, making no changes to the object.

The _set_compound_prop and _set_array_prop methods of BasePlotlyType pass self._skip_invalid to their validator.validate_coerce call.

val = validator.validate_coerce(val, skip_invalid=self._skip_invalid)

BasePlotlyType._process_kwargs should only raise if self._skip_invalid=False

Other potential uses

Context manager

Add on, a skip_invalid context manager:

with fig.skip_invalid():
    # No error raised inside context manager
    fig.data[0].bogus = 23

Read from file

This skip_invalid option would also be useful when loading a figure from a json file / pickle file (See #1098).

import plotly.io as pio
fig = pio.read_json('/path/to/fig.json', skip_invalid=True)

[scjody]

@jonmmease Your proposed changes sound good to me - it sounds like the streambed code will be more straightforward with these changes even. I don't know enough about plotly.py internals to comment on the design details though.

[jonmmease]

Great, I'll add these changes soon and I'll ping you on the PR. It would be great if someone from your side could try out these changes in streambed (probably early next week) and see if there's anything else you all need from plotly.py before upgrading.

[jonmmease]

@scjody I just merged #1162 and #1167. These will be released as part of 3.2.1. With these updates, the use-case of constructing a Figure from a dict that may include some invalid properties works like this:

fig = go.Figure(json_fig, skip_invalid=True)

Then, to convert a Figure into a nested structure of OrderedDict and list instances, use the to_ordered_dict method.

ordered_fig = fig.to_ordered_dict()

Would someone from your team have time to test out a release candidate early next week? Due to a couple of other bug fixes, I'd like to publish the final 3.2.1 by mid-week if possible. Thanks!

[jonmmease]

@scjody I just published 3.2.1rc1 to PyPI. If you all have time, give it a try and let me know if there's anything else that streambed needs.

[scjody]

@jonmmease Thanks for the changes! I tested plotly.py master in streambed and most code translation cases work, but unfortunately the JSON case is broken. With the new code, https://github.com/plotly/streambed/blob/master/shelly/code_translator/translator.py#L67 fails as follows:

Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/django/contrib/staticfiles/handlers.py", line 63, in __call__
    return self.application(environ, start_response)
  File "/usr/local/lib/python2.7/dist-packages/django/core/handlers/wsgi.py", line 157, in __call__
    response = self.get_response(request)
  File "/usr/local/lib/python2.7/dist-packages/django/core/handlers/base.py", line 124, in get_response
    response = self._middleware_chain(request)
  File "/usr/local/lib/python2.7/dist-packages/django/core/handlers/exception.py", line 43, in inner
    response = response_for_exception(request, exc)
  File "/usr/local/lib/python2.7/dist-packages/django/core/handlers/exception.py", line 93, in response_for_exception
    response = handle_uncaught_exception(request, get_resolver(get_urlconf()), sys.exc_info())
  File "/usr/local/lib/python2.7/dist-packages/django/core/handlers/exception.py", line 139, in handle_uncaught_exception
    return debug.technical_500_response(request, *exc_info)
  File "/usr/local/lib/python2.7/dist-packages/django_extensions/management/technical_response.py", line 6, in null_technical_500_response
    six.reraise(exc_type, exc_value, tb)
  File "/usr/local/lib/python2.7/dist-packages/django/core/handlers/exception.py", line 41, in inner
    response = get_response(request)
  File "/usr/local/lib/python2.7/dist-packages/django/core/handlers/base.py", line 249, in _legacy_get_response
    response = self._get_response(request)
  File "/usr/local/lib/python2.7/dist-packages/django/core/handlers/base.py", line 187, in _get_response
    response = self.process_exception_by_middleware(e, request)
  File "/usr/local/lib/python2.7/dist-packages/django/core/handlers/base.py", line 185, in _get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "/usr/local/lib/python2.7/dist-packages/django/views/generic/base.py", line 68, in view
    return self.dispatch(request, *args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/django/utils/decorators.py", line 67, in _wrapper
    return bound_func(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/django/views/decorators/clickjacking.py", line 58, in wrapped_view
    resp = view_func(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/django/utils/decorators.py", line 63, in bound_func
    return func.__get__(self, type(self))(*args2, **kwargs2)
  File "/var/www/streambed/shelly/embedplot/views.py", line 324, in dispatch
    return super(BaseFile, self).dispatch(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/django/views/generic/base.py", line 88, in dispatch
    return handler(request, *args, **kwargs)
  File "/var/www/streambed/shelly/embedplot/views.py", line 224, in get
    return self.render_to_response(fmeta, data)
  File "/var/www/streambed/shelly/embedplot/views.py", line 593, in render_to_response
    code_string, status_code = self.convert_plot(fmeta, data)
  File "/var/www/streambed/shelly/embedplot/views.py", line 567, in convert_plot
    plot_instance=plot)
  File "/var/www/streambed/shelly/code_translator/translator.py", line 55, in translate
    return json.dumps(fig, indent=4, ensure_ascii=False)
  File "/usr/lib/python2.7/json/__init__.py", line 250, in dumps
    sort_keys=sort_keys, **kw).encode(obj)
  File "/usr/lib/python2.7/json/encoder.py", line 209, in encode
    chunks = list(chunks)
  File "/usr/lib/python2.7/json/encoder.py", line 442, in _iterencode
    o = _default(o)
  File "/usr/lib/python2.7/json/encoder.py", line 184, in default
    raise TypeError(repr(o) + " is not JSON serializable")
TypeError: Figure({
    'data': [{'name': 'Col2',
              'type': 'scatter',
              'uid': '0581c7c6-b69f-11e8-a899-080027c729c7',
              'x': [1, 2, 3, 4, 5],
              'xsrc': '._-aA1:7:f6f72f',
              'y': [5, 4, 3, 2, 1],
              'ysrc': '._-aA1:7:f42b8d'}],
    'layout': {'autosize': True,
               'bargap': 0.2,
               'bargroupgap': 0,
               'barmode': 'group',
               'boxgap': 0.3,
               'boxgroupgap': 0.3,
               'boxmode': 'overlay',
               'dragmode': 'zoom',
               'font': {'color': '#444', 'family': '"Open sans", verdana, arial, sans-serif', 'size': 12},
               'height': 392,
               'hidesources': False,
               'hovermode': 'x',
               'legend': {'bgcolor': '#fff',
                          'bordercolor': '#444',
                          'borderwidth': 0,
                          'traceorder': 'normal',
                          'x': 1.02,
                          'xanchor': 'left',
                          'y': 1,
                          'yanchor': 'top'},
               'margin': {'autoexpand': True, 'b': 80, 'l': 80, 'pad': 0, 'r': 80, 't': 100},
               'paper_bgcolor': '#fff',
               'plot_bgcolor': '#fff',
               'separators': '.,',
               'showlegend': True,
               'title': 'Click to enter Plot title',
               'width': 1112,
               'xaxis': {'anchor': 'y',
                         'autorange': True,
                         'domain': [0, 1],
                         'dtick': 0.5,
                         'exponentformat': 'B',
                         'gridcolor': '#eee',
                         'gridwidth': 1,
                         'linecolor': '#444',
                         'linewidth': 1,
                         'mirror': False,
                         'nticks': 0,
                         'position': 0,
                         'range': [0.777777777778, 5.22222222222],
                         'rangemode': 'normal',
                         'showexponent': 'all',
                         'showgrid': True,
                         'showline': False,
                         'showticklabels': True,
                         'tick0': 0,
                         'tickcolor': '#444',
                         'ticklen': 5,
                         'ticks': '',
                         'tickwidth': 1,
                         'title': 'Opa X axis',
                         'type': 'linear',
                         'zeroline': True,
                         'zerolinecolor': '#444',
                         'zerolinewidth': 1},
               'yaxis': {'anchor': 'x',
                         'autorange': True,
                         'domain': [0, 1],
                         'dtick': 1,
                         'exponentformat': 'B',
                         'gridcolor': '#eee',
                         'gridwidth': 1,
                         'linecolor': '#444',
                         'linewidth': 1,
                         'mirror': False,
                         'nticks': 0,
                         'position': 0,
                         'range': [0.777777777778, 5.22222222222],
                         'rangemode': 'normal',
                         'showexponent': 'all',
                         'showgrid': True,
                         'showline': False,
                         'showticklabels': True,
                         'tick0': 0,
                         'tickcolor': '#444',
                         'ticklen': 5,
                         'ticks': '',
                         'tickwidth': 1,
                         'title': 'Click to enter Y axis title',
                         'type': 'linear',
                         'zeroline': True,
                         'zerolinecolor': '#444',
                         'zerolinewidth': 1}}
}) is not JSON serializable

Would it be possible to return something JSON-serializable from to_ordered_dict()? Or is there a way we can get the JSON given a Figure()? I'm not sure what the old code (plotly.py 2.6.0) returned but we were able to serialize it.