Adding Promise support for DjangoConnectionField

[arianon]

Now that Graphene 1.4 and Promise 2.0 are out, I am now using them on my Django application, particularly promise.DataLoader, and I came across the fact that the DjangoConnectionField.connection_resolver method is not compatible with Promises, but fortunately, solving that is trivial, thanks to some tactical copypasting:

from functools import partial
from promise import Promise
from django.db.models import QuerySet, Manager
from graphene_django.fields import DjangoConnectionField
from graphene.relay import PageInfo
from graphql_relay.connection.arrayconnection import connection_from_list_slice


class PromiseDjangoConnectionField(DjangoConnectionField):
    @classmethod
    def connection_from_iterable(cls, args, connection, default_manager, iterable):
        if iterable is None:
            iterable = default_manager

        if isinstance(iterable, Manager):
            iterable = iterable.get_queryset()

        if isinstance(iterable, QuerySet):
            if iterable is not default_manager:
                default_queryset = default_manager.get_queryset()
                iterable = default_queryset & iterable
            _len = iterable.count()
        else:
            _len = len(iterable)

        connection = connection_from_list_slice(
            iterable,
            args,
            slice_start=0,
            list_length=_len,
            list_slice_length=_len,
            connection_type=connection,
            edge_type=connection.Edge,
            pageinfo_type=PageInfo,
        )

        connection.iterable = iterable
        connection.length = _len
        return connection

    @classmethod
    def connection_resolver(cls, resolver, connection, default_manager, max_limit,
                            enforce_first_or_last, root, args, context, info):
        first = args.get('first')
        last = args.get('last')

        if enforce_first_or_last:
            assert first or last, (
                'You must provide a `first` or `last` value to properly paginate the `{}` connection.'
            ).format(info.field_name)

        if max_limit:
            if first:
                assert first <= max_limit, (
                    'Requesting {} records on the `{}` connection exceeds the `first` limit of {} records.'
                ).format(first, info.field_name, max_limit)
                args['first'] = min(first, max_limit)

            if last:
                assert last <= max_limit, (
                    'Requesting {} records on the `{}` connection exceeds the `last` limit of {} records.'
                ).format(first, info.field_name, max_limit)
                args['last'] = min(last, max_limit)
    
        iterable = resolver(root, args, context, info)
        connection_from_iterable = partial(
            cls.connection_from_iterable,
            args,
            connection,
            default_manager
        )

        if Promise.is_thenable(iterable):
            return iterable.then(connection_from_iterable)
        else:
            return connection_from_iterable(iterable)

If @syrusakbary gives me the green light, I'll write a PR with a few tests. 😄

[japrogramer]

Hello, this might be the wrong place to ask, how are you passing your dataloader to django-graphene?
I was thinking of overiding the get_context view method and adding my dataloaders there but I have the lingering feeling that there is a simpler way that is better.

[arianon]

@japrogramer I simply use a Graphene middleware
middlewares.py

from .loaders import Loaders

class LoaderMiddleware:
    def resolve(self, next, root, args, context, info):
        if not hasattr(context, 'loaders'):
            context.loaders = Loaders()
        return next(root, args, context, info)

settings.py

GRAPHENE['MIDDLEWARE'] = [
    'middlewares.LoaderMiddleware'
]

And I access the loaders like so

class Thing(DjangoObjectType):
    class Meta:
        model = ThingModel
    
    def resolve_XXX(self, args, context, info):
        return context.loaders.XXX_loader.load(self.XXX_id)

Note that the API won't be the same in 2.0, though.

[cmmartti]

This should be in the docs somewhere.

For anyone who comes across this, here's how you do it in 2.0:

middleware.py

from .loaders import Loaders

class LoaderMiddleware:	
	def resolve(self, next, root, info, **args):
		if not hasattr(info.context, 'loaders'):
			info.context.loaders = Loaders()
		return next(root, info, **args)

settings.py

GRAPHENE = {
    'MIDDLEWARE': [
        'middlewares.LoaderMiddleware'
    ]
}

Access the loaders in the resolvers using info.context:

class Thing(DjangoObjectType):
    class Meta:
        model = ThingModel
    
    def resolve_XXX(self, info):
        return info.context.loaders.XXX_loader.load(self.XXX_id)

[gotexis]

@cmmartti Hey

When you do from .loaders import Loaders

What exactly is the .loaders file? What will be the Loaders?

[japrogramer]

@gotexis probably dataloaders

e.g.

from promise.dataloader import DataLoader
class GenLoad(DataLoader):
    def batch_load_fn(self, keys):
        # Here we return a promise that will result on the
        # corresponding result for each key in keys
        lookup = {'{0}__in'.format('pk'): keys}
        return Promise.resolve(
            SomeType.get_queryset().order_by().filter(**lookup))

[gotexis]

@japrogramer
That was quick ;)

Seems the man has a middleware applied to graphene in total.

GRAPHENE = {
    'MIDDLEWARE': [
        'middlewares.LoaderMiddleware'
    ]
}

so I doubt it is only a loader for only one model...

[japrogramer]

here is a generic Dataloader generator

def genLoader(Type, attr):
    class GenLoad(DataLoader):
        def batch_load_fn(self, keys):
            # Here we return a promise that will result on the
            # corresponding result for each key in keys
            lookup = {'{0}__in'.format(attr): keys}
            return Promise.resolve(
                Type.get_queryset().order_by().filter(**lookup))
    return GenLoad

[gotexis]

@japrogramer Let me just test it now :) Hopefully it will eliminate the need to write a loader for every model :) And permanently eliminate the N+1 for graphene...

Seriously, I am so confused now. So I guess tis' the sledgehammer we need to eliminate the n+1?

[japrogramer]

;)

[japrogramer]

Also if you do something like this in a mix in for your types,
you can get a multiple items from global ids .. instead of calling get_node indevidually for each

    @classmethod
    def get_many_nodes(cls, info, ids):
        plain_ids = []
        for i in ids:
            plain_ids += relay.node.from_global_id(i)[1]
        return cls.get_queryset(info).filter(pk__in= plain_ids)

# than on your type:
SomeType.get_many_nodes(info, list_of_global_ids)

[japrogramer]

This won't solve n+1, no where in computer science can n+1 be solved only alleviated.
not even quantum computers will solve n+1;

DataLoaders will help if for some reason you keep requesting the same item multiple times per query.
But usually it's nested queries that causes the exponential growth .. so prefetching and select_related is your best option. But there is an open bug on how graphene-django throws away prefetched data .. so ...

[japrogramer]

@gotexis #693 once merged that could alleviate some of your n+1 problems since prefetching should work

[kayluhb]

@japrogramer

here is a generic Dataloader generator

def genLoader(Type, attr):
    class GenLoad(DataLoader):
        def batch_load_fn(self, keys):
            # Here we return a promise that will result on the
            # corresponding result for each key in keys
            lookup = {'{0}__in'.format(attr): keys}
            return Promise.resolve(
                Type.get_queryset().order_by().filter(**lookup))
    return GenLoad

get_queryset requires queryset and info to be defined, so this actually isn't working for me.

This works for me, but I'm not able to persist the filter_fields when using this method.

def generate_loader(Type, attr):
    class GenLoad(DataLoader):
        def batch_load_fn(self, keys):
            # Here we return a promise that will result on the
            # corresponding result for each key in keys
            lookup = {'{0}__in'.format(attr): keys}
            return Promise.resolve(Type._meta.model.objects.filter(**lookup))
    return GenLoad