From 9941fa02301f0732d872722a1c16b6305d9e89f9 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 27 Oct 2020 15:55:23 +0100 Subject: [PATCH] feature: switch to poetry and minor template changes --- setup_template.sh | 12 ++++------ .../.github/workflows/publish.yml | 14 ++++++----- .../.github/workflows/test.yml | 4 ++-- .../dev_requirements.txt | 4 ---- .../pyproject.toml | 24 +++++++++++++++++++ .../requirements.txt | 6 ----- .../setup.cfg | 2 +- .../datastream/__init__.py | 1 + .../datastream/datasets.py | 21 ++++++++++++++++ .../{{cookiecutter.package_name}}/metrics.py | 2 +- .../{evaluate_datasets.py => datasets.py} | 0 11 files changed, 62 insertions(+), 28 deletions(-) delete mode 100644 template/{{cookiecutter.repository_name}}/dev_requirements.txt create mode 100644 template/{{cookiecutter.repository_name}}/pyproject.toml delete mode 100644 template/{{cookiecutter.repository_name}}/requirements.txt create mode 100644 template/{{cookiecutter.repository_name}}/{{cookiecutter.package_name}}/datastream/datasets.py rename template/{{cookiecutter.repository_name}}/{{cookiecutter.package_name}}/problem/{evaluate_datasets.py => datasets.py} (100%) diff --git a/setup_template.sh b/setup_template.sh index f6f0ca2..d3c6863 100755 --- a/setup_template.sh +++ b/setup_template.sh @@ -6,15 +6,11 @@ VERSION=${1:-"3.8"} rm -rf test_template mkdir test_template cd test_template -virtualenv venv -p "python${VERSION}" -source venv/bin/activate -cd .. -pip install -e . -cd test_template +poetry init +poetry run pip install -e ../. echo "test test 0.1.0 test project -y" | python -m workflow.setup_project -pip install -r requirements.txt -deactivate +y" | poetry run python -m workflow.setup_project +poetry install diff --git a/template/{{cookiecutter.repository_name}}/.github/workflows/publish.yml b/template/{{cookiecutter.repository_name}}/.github/workflows/publish.yml index 1a0e27e..18f75d6 100644 --- a/template/{{cookiecutter.repository_name}}/.github/workflows/publish.yml +++ b/template/{{cookiecutter.repository_name}}/.github/workflows/publish.yml @@ -17,14 +17,16 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r dev_requirements.txt - pip install -r requirements.txt + pip install poetry + poetry install - name: Build run: | python setup.py bdist_wheel --python-tag py3 - - name: Upload + - name: Packagr Publish env: - TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }} - run: | - curl -F package=@dist/$(ls dist) https://${TOKEN}@push.fury.io/aiwizo/ + TWINE_USERNAME: "" + TWINE_PASSWORD: ${{ secrets.PACKAGR_TOKEN }} + URL: ${{ secrets.PACKAGR_URL }} + run: + twine upload dist/* --repository-url https://api.packagr.app/${URL}/ {%- endraw %} diff --git a/template/{{cookiecutter.repository_name}}/.github/workflows/test.yml b/template/{{cookiecutter.repository_name}}/.github/workflows/test.yml index a0b0119..9cbbad1 100644 --- a/template/{{cookiecutter.repository_name}}/.github/workflows/test.yml +++ b/template/{{cookiecutter.repository_name}}/.github/workflows/test.yml @@ -32,8 +32,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r dev_requirements.txt - pip install -r requirements.txt + pip install poetry + poetry install - name: Lint with flake8 run: | diff --git a/template/{{cookiecutter.repository_name}}/dev_requirements.txt b/template/{{cookiecutter.repository_name}}/dev_requirements.txt deleted file mode 100644 index 8d34f21..0000000 --- a/template/{{cookiecutter.repository_name}}/dev_requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -flake8 -pytest -setuptools -wheel \ No newline at end of file diff --git a/template/{{cookiecutter.repository_name}}/pyproject.toml b/template/{{cookiecutter.repository_name}}/pyproject.toml new file mode 100644 index 0000000..c26102b --- /dev/null +++ b/template/{{cookiecutter.repository_name}}/pyproject.toml @@ -0,0 +1,24 @@ +[tool.poetry] +name = "{{cookiecutter.repository_name}}" +version = "{{cookiecutter.package_version}}" +description = "{{cookiecutter.package_description}}" +authors = ["Aiwizo AB"] + +[tool.poetry.dependencies] +python = "^3.8" +torch = ">=1.6.0" +numpy = ">=1.19.2" +torchvision = ">=0.7.0" +opencv-python = ">=4.4.0" +ml-workflow = ">=0.8.1" +pytorch-datastream = ">=0.3.6" + +[tool.poetry.dev-dependencies] +guildai = ">=0.7.1.dev6" +data-kale = ">=0.1.3" +pylint = ">=2.6.0" +flake8 = ">=3.8.4" + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" diff --git a/template/{{cookiecutter.repository_name}}/requirements.txt b/template/{{cookiecutter.repository_name}}/requirements.txt deleted file mode 100644 index 88c0122..0000000 --- a/template/{{cookiecutter.repository_name}}/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -guildai>=0.7.0.post1 -torch>=1.6.0 -numpy>=1.18.3 -ml-workflow>=0.7.4 -flake8>=3.8.3 -torchvision>=0.7.0 \ No newline at end of file diff --git a/template/{{cookiecutter.repository_name}}/setup.cfg b/template/{{cookiecutter.repository_name}}/setup.cfg index 20a361a..a00e56f 100644 --- a/template/{{cookiecutter.repository_name}}/setup.cfg +++ b/template/{{cookiecutter.repository_name}}/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = {{cookiecutter.repository_name}} -author = "Aiwizo" +author = "Aiwizo AB" author-email = user@aiwizo.com summary = Simple library for converting kw6 files to jpg and providing a dataset api description-file = diff --git a/template/{{cookiecutter.repository_name}}/{{cookiecutter.package_name}}/datastream/__init__.py b/template/{{cookiecutter.repository_name}}/{{cookiecutter.package_name}}/datastream/__init__.py index 1d6eaa1..42aa2e6 100644 --- a/template/{{cookiecutter.repository_name}}/{{cookiecutter.package_name}}/datastream/__init__.py +++ b/template/{{cookiecutter.repository_name}}/{{cookiecutter.package_name}}/datastream/__init__.py @@ -1,3 +1,4 @@ +from {{cookiecutter.package_name}}.datastream.datasets import datasets from {{cookiecutter.package_name}}.datastream.augmenter import augmenter from {{cookiecutter.package_name}}.datastream.evaluate_datastreams import ( evaluate_datastreams diff --git a/template/{{cookiecutter.repository_name}}/{{cookiecutter.package_name}}/datastream/datasets.py b/template/{{cookiecutter.repository_name}}/{{cookiecutter.package_name}}/datastream/datasets.py new file mode 100644 index 0000000..4c0887b --- /dev/null +++ b/template/{{cookiecutter.repository_name}}/{{cookiecutter.package_name}}/datastream/datasets.py @@ -0,0 +1,21 @@ +import numpy as np +import pandas as pd +from datastream import Datastream + +from {{cookiecutter.package_name}} import problem + + +def datasets(): + datasets = problem.datasets() + datasets['train'] = datasets['train'].split( + key_column='index', + proportions=dict(gradient=0.8, early_stopping=0.2), + stratify_column='class_name', + filepath='{{cookiecutter.package_name}}/splits/early_stopping.json', + ) + + return dict( + gradient=datasets['train']['gradient'], + early_stopping=datasets['train']['early_stopping'], + compare=datasets['compare'], + ) diff --git a/template/{{cookiecutter.repository_name}}/{{cookiecutter.package_name}}/metrics.py b/template/{{cookiecutter.repository_name}}/{{cookiecutter.package_name}}/metrics.py index a84d6ae..69fdef4 100644 --- a/template/{{cookiecutter.repository_name}}/{{cookiecutter.package_name}}/metrics.py +++ b/template/{{cookiecutter.repository_name}}/{{cookiecutter.package_name}}/metrics.py @@ -12,7 +12,7 @@ def train_metrics(): def progress_metrics(): - return dict( + return dict( batch_loss=ignite.metrics.RunningAverage( output_transform=lambda output: output['loss'], epoch_bound=False, diff --git a/template/{{cookiecutter.repository_name}}/{{cookiecutter.package_name}}/problem/evaluate_datasets.py b/template/{{cookiecutter.repository_name}}/{{cookiecutter.package_name}}/problem/datasets.py similarity index 100% rename from template/{{cookiecutter.repository_name}}/{{cookiecutter.package_name}}/problem/evaluate_datasets.py rename to template/{{cookiecutter.repository_name}}/{{cookiecutter.package_name}}/problem/datasets.py