From 0eb619c0179a7b27c88e65e7f1b6cdf6120b5403 Mon Sep 17 00:00:00 2001 From: "bart.leboeuf" Date: Thu, 19 Jan 2023 19:03:35 +0100 Subject: [PATCH 1/2] Feat: add concurrency to findMatches methods --- CHANGELOG.md | 5 ++ MTM/__init__.py | 133 +++++++++++++++++++++++++++++------------------- MTM/version.py | 2 +- 3 files changed, 88 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f6a564..33aaae6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +### [1.6.4] - 2023-01-19 + +### Changed +- Improve speed by adding concurrency in the findMatches method, using half the number of cpu cores available. + ### [1.6.3] - 2021-11-24 ### Changed diff --git a/MTM/__init__.py b/MTM/__init__.py index b519984..432ca04 100644 --- a/MTM/__init__.py +++ b/MTM/__init__.py @@ -1,13 +1,16 @@ """Main code for Multi-Template-Matching (MTM).""" +import os +import warnings +from concurrent.futures import ThreadPoolExecutor, as_completed + import cv2 -import numpy as np +import numpy as np import pandas as pd -import warnings +from scipy.signal import find_peaks from skimage.feature import peak_local_max -from scipy.signal import find_peaks -from .version import __version__ from .NMS import NMS +from .version import __version__ __all__ = ['NMS'] @@ -33,7 +36,7 @@ def _findLocalMax_(corrMap, score_threshold=0.6): peaks = [[i,0] for i in peaks[0]] - else: # Correlatin map is 2D + else: # Correlation map is 2D peaks = peak_local_max(corrMap, threshold_abs=score_threshold, exclude_border=False).tolist() return peaks @@ -116,82 +119,110 @@ def findMatches(listTemplates, image, method=cv2.TM_CCOEFF_NORMED, N_object=floa ------- - Pandas DataFrame with 1 row per hit and column "TemplateName"(string), "BBox":(X, Y, Width, Height), "Score":float """ - if N_object != float("inf") and type(N_object) != int: + if N_object != float("inf") and not isinstance(N_object, int): raise TypeError("N_object must be an integer") ## Crop image to search region if provided if searchBox is not None: xOffset, yOffset, searchWidth, searchHeight = searchBox image = image[yOffset : yOffset+searchHeight, xOffset : xOffset+searchWidth] - else: xOffset=yOffset=0 - + # Check that the template are all smaller are equal to the image (original, or cropped if there is a search region) for index, tempTuple in enumerate(listTemplates): - + if not isinstance(tempTuple, tuple) or len(tempTuple)==1: raise ValueError("listTemplates should be a list of tuples as ('name','array') or ('name', 'array', 'mask')") - + templateSmallerThanImage = all(templateDim <= imageDim for templateDim, imageDim in zip(tempTuple[1].shape, image.shape)) - + if not templateSmallerThanImage : fitIn = "searchBox" if (searchBox is not None) else "image" raise ValueError("Template '{}' at index {} in the list of templates is larger than {}.".format(tempTuple[0], index, fitIn) ) - + listHit = [] - for tempTuple in listTemplates: + ## Use multi-threading to iterate through all templates, using half the number of cpu cores available. + with ThreadPoolExecutor(max_workers=round(os.cpu_count()*.5)) as executor: + futures = [executor.submit(_multi_compute, tempTuple, image, method, N_object, score_threshold, xOffset, yOffset, listHit) for tempTuple in listTemplates] + for future in as_completed(futures): + _ = future.result() - templateName, template = tempTuple[:2] - mask = None + if listHit: + return pd.DataFrame(listHit) # All possible hits before Non-Maxima Supression + else: + return pd.DataFrame(columns=["TemplateName", "BBox", "Score"]) - if len(tempTuple)>=3: # ie a mask is also provided - if method in (0,3): - mask = tempTuple[2] - else: - warnings.warn("Template matching method not supporting the use of Mask. Use 0/TM_SQDIFF or 3/TM_CCORR_NORMED.") - #print('\nSearch with template : ',templateName) - corrMap = computeScoreMap(template, image, method, mask=mask) +def _multi_compute(tempTuple, image, method, N_object, score_threshold, xOffset, yOffset, listHit): + """ + Find all possible template locations satisfying the score threshold provided a template to search and an image. + Add the hits in the list of hits. + + Parameters + ---------- + - tempTuple : a tuple (LabelString, template, mask (optional)) + template to search in each image, associated to a label + labelstring : string + template : numpy array (grayscale or RGB) + mask (optional): numpy array, should have the same dimensions and type than the template - ## Find possible location of the object - if N_object==1: # Detect global Min/Max - minVal, maxVal, minLoc, maxLoc = cv2.minMaxLoc(corrMap) + - image : Grayscale or RGB numpy array + image in which to perform the search, it should be the same bitDepth and number of channels than the templates - if method in (0,1): - peaks = [minLoc[::-1]] # opposite sorting than in the multiple detection + - method : int + one of OpenCV template matching method (0 to 5), default 5=0-mean cross-correlation - else: - peaks = [maxLoc[::-1]] + - N_object: int or float("inf") + expected number of objects in the image, default to infinity if unknown + - score_threshold: float in range [0,1] + if N_object>1, returns local minima/maxima respectively below/above the score_threshold - else:# Detect local max or min - if method in (0,1): # Difference => look for local minima - peaks = _findLocalMin_(corrMap, score_threshold) + - xOffset : int + optional the x offset if the search area is provided - else: - peaks = _findLocalMax_(corrMap, score_threshold) + - yOffset : int + optional the y offset if the search area is provided + - listHit : the list of hits which we want to add the discovered hit + expected array of hits + """ + templateName, template = tempTuple[:2] + mask = None - #print('Initially found',len(peaks),'hit with this template') + if len(tempTuple)>=3: # ie a mask is also provided + if method in (0,3): + mask = tempTuple[2] + else: + warnings.warn("Template matching method not supporting the use of Mask. Use 0/TM_SQDIFF or 3/TM_CCORR_NORMED.") + #print('\nSearch with template : ',templateName) + corrMap = computeScoreMap(template, image, method, mask=mask) - # Once every peak was detected for this given template - ## Create a dictionnary for each hit with {'TemplateName':, 'BBox': (x,y,Width, Height), 'Score':coeff} + ## Find possible location of the object + if N_object==1: # Detect global Min/Max + _, _, minLoc, maxLoc = cv2.minMaxLoc(corrMap) + if method in (0,1): + peaks = [minLoc[::-1]] # opposite sorting than in the multiple detection + else: + peaks = [maxLoc[::-1]] + else:# Detect local max or min + if method in (0,1): # Difference => look for local minima + peaks = _findLocalMin_(corrMap, score_threshold) + else: + peaks = _findLocalMax_(corrMap, score_threshold) - height, width = template.shape[0:2] # slicing make sure it works for RGB too + #print('Initially found',len(peaks),'hit with this template') - for peak in peaks : - coeff = corrMap[tuple(peak)] - newHit = {'TemplateName':templateName, 'BBox': ( int(peak[1])+xOffset, int(peak[0])+yOffset, width, height ) , 'Score':coeff} + # Once every peak was detected for this given template + ## Create a dictionnary for each hit with {'TemplateName':, 'BBox': (x,y,Width, Height), 'Score':coeff} - # append to list of potential hit before Non maxima suppression - listHit.append(newHit) + height, width = template.shape[0:2] # slicing make sure it works for RGB too - if listHit: - return pd.DataFrame(listHit) # All possible hits before Non-Maxima Supression - else: - return pd.DataFrame(columns=["TemplateName", "BBox", "Score"]) # empty df with correct column header + for peak in peaks : + # append to list of potential hit before Non maxima suppression + listHit.append({'TemplateName':templateName, 'BBox': ( int(peak[1])+xOffset, int(peak[0])+yOffset, width, height ) , 'Score':corrMap[tuple(peak)]}) # empty df with correct column header def matchTemplates(listTemplates, image, method=cv2.TM_CCOEFF_NORMED, N_object=float("inf"), score_threshold=0.5, maxOverlap=0.25, searchBox=None): @@ -239,7 +270,7 @@ def matchTemplates(listTemplates, image, method=cv2.TM_CCOEFF_NORMED, N_object=f tableHit = findMatches(listTemplates, image, method, N_object, score_threshold, searchBox) if method == 0: raise ValueError("The method TM_SQDIFF is not supported. Use TM_SQDIFF_NORMED instead.") - sortAscending = True if method==1 else False + sortAscending = (method==1) return NMS(tableHit, score_threshold, sortAscending, N_object, maxOverlap) @@ -275,7 +306,7 @@ def drawBoxesOnRGB(image, tableHit, boxThickness=2, boxColor=(255, 255, 00), sho if image.ndim == 2: outImage = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB) # convert to RGB to be able to show detections as color box on grayscale image else: outImage = image.copy() - for index, row in tableHit.iterrows(): + for _, row in tableHit.iterrows(): x,y,w,h = row['BBox'] cv2.rectangle(outImage, (x, y), (x+w, y+h), color=boxColor, thickness=boxThickness) if showLabel: cv2.putText(outImage, text=row['TemplateName'], org=(x, y), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=labelScale, color=labelColor, lineType=cv2.LINE_AA) @@ -315,9 +346,9 @@ def drawBoxesOnGray(image, tableHit, boxThickness=2, boxColor=255, showLabel=Fal if image.ndim == 3: outImage = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY) # convert to RGB to be able to show detections as color box on grayscale image else: outImage = image.copy() - for index, row in tableHit.iterrows(): + for _, row in tableHit.iterrows(): x,y,w,h = row['BBox'] cv2.rectangle(outImage, (x, y), (x+w, y+h), color=boxColor, thickness=boxThickness) if showLabel: cv2.putText(outImage, text=row['TemplateName'], org=(x, y), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=labelScale, color=labelColor, lineType=cv2.LINE_AA) - return outImage \ No newline at end of file + return outImage diff --git a/MTM/version.py b/MTM/version.py index 94cbd5e..c33dda7 100644 --- a/MTM/version.py +++ b/MTM/version.py @@ -2,4 +2,4 @@ # 1) we don't load dependencies by storing it in __init__.py # 2) we can import it in setup.py for the same reason # 3) we can import it into your module module -__version__ = '1.6.3' \ No newline at end of file +__version__ = '1.6.4' \ No newline at end of file From 74660d3e30ce5cb1bb8adc809ddffbe18a803b52 Mon Sep 17 00:00:00 2001 From: Laurent Thomas Date: Fri, 3 Mar 2023 11:16:56 +0100 Subject: [PATCH 2/2] few comments README + update date for release 1.6.4 --- CHANGELOG.md | 3 ++- MTM/__init__.py | 1 + README.md | 11 ++++++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33aaae6..0d49534 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -### [1.6.4] - 2023-01-19 +### [1.6.4] - 2023-03-03 ### Changed - Improve speed by adding concurrency in the findMatches method, using half the number of cpu cores available. +- Mention installation in editable mode in README ### [1.6.3] - 2021-11-24 diff --git a/MTM/__init__.py b/MTM/__init__.py index 432ca04..f9dc055 100644 --- a/MTM/__init__.py +++ b/MTM/__init__.py @@ -222,6 +222,7 @@ def _multi_compute(tempTuple, image, method, N_object, score_threshold, xOffset, for peak in peaks : # append to list of potential hit before Non maxima suppression + # no need to lock the list, append is thread-safe listHit.append({'TemplateName':templateName, 'BBox': ( int(peak[1])+xOffset, int(peak[0])+yOffset, width, height ) , 'Score':corrMap[tuple(peak)]}) # empty df with correct column header diff --git a/README.md b/README.md index 60d145a..6344009 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,10 @@ The main function `MTM.matchTemplates` returns the best predicted locations prov The branch opencl contains some test using the UMat object to run on GPU, but it is actually slow, which can be expected for small dataset as the transfer of the data between the CPU and GPU is slow. -__** News **__ : You might be interested to test the newer python implementation which is more object-oriented and only relying on scikit-image and shapely.* +# News +- 03/03/2023 : Version 1.6.4 contributed by @bartleboeuf comes with speed enhancement thanks to parallelizing of the individual template searches. +Thanks for this first PR !! +- 10/11/2021 : You might be interested to test the newer python implementation which is more object-oriented and only relying on scikit-image and shapely.* https://github.com/multi-template-matching/mtm-python-oop # Installation @@ -16,6 +19,12 @@ Using pip in a python environment, `pip install Multi-Template-Matching` Once installed, `import MTM`should work. Example jupyter notebooks can be downloaded from the tutorial folder of the github repository and executed in the newly configured python environement. +## Install in dev mode +If you want to contribute or experiment with the source code, you can install the package "from source", by first downloading or cloning the repo. +Then opening a command prompt in the repo's root directory (the one containing this README) and calling `pip install -e .` (mind the final dot). +- the `-e` flag stands for editable and make sure that any change to the source code will be directly reflected when you import the package in your script +- the . just tell pip to look for the package to install in the current directory + # Documentation The [wiki](https://github.com/multi-template-matching/MultiTemplateMatching-Python/wiki) section of the repo contains a mini API documentation with description of the key functions of the package. The [website](https://multi-template-matching.github.io/Multi-Template-Matching/) of the project contains some more general documentation.