External information sources (#7)

* prepares database for embedders as code

* build api around embedders

* adds embedder states

* adds option to add external information sources like model callbacks

* update to differentiate between callbacks and heuristics

* implements suggestions

* removes embedder and implements suggestion from PR 39

* implements PR 7

Author: jhoetter

business_objects/attribute.py

@@ -39,6 +39,17 @@ def get_by_name(project_id: str, name: str) -> Attribute:
     )
 
 
+def get_all_by_names(project_id: str, attribute_names: List[str]) -> List[Attribute]:
+    return (
+        session.query(Attribute)
+        .filter(
+            Attribute.project_id == project_id,
+            Attribute.name.in_(attribute_names),
+        )
+        .all()
+    )
+
+
 def get_all(project_id: str) -> List[Attribute]:
     return session.query(Attribute).filter(Attribute.project_id == project_id).all()
 

business_objects/information_source.py

@@ -22,6 +22,17 @@ def get(project_id: str, source_id: str) -> InformationSource:
     )
 
 
+def get_by_name(project_id: str, name: str) -> InformationSource:
+    return (
+        session.query(InformationSource)
+        .filter(
+            InformationSource.project_id == project_id,
+            InformationSource.name == name,
+        )
+        .first()
+    )
+
+
 def get_all(project_id: str) -> List[InformationSource]:
     return (
         session.query(InformationSource)
@@ -168,7 +179,13 @@ def get_exclusion_record_ids_for_task(task_id: str) -> List[str]:
     return exclusion_ids
 
 
-def get_overview_data(project_id: str) -> List[Dict[str, Any]]:
+def get_overview_data(
+    project_id: str, is_model_callback: bool = False
+) -> List[Dict[str, Any]]:
+    if is_model_callback:
+        type_selection = " = 'MODEL_CALLBACK'"
+    else:
+        type_selection = " != 'MODEL_CALLBACK'"
     query = f"""
     SELECT array_agg(row_to_json(data_select))
     FROM (
@@ -205,14 +222,16 @@ def get_overview_data(project_id: str) -> List[Dict[str, Any]]:
             GROUP BY source_id) stats
             ON _is.id = stats.source_id
         WHERE _is.project_id = '{project_id}'
+        AND _is.type {type_selection}
         ORDER BY "createdAt" DESC,name
         )data_select """
     values = general.execute_first(query)
 
     if values:
         return values[0]
 
-def continue_payload(project_id:str,source_id:str,payload_id:str)->bool:
+
+def continue_payload(project_id: str, source_id: str, payload_id: str) -> bool:
     query = f"""
     SELECT isp.state
     FROM information_source_payload isp
@@ -223,7 +242,7 @@ def continue_payload(project_id:str,source_id:str,payload_id:str)->bool:
     AND isp.project_id = '{project_id}' """
 
     value = general.execute_first(query)
-    if not value or value[0]!= "CREATED":
+    if not value or value[0] != "CREATED":
         return False
     return True
 
@@ -444,8 +463,16 @@ def update_quantity_stats(
 
 
 def update_is_selected_for_project(
-    project_id: str, update_value: bool, with_commit: bool = False
+    project_id: str,
+    update_value: bool,
+    with_commit: bool = False,
+    is_model_callback: bool = False,
 ) -> None:
+
+    if is_model_callback:
+        type_selection = " = 'MODEL_CALLBACK'"
+    else:
+        type_selection = " != 'MODEL_CALLBACK'"
     if update_value:
         str_value = "TRUE"
     else:
@@ -454,7 +481,9 @@ def update_is_selected_for_project(
     query = f"""
     UPDATE information_source
     SET is_selected = {str_value}
-    WHERE project_id = '{project_id}' """
+    WHERE project_id = '{project_id}'
+    AND type {type_selection}
+    """
     general.execute(query)
     general.flush_or_commit(with_commit)
 

business_objects/labeling_task.py

@@ -32,6 +32,14 @@ def get_task_name_id_dict(project_id: str) -> Dict[str, str]:
     return {labeling_task.name: labeling_task.id for labeling_task in labeling_tasks}
 
 
+def get_labeling_task_by_name(project_id: str, task_name: str) -> LabelingTask:
+    return (
+        session.query(LabelingTask)
+        .filter(LabelingTask.project_id == project_id, LabelingTask.name == task_name)
+        .first()
+    )
+
+
 def get_labeling_tasks_by_selected_sources(project_id: str) -> List[LabelingTask]:
     return (
         session.query(LabelingTask)

business_objects/record_label_association.py

@@ -474,6 +474,20 @@ def delete_by_source_id(
     general.flush_or_commit(with_commit)
 
 
+def delete_by_source_id_and_record_ids(
+    project_id: str,
+    information_source_id: str,
+    record_ids: List[str],
+    with_commit: bool = False,
+) -> None:
+    session.query(RecordLabelAssociation).filter(
+        RecordLabelAssociation.project_id == project_id,
+        RecordLabelAssociation.source_id == information_source_id,
+        RecordLabelAssociation.record_id.in_(record_ids),
+    ).delete()
+    general.flush_or_commit(with_commit)
+
+
 def delete_record_label_associations(
     project_id: str, record_task_concatenation: str, with_commit: bool = False
 ) -> None:

enums.py

@@ -27,6 +27,7 @@ class LabelSource(Enum):
     # WEAK_SUPERVISION = Output of the Weak Supervision Model - ehemeals "programmatic"
     WEAK_SUPERVISION = "WEAK_SUPERVISION"
     INFORMATION_SOURCE = "INFORMATION_SOURCE"
+    MODEL_CALLBACK = "MODEL_CALLBACK"
 
 
 class InformationSourceType(Enum):
@@ -186,6 +187,7 @@ class NotificationType(Enum):
     WEAK_SUPERVISION_TASK_STARTED = "WEAK_SUPERVISION_TASK_STARTED"
     WEAK_SUPERVISION_TASK_DONE = "WEAK_SUPERVISION_TASK_DONE"
     WEAK_SUPERVISION_TASK_FAILED = "WEAK_SUPERVISION_TASK_FAILED"
+
     INFORMATION_SOURCE_STARTED = "INFORMATION_SOURCE_STARTED"
     INFORMATION_SOURCE_PREPARATION_STARTED = "INFORMATION_SOURCE_PREPARATION_STARTED"
     INFORMATION_SOURCE_COMPLETED = "INFORMATION_SOURCE_COMPLETED"

models.py

@@ -555,7 +555,6 @@ class RecordAttributeTokenStatistics(Base):
     num_token = Column(Integer)
 
 
-# -------------------- EMBEDDING_ --------------------
 class Embedding(Base):
     __tablename__ = Tablenames.EMBEDDING.value
     id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)