Merge remote-tracking branch 'upstream/develop' into feature/text-extract

# Conflicts:
#	app/templates/partials/user_profile/sidebar_left/profile_card/user_details.html

Author: Gonzom

app/database/models.py

@@ -442,6 +442,20 @@ class WikipediaEvents(Base):
     date_inserted = Column(DateTime, default=datetime.utcnow)
 
 
+class CoronaStats(Base):
+    __tablename__ = "corona_stats"
+
+    id = Column(Integer, primary_key=True, index=True)
+    date_ = Column(DateTime, nullable=False)
+    date_inserted = Column(DateTime, default=datetime.utcnow)
+    vaccinated = Column(Integer, nullable=False)
+    vaccinated_total = Column(Integer, nullable=False)
+    vaccinated_population_perc = Column(Integer, nullable=False)
+    vaccinated_second_dose = Column(Integer, nullable=False)
+    vaccinated_second_dose_total = Column(Integer, nullable=False)
+    vaccinated_second_dose_perc = Column(Float, nullable=False)
+
+
 class Quote(Base):
     __tablename__ = "quotes"
 

app/internal/corona_stats.py

@@ -0,0 +1,153 @@
+import json
+import random
+from datetime import date, datetime
+from typing import Any, Dict
+
+import httpx
+from fastapi import Depends
+from loguru import logger
+from sqlalchemy import desc, func
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.orm import Session
+from sqlalchemy.orm.exc import NoResultFound
+
+from app.database.models import CoronaStats
+from app.dependencies import get_db
+
+DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+CORONA_API_URL = (
+    "https://datadashboardapi.health.gov.il/api/queries/vaccinated"
+)
+USER_AGENT_OPTIONS = [
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5)"
+    " AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0)"
+    " Gecko/20100101 Firefox/77.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:77.0) "
+    "Gecko/20100101 Firefox/77.0",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko)"
+    " Chrome/83.0.4103.97 Safari/537.36",
+]
+
+
+def create_stats_object(corona_stats_data: Dict[str, Any]) -> CoronaStats:
+    """Dict -> DB Object"""
+    return CoronaStats(
+        date_=datetime.strptime(
+            corona_stats_data.get("Day_Date"),
+            DATETIME_FORMAT,
+        ),
+        vaccinated=corona_stats_data.get("vaccinated"),
+        vaccinated_total=corona_stats_data.get("vaccinated_cum"),
+        vaccinated_population_perc=corona_stats_data.get(
+            "vaccinated_population_perc",
+        ),
+        vaccinated_second_dose=corona_stats_data.get(
+            "vaccinated_seconde_dose",
+        ),
+        vaccinated_second_dose_total=corona_stats_data.get(
+            "vaccinated_seconde_dose_cum",
+        ),
+        vaccinated_second_dose_perc=corona_stats_data.get(
+            "vaccinated_seconde_dose_population_perc",
+        ),
+    )
+
+
+def serialize_stats(stats_object: CoronaStats) -> Dict[str, Any]:
+    """ DB Object -> Dict """
+    return {
+        "vaccinated_second_dose_perc": (
+            stats_object.vaccinated_second_dose_perc
+        ),
+        "vaccinated_second_dose_total": (
+            stats_object.vaccinated_second_dose_total
+        ),
+    }
+
+
+def serialize_dict_stats(stats_dict: Dict[str, Any]) -> Dict[str, Any]:
+    """ api Dict -> pylender Dict """
+    return {
+        "vaccinated_second_dose_perc": (
+            stats_dict.get("vaccinated_seconde_dose_population_perc")
+        ),
+        "vaccinated_second_dose_total": (
+            stats_dict.get("vaccinated_seconde_dose_cum")
+        ),
+    }
+
+
+def save_corona_stats(
+    corona_stats_data: Dict[str, Any],
+    db: Session = Depends(get_db),
+) -> None:
+    db.add(create_stats_object(corona_stats_data))
+    db.commit()
+
+
+def insert_to_db_if_needed(
+    corona_stats_data: Dict[str, Any],
+    db: Session = Depends(get_db),
+) -> Dict[str, Any]:
+    """ gets the latest data inserted to gov database """
+    latest_date = datetime.strptime(
+        corona_stats_data.get("Day_Date"),
+        DATETIME_FORMAT,
+    )
+    latest_saved = None
+    try:
+        latest_saved = (
+            db.query(CoronaStats).order_by(desc(CoronaStats.date_)).one()
+        )
+    except NoResultFound:
+        # on first system load, the table is empty
+        save_corona_stats(corona_stats_data, db)
+        return corona_stats_data
+
+    if latest_saved is not None:
+        # on more recent data arrival, we update the database
+        if latest_saved.date_ < latest_date:
+            save_corona_stats(corona_stats_data, db)
+            return corona_stats_data
+        else:
+            return serialize_stats(latest_saved)
+
+
+async def get_vacinated_data() -> Dict[str, Any]:
+    async with httpx.AsyncClient() as client:
+        headers = {"User-Agent": random.choice(USER_AGENT_OPTIONS)}
+        res = await client.get(CORONA_API_URL, headers=headers)
+        return json.loads(res.text)[-1]
+
+
+def get_vacinated_data_from_db(db: Session = Depends(get_db)) -> CoronaStats:
+    # pulls once a day, it won't be the most updated data
+    # but we dont want to be blocked for too many requests
+    return (
+        db.query(CoronaStats)
+        .filter(func.date(CoronaStats.date_inserted) == date.today())
+        .one()
+    )
+
+
+async def get_corona_stats(db: Session = Depends(get_db)) -> Dict[str, Any]:
+    try:
+        db_data = get_vacinated_data_from_db(db)
+        corona_stats_data = serialize_stats(db_data)
+
+    except NoResultFound:
+        try:
+            response_data = await get_vacinated_data()
+            insert_to_db_if_needed(response_data, db)
+            corona_stats_data = serialize_dict_stats(response_data)
+        except json.decoder.JSONDecodeError:
+            corona_stats_data = {"error": "No data"}
+
+    except (SQLAlchemyError, AttributeError) as e:
+        logger.exception(f"corona stats failed with error: {e}")
+        corona_stats_data = {"error": "No data"}
+    return corona_stats_data

app/internal/on_this_day_events.py

@@ -43,6 +43,6 @@ def get_on_this_day_events(db: Session = Depends(get_db)) -> Dict[str, Any]:
     except NoResultFound:
         data = insert_on_this_day_data(db)
     except (SQLAlchemyError, AttributeError) as e:
-        logger.error(f"on this day failed with error: {e}")
+        logger.exception(f"on this day failed with error: {e}")
         data = {"events": [], "wikipedia": "https://en.wikipedia.org/"}
     return data

app/main.py

@@ -77,6 +77,7 @@ def create_tables(engine, psql_environment):
     profile,
     register,
     search,
+    settings,
     telegram,
     user,
     weekview,
@@ -128,6 +129,7 @@ async def swagger_ui_redirect():
     register.router,
     salary.router,
     search.router,
+    settings.router,
     telegram.router,
     user.router,
     weekview.router,

app/media/arrow-left.png

@@ -10,6 +10,7 @@
 from app import config
 from app.database.models import User
 from app.dependencies import GOOGLE_ERROR, MEDIA_PATH, get_db, templates
+from app.internal.corona_stats import get_corona_stats
 from app.internal.import_holidays import (
     get_holidays_from_file,
     save_holidays_to_db,
@@ -70,6 +71,7 @@ async def profile(
     ]
 
     on_this_day_data = get_on_this_day_events(session)
+    corona_stats_data = await get_corona_stats(session)
 
     return templates.TemplateResponse(
         "profile.html",
@@ -80,6 +82,7 @@ async def profile(
             "signs": signs,
             "google_error": GOOGLE_ERROR,
             "on_this_day_data": on_this_day_data,
+            "corona_stats_data": corona_stats_data,
             "privacy": PrivacyKinds,
         },
     )

app/routers/profile.py

@@ -0,0 +1,20 @@
+from fastapi import APIRouter, Request
+
+from app.dependencies import templates
+
+
+router = APIRouter(
+    prefix="/settings",
+    tags=["settings"],
+    responses={404: {"description": "Not found"}},
+)
+
+
+@router.get("/")
+async def settings(request: Request) -> templates.TemplateResponse:
+    return templates.TemplateResponse(
+        "settings.html",
+        {
+            "request": request,
+        },
+    )

app/routers/settings.py

@@ -116,7 +116,6 @@ header {
   position: sticky;
   top: 0;
   display: flex;
-  grid-flow: row wrap;
   margin: 0 var(--space_s);
   margin: 0 1rem 0 1rem;
   background-color: var(--backgroundcol);

app/static/grid_style.css

@@ -0,0 +1,23 @@
+document.addEventListener('DOMContentLoaded', () => {
+    const tabBtn = document.getElementsByClassName("tab");
+    for (let i = 0; i < tabBtn.length; i++) {
+        const btn = document.getElementById("tab" + i);
+        btn.addEventListener('click', () => {
+            tabClick(btn.id, tabBtn);
+        });
+    }
+});
+
+
+function tabClick(tab_id, tabBtn) {
+    let shownTab = document.querySelector(".tab-show");
+    let selectedTabContent = document.querySelector(`#${tab_id}-content`);
+    shownTab.classList.remove("tab-show");
+    shownTab.classList.add("tab-hide");
+    for (btn of tabBtn) {
+        btn.children[0].classList.remove("active");
+    }
+    document.getElementById(tab_id).classList.add("active");
+    selectedTabContent.classList.remove("tab-hide");
+    selectedTabContent.classList.add("tab-show");
+}