Show timings for index page data gathering

This commit is contained in:
Edward Betts 2024-01-08 15:22:16 +00:00
parent cd0ffb3390
commit 1453c4015c
2 changed files with 49 additions and 20 deletions

View file

@ -6,6 +6,7 @@ import itertools
import os import os
import typing import typing
from datetime import date, datetime, timedelta from datetime import date, datetime, timedelta
from time import time
import dateutil.rrule import dateutil.rrule
import dateutil.tz import dateutil.tz
@ -353,6 +354,19 @@ def busy_event(e: Event) -> bool:
return "rebels" not in lc_title and "south west data social" not in lc_title return "rebels" not in lc_title and "south west data social" not in lc_title
async def time_function(
name: str,
func: typing.Callable[..., typing.Coroutine[typing.Any, typing.Any, typing.Any]],
*args,
**kwargs,
) -> tuple[str, typing.Any, float]:
"""Time the execution of an asynchronous function."""
start_time = time()
result = await func(*args, **kwargs)
end_time = time()
return name, result, end_time - start_time
async def get_data( async def get_data(
now: datetime, config: flask.config.Config now: datetime, config: flask.config.Config
) -> typing.Mapping[str, str | object]: ) -> typing.Mapping[str, str | object]:
@ -369,28 +383,37 @@ async def get_data(
minus_365 = now - timedelta(days=365) minus_365 = now - timedelta(days=365)
plus_365 = now + timedelta(days=365) plus_365 = now + timedelta(days=365)
( t0 = time()
gbpusd, result_list = await asyncio.gather(
gwr_advance_tickets, time_function("gbpusd", fx.get_gbpusd, config),
bank_holiday, time_function("gwr_advance_tickets", gwr.advance_ticket_date, data_dir),
rockets, time_function(
backwell_bins, "bank_holiday", uk_holiday.bank_holiday_list, last_year, next_year, data_dir
bristol_bins, ),
) = await asyncio.gather( time_function("rockets", thespacedevs.get_launches, rocket_dir, limit=40),
fx.get_gbpusd(config), time_function("backwell_bins", waste_collection_events, data_dir),
gwr.advance_ticket_date(data_dir), time_function("bristol_bins", bristol_waste_collection_events, data_dir, today),
uk_holiday.bank_holiday_list(last_year, next_year, data_dir),
thespacedevs.get_launches(rocket_dir, limit=40),
waste_collection_events(data_dir),
bristol_waste_collection_events(data_dir, today),
) )
results = {call[0]: call[1] for call in result_list}
gwr_advance_tickets = results["gwr_advance_tickets"]
data_gather_seconds = time() - t0
t0 = time()
stock_market_times = stock_market.open_and_close()
stock_market_times_seconds = time() - t0
reply: dict[str, typing.Any] = { reply: dict[str, typing.Any] = {
"now": now, "now": now,
"gbpusd": gbpusd, "gbpusd": results["gbpusd"],
"stock_markets": stock_market.open_and_close(), "stock_markets": stock_market_times,
"rockets": rockets, "rockets": results["rockets"],
"gwr_advance_tickets": gwr_advance_tickets, "gwr_advance_tickets": gwr_advance_tickets,
"data_gather_seconds": data_gather_seconds,
"stock_market_times_seconds": stock_market_times_seconds,
"timings": [(call[0], call[2]) for call in result_list],
} }
my_data = config["PERSONAL_DATA"] my_data = config["PERSONAL_DATA"]
@ -409,7 +432,7 @@ async def get_data(
us_hols = us_holidays(last_year, next_year) us_hols = us_holidays(last_year, next_year)
holidays: list[Holiday] = bank_holiday + us_hols holidays: list[Holiday] = results["bank_holiday"] + us_hols
for country in ( for country in (
"at", "at",
"be", "be",
@ -441,7 +464,7 @@ async def get_data(
events += accommodation_events events += accommodation_events
events += travel.all_events(my_data) events += travel.all_events(my_data)
events += conference.get_list(os.path.join(my_data, "conferences.yaml")) events += conference.get_list(os.path.join(my_data, "conferences.yaml"))
events += backwell_bins + bristol_bins events += results["backwell_bins"] + results["bristol_bins"]
events += read_events_yaml(my_data, last_year, next_year) events += read_events_yaml(my_data, last_year, next_year)
events += subscription.get_events(os.path.join(my_data, "subscriptions.yaml")) events += subscription.get_events(os.path.join(my_data, "subscriptions.yaml"))
events += economist.publication_dates(last_week, next_year) events += economist.publication_dates(last_week, next_year)
@ -464,7 +487,7 @@ async def get_data(
for market in overlapping_markets: for market in overlapping_markets:
events.remove(market) events.remove(market)
for launch in rockets: for launch in results["rockets"]:
dt = None dt = None
if launch["net_precision"] == "Day": if launch["net_precision"] == "Day":

View file

@ -125,6 +125,12 @@
</li> </li>
<li>Bristol Sunrise: {{ sunrise.strftime("%H:%M:%S") }} / <li>Bristol Sunrise: {{ sunrise.strftime("%H:%M:%S") }} /
Sunset: {{ sunset.strftime("%H:%M:%S") }}</li> Sunset: {{ sunset.strftime("%H:%M:%S") }}</li>
<li>Data gather took {{ "%.1f" | format(data_gather_seconds) }} seconds</li>
<li>Stock market open/close took
{{ "%.1f" | format(stock_market_times_seconds) }} seconds</li>
{% for name, seconds in timings %}
<li>{{ name }} took {{ "%.1f" | format(seconds) }} seconds</li>
{% endfor %}
</ul> </ul>
<h3>Stock markets</h3> <h3>Stock markets</h3>