Show timings for index page data gathering
This commit is contained in:
parent
cd0ffb3390
commit
1453c4015c
|
@ -6,6 +6,7 @@ import itertools
|
||||||
import os
|
import os
|
||||||
import typing
|
import typing
|
||||||
from datetime import date, datetime, timedelta
|
from datetime import date, datetime, timedelta
|
||||||
|
from time import time
|
||||||
|
|
||||||
import dateutil.rrule
|
import dateutil.rrule
|
||||||
import dateutil.tz
|
import dateutil.tz
|
||||||
|
@ -353,6 +354,19 @@ def busy_event(e: Event) -> bool:
|
||||||
return "rebels" not in lc_title and "south west data social" not in lc_title
|
return "rebels" not in lc_title and "south west data social" not in lc_title
|
||||||
|
|
||||||
|
|
||||||
|
async def time_function(
|
||||||
|
name: str,
|
||||||
|
func: typing.Callable[..., typing.Coroutine[typing.Any, typing.Any, typing.Any]],
|
||||||
|
*args,
|
||||||
|
**kwargs,
|
||||||
|
) -> tuple[str, typing.Any, float]:
|
||||||
|
"""Time the execution of an asynchronous function."""
|
||||||
|
start_time = time()
|
||||||
|
result = await func(*args, **kwargs)
|
||||||
|
end_time = time()
|
||||||
|
return name, result, end_time - start_time
|
||||||
|
|
||||||
|
|
||||||
async def get_data(
|
async def get_data(
|
||||||
now: datetime, config: flask.config.Config
|
now: datetime, config: flask.config.Config
|
||||||
) -> typing.Mapping[str, str | object]:
|
) -> typing.Mapping[str, str | object]:
|
||||||
|
@ -369,28 +383,37 @@ async def get_data(
|
||||||
minus_365 = now - timedelta(days=365)
|
minus_365 = now - timedelta(days=365)
|
||||||
plus_365 = now + timedelta(days=365)
|
plus_365 = now + timedelta(days=365)
|
||||||
|
|
||||||
(
|
t0 = time()
|
||||||
gbpusd,
|
result_list = await asyncio.gather(
|
||||||
gwr_advance_tickets,
|
time_function("gbpusd", fx.get_gbpusd, config),
|
||||||
bank_holiday,
|
time_function("gwr_advance_tickets", gwr.advance_ticket_date, data_dir),
|
||||||
rockets,
|
time_function(
|
||||||
backwell_bins,
|
"bank_holiday", uk_holiday.bank_holiday_list, last_year, next_year, data_dir
|
||||||
bristol_bins,
|
),
|
||||||
) = await asyncio.gather(
|
time_function("rockets", thespacedevs.get_launches, rocket_dir, limit=40),
|
||||||
fx.get_gbpusd(config),
|
time_function("backwell_bins", waste_collection_events, data_dir),
|
||||||
gwr.advance_ticket_date(data_dir),
|
time_function("bristol_bins", bristol_waste_collection_events, data_dir, today),
|
||||||
uk_holiday.bank_holiday_list(last_year, next_year, data_dir),
|
|
||||||
thespacedevs.get_launches(rocket_dir, limit=40),
|
|
||||||
waste_collection_events(data_dir),
|
|
||||||
bristol_waste_collection_events(data_dir, today),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
results = {call[0]: call[1] for call in result_list}
|
||||||
|
|
||||||
|
gwr_advance_tickets = results["gwr_advance_tickets"]
|
||||||
|
|
||||||
|
data_gather_seconds = time() - t0
|
||||||
|
t0 = time()
|
||||||
|
|
||||||
|
stock_market_times = stock_market.open_and_close()
|
||||||
|
stock_market_times_seconds = time() - t0
|
||||||
|
|
||||||
reply: dict[str, typing.Any] = {
|
reply: dict[str, typing.Any] = {
|
||||||
"now": now,
|
"now": now,
|
||||||
"gbpusd": gbpusd,
|
"gbpusd": results["gbpusd"],
|
||||||
"stock_markets": stock_market.open_and_close(),
|
"stock_markets": stock_market_times,
|
||||||
"rockets": rockets,
|
"rockets": results["rockets"],
|
||||||
"gwr_advance_tickets": gwr_advance_tickets,
|
"gwr_advance_tickets": gwr_advance_tickets,
|
||||||
|
"data_gather_seconds": data_gather_seconds,
|
||||||
|
"stock_market_times_seconds": stock_market_times_seconds,
|
||||||
|
"timings": [(call[0], call[2]) for call in result_list],
|
||||||
}
|
}
|
||||||
|
|
||||||
my_data = config["PERSONAL_DATA"]
|
my_data = config["PERSONAL_DATA"]
|
||||||
|
@ -409,7 +432,7 @@ async def get_data(
|
||||||
|
|
||||||
us_hols = us_holidays(last_year, next_year)
|
us_hols = us_holidays(last_year, next_year)
|
||||||
|
|
||||||
holidays: list[Holiday] = bank_holiday + us_hols
|
holidays: list[Holiday] = results["bank_holiday"] + us_hols
|
||||||
for country in (
|
for country in (
|
||||||
"at",
|
"at",
|
||||||
"be",
|
"be",
|
||||||
|
@ -441,7 +464,7 @@ async def get_data(
|
||||||
events += accommodation_events
|
events += accommodation_events
|
||||||
events += travel.all_events(my_data)
|
events += travel.all_events(my_data)
|
||||||
events += conference.get_list(os.path.join(my_data, "conferences.yaml"))
|
events += conference.get_list(os.path.join(my_data, "conferences.yaml"))
|
||||||
events += backwell_bins + bristol_bins
|
events += results["backwell_bins"] + results["bristol_bins"]
|
||||||
events += read_events_yaml(my_data, last_year, next_year)
|
events += read_events_yaml(my_data, last_year, next_year)
|
||||||
events += subscription.get_events(os.path.join(my_data, "subscriptions.yaml"))
|
events += subscription.get_events(os.path.join(my_data, "subscriptions.yaml"))
|
||||||
events += economist.publication_dates(last_week, next_year)
|
events += economist.publication_dates(last_week, next_year)
|
||||||
|
@ -464,7 +487,7 @@ async def get_data(
|
||||||
for market in overlapping_markets:
|
for market in overlapping_markets:
|
||||||
events.remove(market)
|
events.remove(market)
|
||||||
|
|
||||||
for launch in rockets:
|
for launch in results["rockets"]:
|
||||||
dt = None
|
dt = None
|
||||||
|
|
||||||
if launch["net_precision"] == "Day":
|
if launch["net_precision"] == "Day":
|
||||||
|
|
|
@ -125,6 +125,12 @@
|
||||||
</li>
|
</li>
|
||||||
<li>Bristol Sunrise: {{ sunrise.strftime("%H:%M:%S") }} /
|
<li>Bristol Sunrise: {{ sunrise.strftime("%H:%M:%S") }} /
|
||||||
Sunset: {{ sunset.strftime("%H:%M:%S") }}</li>
|
Sunset: {{ sunset.strftime("%H:%M:%S") }}</li>
|
||||||
|
<li>Data gather took {{ "%.1f" | format(data_gather_seconds) }} seconds</li>
|
||||||
|
<li>Stock market open/close took
|
||||||
|
{{ "%.1f" | format(stock_market_times_seconds) }} seconds</li>
|
||||||
|
{% for name, seconds in timings %}
|
||||||
|
<li>{{ name }} took {{ "%.1f" | format(seconds) }} seconds</li>
|
||||||
|
{% endfor %}
|
||||||
</ul>
|
</ul>
|
||||||
|
|
||||||
<h3>Stock markets</h3>
|
<h3>Stock markets</h3>
|
||||||
|
|
Loading…
Reference in a new issue