#######################################################################################################################
# IMPORT LIBRARIES
#######################################################################################################################
from vois.vuetify import settings
settings.dark_mode = False
settings.color_first = '#68aad2'
settings.color_second = '#d8e7f5'
settings.button_rounded = False
import pandas as pd
from datetime import datetime
import io
from cairosvg import svg2png
from docx import Document
from docx.shared import Inches
from ipywidgets import widgets, Layout, HTML
from IPython.display import display
from ipyleaflet import basemaps
import ipyvuetify as v
from vois.vuetify import app, selectMultiple, label, datatable, toggle, tooltip, slider, switch, fab
from vois import svgMap, leafletMap, svgUtils, geojsonUtils
import EnergyConsumption
import plotly.express as px
import plotly.graph_objects as go
#######################################################################################################################
# Define subdivision of the app content
#######################################################################################################################
#border = '1px solid lightgrey'
border = 'none'
# Dimensioning
widthinpx = 260
widthControls = '%dpx' % widthinpx
heightinpx = 830
height = '%dpx' % heightinpx
height_net = '%dpx' % (heightinpx-10)
outControls = widgets.Output(layout=Layout(width=widthControls, min_width=widthControls, height=height, border=border))
outDisplay = widgets.Output(layout=Layout(width='90%', height=height, border=border))
widthmapinpx = 360
widthMapControls = '%dpx' % widthmapinpx
outMapControls = widgets.Output(layout=Layout(width=widthMapControls, min_width=widthMapControls, height=height_net, border=border))
outMap = widgets.Output(layout=Layout(width='90%', height=height_net, border=border))
widthanimpx = widthmapinpx - 20
widthanim = '%dpx' % (widthanimpx+10)
outAnimation = widgets.Output(layout=Layout(width=widthanim, min_width=widthanim, height=widthanim, border=border))
#######################################################################################################################
# Load data
#######################################################################################################################
g_df = EnergyConsumption.loadData()
#######################################################################################################################
# Global variables
#######################################################################################################################
g_minyear = int(g_df['TIME_PERIOD'].min())
g_maxyear = int(g_df['TIME_PERIOD'].max())
g_view = 0 # Current View: 0=Chart, 1=Table, 2=Static Map, 3=Dynamic Map
g_countries = [] # Selected countries codes
g_dtfiltered = None # Filtered dataframe
g_currentgeo = '' # list of comma-separated names of the selected countries
g_sector = 'FC_E'
g_units = 'Thousand tonnes of oil equivalent'
g_year = g_maxyear
g_usepop = False
g_map = None # Dynamic map
g_center = [56,8] # initial center of the dynamic map
g_zoom = 4 # initial zoom of the dynamic map
# Ordered list of sectors
g_sectors = ['FC_E', 'FC_IND_E', 'FC_TRA_E', 'FC_OTH_CP_E', 'FC_OTH_HH_E']
# Short names of the sectors
g_sectorTitle = {
'FC_E': 'Total',
'FC_IND_E': 'Industrial',
'FC_TRA_E': 'Transports',
'FC_OTH_CP_E': 'Commercial',
'FC_OTH_HH_E': 'Households',
}
# Long names of the sectors
g_sectorName = {
'FC_E': 'Total energy consumption',
'FC_IND_E': 'Industrial energy consumption',
'FC_TRA_E': 'Transports energy consumption',
'FC_OTH_CP_E': 'Commercial energy consumption',
'FC_OTH_HH_E': 'Households energy consumption',
}
# Last Plotly figure
g_last_fig = None
# Last SVG static map
g_last_svg = None
# Color sequence to use in the Plotly chart
g_colorsequence = px.colors.sequential.Blues[::-1]
#######################################################################################################################
# Create the controls
#######################################################################################################################
# Update the filtered dataframe
def UpdateDataframe():
global g_dtfiltered, g_currentgeo
# Filter dataset on country and sector
if len(g_countries) == 0:
codes = ['EU27_2020']
g_currentgeo = 'Europe27'
else:
codes = g_countries
g_currentgeo = ', '.join([EnergyConsumption.code2name[x] for x in g_countries])
g_dtfiltered = g_df[(g_df['geo'].isin(codes))&(g_df['nrg_bal']==g_sector)].copy()
g_dtfiltered.rename({'TIME_PERIOD': 'Year', 'OBS_VALUE': g_units}, axis=1, inplace=True)
# Mapping of country name to country code. If name is None returns code for EU27
def countries_mapping(name):
if name is None:
return 'EU27_2020'
else:
return EnergyConsumption.name2code[name]
# Mapping of country code to country name
def countries_reverse_mapping(code):
if code is 'EU27_2020':
return ''
else:
return EnergyConsumption.code2name[code]
# Selection of a country
def onchange_country():
global g_countries
g_countries = selcountry.value
UpdateDataframe()
displayCurrentView()
displayAnimation()
urlUpdate()
# Selection of a sector
def onchange_sector(value):
global g_sector
g_sector = g_sectors[value]
UpdateDataframe()
displayCurrentView()
urlUpdate()
labelSector = label.label('Sector:', textweight=450, height=26)
labelEmpty = label.label('', textweight=400, height=20)
selcountry = selectMultiple.selectMultiple('Country:', EnergyConsumption.eunames, width=widthinpx-30,
mapping=countries_mapping, reverse_mapping=countries_reverse_mapping,
onchange=onchange_country, marginy=2)
selsector = toggle.toggle(0, [g_sectorTitle[x] for x in g_sectors], tooltips=[g_sectorName[x] for x in g_sectors], onchange=onchange_sector, row=False, width=widthinpx-30)
outControls.clear_output()
with outControls:
display(selcountry.draw())
display(labelEmpty.draw())
display(labelSector.draw())
display(selsector.draw())
UpdateDataframe()
#######################################################################################################################
# Display filtered datatable in the outDisplay
#######################################################################################################################
def displayDatatable():
outDisplay.clear_output(wait=True)
d = datatable.datatable(data=g_dtfiltered, height=height_net)
with outDisplay:
display(d)
#######################################################################################################################
# Display Plotly Bar Chart in the outDisplay
#######################################################################################################################
def displayChart():
global g_last_fig
outDisplay.clear_output(wait=False)
with outDisplay:
title = g_sectorName[g_sector] + ' for ' + g_currentgeo
if len(g_countries) <= 1:
g_last_fig = px.bar(g_dtfiltered, x='Year', y=g_units, color="Country", template='plotly_white', text_auto=True, color_discrete_sequence=g_colorsequence)
g_last_fig.update_xaxes(tickvals=g_dtfiltered['Year'])
else:
g_last_fig = go.Figure()
i = 0
allyears = set()
for code in g_countries:
dfsel = g_dtfiltered[g_dtfiltered['geo']==code]
years = dfsel['Year'].unique()
allyears.update(years)
g_last_fig.add_trace(go.Bar(x=years, y=dfsel[g_units], name=EnergyConsumption.code2name[code], textposition="inside", texttemplate="%{y}", marker_color=g_colorsequence[i]))
i += 1
i = i % len(g_colorsequence)
g_last_fig.update_layout(barmode='group', template='plotly_white', legend_title='Country', xaxis_title="Year", yaxis_title="Thousand tonnes of oil equivalent")
g_last_fig.update_xaxes(tickvals=sorted(list(allyears)))
g_last_fig.update_layout(height=heightinpx-10, margin=dict(t=84, l=0, r=0, b=0), title={'text': title})
g_last_fig.show(config={'displaylogo': False, 'displayModeBar': False})
#######################################################################################################################
# Display Pie Chart animation
#######################################################################################################################
def displayAnimation():
outAnimation.clear_output(wait=True)
if len(g_countries) == 0: codes = ['EU27_2020']
else: codes = g_countries
df_country_year = g_df[(g_df['geo'].isin(codes))&(g_df['TIME_PERIOD']==g_year)]
df_country_year = df_country_year.groupby(["nrg_bal"])["OBS_VALUE"].sum().to_frame().reset_index()
sectors = list(df_country_year['nrg_bal'])
values = list(df_country_year['OBS_VALUE'])
chartvalues = []
chartlabels = []
chartsectors = []
if 'FC_E' in sectors:
totalindex = sectors.index('FC_E')
totalvalue = values[totalindex]
total = 0.0
for s,v in zip(sectors,values):
if s != 'FC_E':
total += v
chartvalues.append(round(v,2))
chartlabels.append(g_sectorTitle[s])
chartsectors.append(s)
chartvalues.append(round(totalvalue - total,2))
chartlabels.append('Other')
chartsectors.append(None)
def onclick(arg):
newsector = chartsectors[arg]
if not newsector is None:
g_sector = newsector
selsector.value = g_sectors.index(g_sector)
out, txt = svgUtils.AnimatedPieChart(values=chartvalues, labels=chartlabels, decimals=1,
centerfontsize=28, fontsize=16, textweight=500, colors=px.colors.sequential.Blues, backcolor='#e0e0e0',
centertext='Sector', onclick=onclick, dimension=widthanimpx-15, duration=1.0)
with outAnimation:
display(out)
#######################################################################################################################
# Controls on the map
#######################################################################################################################
# Selection of a year
def onchange_year(value):
global g_year
g_year = value
UpdateDataframe()
displayCurrentView()
displayAnimation()
urlUpdate()
labelYear = label.label('Select the Year:', textweight=400, height=26, margins=3, margintop=0)
sliderYear = slider.slider(g_year, g_minyear,g_maxyear, onchange=onchange_year)
labelPieChart = label.label('Subdivision by sector:', textweight=400, height=26, margins=3, margintop=10)
def on_popswitch_change(arg):
global g_usepop
g_usepop = arg
UpdateDataframe()
displayCurrentView()
urlUpdate()
popswitch = switch.switch(g_usepop, "Normalize by Population", onchange=on_popswitch_change)
# Display the Map controls
def displayMapControls():
outDisplay.clear_output(wait=True)
with outDisplay:
display(widgets.HBox([outMapControls,outMap]))
displayAnimation()
outMapControls.clear_output()
with outMapControls:
display(labelYear.draw())
display(sliderYear.draw())
display(v.Html(tag='div',children=[tooltip.tooltip('Display absolute values or values per 100K inhabitants',popswitch.draw())], style_="overflow: hidden"))
display(labelPieChart.draw())
display(outAnimation)
#######################################################################################################################
# Prepare the pandas dataframe for the Map display (returns a df)
#######################################################################################################################
def dataframeForMap():
dfmap = g_df[(g_df['TIME_PERIOD']==int(g_year))&(g_df['nrg_bal']==g_sector)].copy()
dfmap = dfmap[dfmap['geo'].isin(svgMap.country_codes)]
if g_usepop:
dfmap['value'] = 100000.0 * dfmap['OBS_VALUE'] / dfmap['Population2021']
else:
dfmap['value'] = dfmap['OBS_VALUE']
return dfmap
# Return the units to write in the map legends
def legendForUnits():
if g_usepop:
return 'KTOE per 100K inhabit.'
else:
return g_units
#######################################################################################################################
# Display Static Map in the outDisplay
#######################################################################################################################
def displayStaticMap():
global g_last_svg
# Prepare the pandas dataframe
dfmap = dataframeForMap()
# From lighter to darkest!
colorlist = g_colorsequence[::-1]
# Generate the map
selected = []
if g_countries == ['EU27_2020']: selected = []
else: selected = g_countries
g_last_svg = svgMap.svgMapEurope(dfmap, code_column='geo', value_column='value', codes_selected=selected, stroke_selected='red',
colorlist=colorlist, stdevnumber=2.0,
onhoverfill='#f8bd1a', width=1480-2*widthinpx, stroke_width=3.0, hoveronempty=False,
legendtitle=str(g_year) + ' ' + g_sectorName[g_sector], legendunits=legendForUnits())
# Display the map
outMap.clear_output(wait=True)
with outMap:
display(HTML(g_last_svg))
#######################################################################################################################
# Display Dynamic Map in the outDisplay
#######################################################################################################################
# Store center and zoom at each zoom or panning of the user
def map_on_bounds_changed(args):
global g_center, g_zoom
if not g_map is None:
g_center = g_map.center
g_zoom = g_map.zoom
# Display the dynamic map
def displayDynamicMap():
global g_map
# Prepare the pandas dataframe
dfmap = dataframeForMap()
# Change code from Greece (sob!)
dfmap['geo'].replace('EL','GR',inplace=True)
# From lighter to darkest!
colorlist = g_colorsequence[::-1]
selected = []
if g_countries == ['EU27_2020']: selected = []
else: selected = g_countries
selected = ['GR' if x=='EL' else x for x in selected]
height = '%dpx' % (heightinpx-20)
# Generate the map
g_map = leafletMap.geojsonMap(dfmap,
'./data/ne_50m_admin_0_countries.geojson',
'ISO_A2_EH',
code_column='geo',
value_column='value',
codes_selected=selected,
stroke_selected='red',
colorlist=colorlist,
stdevnumber=2.0,
stroke_width=0.6,
stroke='#010101',
width='70%',
height=height,
center=g_center,
zoom=g_zoom,
basemap=basemaps.Esri.WorldTopoMap,
style ={'opacity': 1, 'dashArray': '0', 'fillOpacity': 0.85},
hover_style={'opacity': 1, 'dashArray': '0', 'fillOpacity': 0.99})
g_map.observe(map_on_bounds_changed, names='bounds')
# Generate the legend
svg = svgUtils.graduatedLegend(dfmap, code_column='geo', value_column='value',
codes_selected=selected, stroke_selected='red',
colorlist=colorlist, stdevnumber=2.0,
legendtitle=str(g_year) + ' ' + g_sectorName[g_sector],
legendunits=legendForUnits(),
fontsize=15, width=310, height=heightinpx-60)
# Display the legend
outlegend = widgets.Output(layout=Layout(width='360px',height=height))
with outlegend:
display(HTML(svg))
# Display the map
outMap.clear_output(wait=True)
with outMap:
display(widgets.HBox([g_map,outlegend]))
#######################################################################################################################
# Display the current view in the outDisplay
#######################################################################################################################
def displayCurrentView():
if g_view == 0:
displayChart()
elif g_view == 1:
displayDatatable()
elif g_view == 2:
displayStaticMap()
elif g_view == 3:
displayDynamicMap()
#######################################################################################################################
# DOWNLOAD FAB
#######################################################################################################################
# Download Chart in PNG
def ondownloadCHART():
global g_view
if g_view != 0:
g_view = 0
g_app.setActiveTab(g_view)
on_click_tab(g_maintabs[g_view])
if not g_last_fig is None:
filename = 'Energy_' + datetime.today().strftime('%Y-%m-%d_%H-%M-%S')
png = g_last_fig.to_image(format="png", width=1800)
buf = io.BytesIO(png)
buf.seek(0)
barray = buf.read()
#with open("chart.png", "wb") as file:
# file.write(barray)
g_app.downloadBytes(barray,'%s.png' % filename)
# Download data in CSV format
def ondownloadCSV():
if not g_dtfiltered is None:
filename = 'Energy_' + datetime.today().strftime('%Y-%m-%d_%H-%M-%S')
df = g_dtfiltered.copy()
df = df.reset_index(drop=True)
buf = io.StringIO()
df.to_csv(buf)
buf.seek(0)
text = buf.getvalue()
g_app.downloadText(text,"%s.csv" % filename)
# Download Map in PNG
def ondownloadMAP():
global g_view
g_app.dialogWaitOpen(text='Please wait for Map export...')
if g_view != 2:
g_view = 2
g_app.setActiveTab(g_view)
on_click_tab(g_maintabs[g_view])
if not g_last_svg is None:
filename = 'Energy_' + datetime.today().strftime('%Y-%m-%d_%H-%M-%S')
svg_picture = g_last_svg
svg_picture = svg_picture.replace(' ','&nbsp;')
svg_picture = svg_picture.replace('style="font-family: Roboto;"','')
png = svg2png(bytestring=svg_picture, parent_width=1850, parent_height=600)
buf = io.BytesIO(png)
buf.seek(0)
barray = buf.read()
#with open("map.png", "wb") as file:
# file.write(barray)
g_app.downloadBytes(barray,'%s.png' % filename)
g_app.dialogWaitClose()
#######################################################################################################################
# Utility functions for adding images and tables to .DOCX reports
#######################################################################################################################
# Add a picture to the report document
def add_image(svg_picture, document):
svg_picture = svg_picture.replace(' ','&nbsp;')
svg_picture = svg_picture.replace('style="font-family: Roboto;"','')
svg_picture = svg_picture.replace('<br>',' ')
png = svg2png(bytestring=svg_picture, parent_width=1400, parent_height=600)
file = io.BytesIO(png)
document.add_picture(file, width=Inches(7.0))
# Add a pandas dataframe as a table to the report document
def add_table(df, document):
table = document.add_table(1, len(df.columns))
# Columns
heading_cells = table.rows[0].cells
for i in range(len(df.columns)):
heading_cells[i].text = df.columns[i]
heading_cells[i].paragraphs[0].runs[0].font.bold = True
# Rows
for i in range(df.shape[0]):
cells = table.add_row().cells
row = df.iloc[i]
for i in range(len(df.columns)):
cells[i].text = str(row[df.columns[i]])
#######################################################################################################################
# GENERATE A REPORT IN MICROSOFT WORD .DOCX FORMAT
#######################################################################################################################
def ondownloadREPORT():
global g_view
if g_view != 0:
g_view = 0
g_app.setActiveTab(g_view)
on_click_tab(g_maintabs[g_view])
g_app.dialogWaitOpen(text='Please wait for Word .docx report generation...')
document = Document()
document.add_heading('Report from the Energy consumption dashboard', 0)
# Add a table
document.add_heading('Filtered table:', level=2)
df = g_dtfiltered[['nrg_bal', 'siec', 'geo', 'Year', 'Thousand tonnes of oil equivalent', 'Country']]
add_table(df, document)
document.add_page_break()
# Add chart as an image
png = g_last_fig.to_image(format="png", width=1800)
imagetitle = str(g_year) + ' ' + g_sectorName[g_sector] + ' in ' + legendForUnits() + ':'
document.add_heading(imagetitle, level=2)
file = io.BytesIO(png)
document.add_picture(file, width=Inches(7.0))
# Add map as an image
if g_last_svg is None:
displayStaticMap()
if not g_last_svg is None:
imagetitle = str(g_year) + ' ' + g_sectorName[g_sector] + ' in ' + legendForUnits() + ':'
document.add_heading(imagetitle, level=2)
add_image(g_last_svg, document)
buf = io.BufferedRandom(io.BytesIO())
document.save(buf)
buf.seek(0)
barray = buf.read()
g_app.downloadBytes(barray,'Energy.docx')
g_app.dialogWaitClose()
#######################################################################################################################
# DEFINE THE APP
#######################################################################################################################
g_maintabs = ['Chart', 'Table', 'Static Map', 'Dynamic Map']
# Click on a tab of the title: change the current view
def on_click_tab(arg):
global g_view, g_center, g_zoom
if arg == g_maintabs[0]:
g_view = 0
elif arg == g_maintabs[1]:
g_view = 1
elif arg == g_maintabs[2]:
displayMapControls()
g_view = 2
else:
g_center = [56,8]
g_zoom = 4
displayMapControls()
g_view = 3
displayCurrentView()
urlUpdate()
# Click on the credits text
def on_click_credits():
g_app.snackbar('Credits')
# Click on the logo
def on_click_logo():
g_app.urlOpen('https://ec.europa.eu/info/index_en')
# Click on the footer buttons
def on_click_footer(arg):
g_app.snackbar(arg)
# Click on the footer minipanel
def on_click_minipanel(arg):
if arg == 0: ondownloadCHART()
elif arg == 1: ondownloadCSV()
elif arg == 2: ondownloadMAP()
else: ondownloadREPORT()
# Update the URL
def urlUpdate():
url = "?view=%d&countries=%s§or=%s&year=%d&usepop=%d" % (int(g_view), ",".join(g_countries), str(g_sector), int(g_year), int(g_usepop))
g_app.urlUpdate(url)
g_app = app.app(title='Energy consumption example dashboard',
titlecredits='Created by Unit I.3',
titlewidth='60%',
footercolor='#dfdfe4',
footercredits='Data',
footercreditstooltip='Eurostat - European Commission',
footercreditsurl='https://ec.europa.eu/eurostat/data/database',
titletabs=g_maintabs,
titletabsstile='font-weight: 700; font-size: 17px;',
titletabsactive=g_view,
titletabsactiveparameter='view',
dark=False,
backgroundimageurl='https://picsum.photos/id/293/1920/1080',
sidepaneltitle='Help',
sidepaneltext="Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
sidepanelcontent=[v.Icon(class_='pa-0 ma-0 ml-2', children=['mdi-help'])],
minipanelicons=['mdi-chart-bar', 'mdi-table-large', 'mdi-map-legend', 'mdi-file-chart-outline'],
minipaneltooltips=['Download Chart', 'Download Table', 'Download Map', 'Generate Report in Word .docx format'],
minipanellarge=True, minipanelopen=False,
onclickminipanel=on_click_minipanel,
onclicktab=on_click_tab,
onclickcredits=on_click_credits,
onclicklogo=on_click_logo,
onclickfooter=on_click_footer)
b = g_app.fab(left='96%', top='108px', items=['Download Chart', 'Download Table', 'Download Map'], onclick=[ondownloadCHART,ondownloadCSV,ondownloadMAP])
# Read the URL parameters
g_view = int( g_app.urlParameter('view', g_view))
g_sector = str( g_app.urlParameter('sector', g_sector))
g_year = int( g_app.urlParameter('year', g_year))
g_usepop = bool(g_app.urlParameter('usepop', g_usepop))
countries = str( g_app.urlParameter('countries', ''))
if len(countries) <= 0: g_countries = []
else: g_countries = countries.split(',')
# Test of parameter setting
#g_view = 3
#g_sector = 'FC_TRA_E'
#g_year = 2019
#g_usepop = True
#g_countries = ['IT']
# Set the interface elements to the values read from the URL parameters
g_app.setActiveTab(g_view)
on_click_tab(g_maintabs[g_view])
if g_sector in g_sectors:
selsector.value = g_sectors.index(g_sector)
sliderYear.value = g_year
popswitch.value = g_usepop
selcountry.value = g_countries
UpdateDataframe()
displayCurrentView()
# Display content
with g_app.outcontent:
display(widgets.HBox([outControls,outDisplay]))
g_app.show()