koodiklinikka/palkkakysely
1
0
Fork 0
mirror of https://github.com/koodiklinikka/palkkakysely.git synced 2026-02-07 14:50:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge massage scripts (for templating), prettify HTML

Browse Source
This commit is contained in:
Aarni Koskela
2022-10-10 12:31:20 +03:00
parent 335cf15064
commit aa060b10a4
8 changed files with 144 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
Makefile
View File

@@ -10,7 +10,7 @@ export YEAR
.PHONY: $(DATA_DIR)/results.xlsx $(DATA_DIR)/results.tsv
all: all-data copy-raw-data copy-massaged-data static charts profiling
all: all-data copy-raw-data massage charts profiling
$(OUT_DIR):
mkdir -p $(OUT_DIR)
@@ -19,11 +19,8 @@ copy-raw-data: all-data $(OUT_DIR)
cp $(DATA_DIR)/results.xlsx $(OUT_DIR)/raw.xlsx
cp $(DATA_DIR)/results.tsv $(OUT_DIR)/raw.tsv
copy-massaged-data: all-data
python -m pulkka.copy_massaged_data
static: all-data
python -m pulkka.massage_templates
massage: all-data
python -m pulkka.massage_outputs
charts: all-data
python -m pulkka.generate_charts

19
pulkka/copy_massaged_data.py
View File

@@ -1,19 +0,0 @@
from pulkka.config import OUT_DIR
from pulkka.data_ingest import read_data
def main():
df = read_data()
df.to_html(OUT_DIR / "data.html", index=False, na_rep="")
df.to_csv(OUT_DIR / "data.csv", index=False)
df.to_excel(OUT_DIR / "data.xlsx", index=False)
df.to_json(
OUT_DIR / "data.json",
orient="records",
date_format="iso",
force_ascii=False,
)
if __name__ == "__main__":
main()

65
pulkka/massage_outputs.py Normal file
View File

@@ -0,0 +1,65 @@
import datetime
import glob
import io
import os
import jinja2
import numpy
import pandas
from pulkka.config import OUT_DIR, YEAR
from pulkka.data_ingest import read_data
TEMPLATE_DIR = os.path.realpath("./template")
def write_massaged_files(env, df):
with open(OUT_DIR / 'data.html', 'w') as f:
with io.StringIO() as s:
df.to_html(s, index=False, na_rep="", border=0)
table_html = s.getvalue()
f.write(env.get_template('_table.html').render(table_html=table_html, body_class="table-body"))
df.to_csv(OUT_DIR / "data.csv", index=False)
df.to_excel(OUT_DIR / "data.xlsx", index=False)
df.to_json(
OUT_DIR / "data.json",
orient="records",
date_format="iso",
force_ascii=False,
)
print("Wrote data files")
def render_statics(env):
for filename in glob.glob(os.path.join(TEMPLATE_DIR, "*.html")):
basename = os.path.relpath(filename, TEMPLATE_DIR)
if basename.startswith("_"):
continue
out_filename = OUT_DIR / basename
with open(filename, "r") as inf:
tpl: jinja2.Template = env.from_string(inf.read())
content = tpl.render(body_class="static-body")
with open(out_filename, "w") as outf:
outf.write(content)
print(filename, "=>", out_filename)
def main():
df = read_data()
env = jinja2.Environment(
autoescape=True,
loader=jinja2.FileSystemLoader(TEMPLATE_DIR),
)
env.globals.update({
"date": datetime.datetime.utcnow(),
"pd": pandas,
"np": numpy,
"df": df,
"year": YEAR,
})
render_statics(env)
write_massaged_files(env, df)
if __name__ == "__main__":
main()

35
pulkka/massage_templates.py
View File

@@ -1,35 +0,0 @@
import datetime
import glob
import os
import jinja2
import numpy
import pandas
from pulkka.config import OUT_DIR, YEAR
from pulkka.data_ingest import read_data
def main():
env = jinja2.Environment(
autoescape=True,
)
data = {
"date": datetime.datetime.utcnow(),
"pd": pandas,
"np": numpy,
"df": read_data(),
"year": YEAR,
}
for filename in glob.glob("template/*"):
out_filename = OUT_DIR / os.path.relpath(filename, "template")
with open(filename, "r") as inf:
tpl: jinja2.Template = env.from_string(inf.read())
content = tpl.render(data)
with open(out_filename, "w") as outf:
outf.write(content)
print(filename, "=>", out_filename)
if __name__ == "__main__":
main()

15
template/_base.html Normal file
View File

@@ -0,0 +1,15 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport"
content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<style>{% include "style.css" %}</style>
<title>{% block title %}Koodiklinikan palkkakysely, {{year}}{% endblock %}</title>
</head>
<body class="{{ body_class|default("") }}">
{% block body %}{% endblock %}
{% block footer %}{% endblock %}
</body>
</html>

5
template/_table.html Normal file
View File

@@ -0,0 +1,5 @@
{% extends "_base.html" %}
{% block title %}Koodiklinikan palkkakysely, {{ year }}{% endblock %}
{% block body %}
{{ table_html|safe }}
{% endblock %}

81
template/index.html
View File

@@ -1,44 +1,37 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport"
content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<link rel="stylesheet" href="style.css">
<title>Koodiklinikan palkkakysely, {{year}}</title>
</head>
<body>
<h1>Koodiklinikan palkkakysely, {{year}}</h1>
<h2>Tunnusluvut</h2>
<ul>
<li><i>n</i> = {{ df|length }}</li>
{% with num_kk = df[pd.to_numeric(df['Kuukausipalkka'], errors='coerce').notnull()]['Kuukausipalkka'] %}
<li>Keskimääräinen kuukausipalkka = {{ num_kk.mean()|round(0) }} €</li>
<li>Mediaanikuukausipalkka = {{ num_kk.median()|round(0) }} €</li>
{% endwith %}
{% with num_v = df[pd.to_numeric(df['Vuositulot'], errors='coerce').notnull()]['Vuositulot'] %}
<li>Keskimääräiset vuositulot = {{ num_v.mean()|round(0) }} €</li>
<li>Mediaanivuositulot = {{ num_v.median()|round(0) }} €</li>
{% endwith %}
</ul>
<h2>Työkalut</h2>
<ul>
<li><a href="charts.html">Kaaviot</a></li>
<li><a href="profiling_report.html">Lähdedatan analyysi</a></li>
<li><a href="/palkkakysely/analysaattori/?url=/palkkakysely/{{ year }}/data.json">Pivot-työkalu</a></li>
</ul>
<h2>Data</h2>
<ul>
<li><a href="data.csv">Lähdedata (CSV)</a></li>
<li><a href="data.html">Lähdedata (HTML)</a></li>
<li><a href="data.json">Lähdedata (JSON)</a></li>
<li><a href="data.xlsx">Lähdedata (XLSX)</a></li>
<li><a href="raw.tsv">Raakadata (TSV)</a></li>
<li><a href="raw.xlsx">Raakadata (XLSX)</a></li>
</ul>
<footer>
Generoitu {{ date }}
</footer>
</body>
</html>
{% extends "_base.html" %}
{% block title %}Koodiklinikan palkkakysely, {{ year }}{% endblock %}
{% block body %}
<h1>Koodiklinikan palkkakysely, {{ year }}</h1>
<h2>Tunnusluvut</h2>
<ul>
<li><i>n</i> = {{ df|length }}</li>
{% with num_kk = df[pd.to_numeric(df['Kuukausipalkka'], errors='coerce').notnull()]['Kuukausipalkka'] %}
<li>Keskimääräinen kuukausipalkka = {{ num_kk.mean()|round(0) }} €</li>
<li>Mediaanikuukausipalkka = {{ num_kk.median()|round(0) }} €</li>
{% endwith %}
{% with num_v = df[pd.to_numeric(df['Vuositulot'], errors='coerce').notnull()]['Vuositulot'] %}
<li>Keskimääräiset vuositulot = {{ num_v.mean()|round(0) }} €</li>
<li>Mediaanivuositulot = {{ num_v.median()|round(0) }} €</li>
{% endwith %}
</ul>
<h2>Työkalut</h2>
<ul>
<li><a href="charts.html">Kaaviot</a></li>
<li><a href="profiling_report.html">Lähdedatan analyysi</a></li>
<li><a href="/palkkakysely/analysaattori/?url=/palkkakysely/{{ year }}/data.json">Pivot-työkalu</a></li>
</ul>
<h2>Data</h2>
<ul>
<li><a href="data.csv">Lähdedata (CSV)</a></li>
<li><a href="data.html">Lähdedata (HTML)</a></li>
<li><a href="data.json">Lähdedata (JSON)</a></li>
<li><a href="data.xlsx">Lähdedata (XLSX)</a></li>
<li><a href="raw.tsv">Raakadata (TSV)</a></li>
<li><a href="raw.xlsx">Raakadata (XLSX)</a></li>
</ul>
{% endblock %}
{% block footer %}
<footer>
Generoitu {{ date }}
</footer>
{% endblock %}

21
template/style.css
View File

@@ -1,10 +1,27 @@
body {
max-width: 650px;
margin: 40px auto;
padding: 0 10px;
font: 18px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
}
body.static-body {
max-width: 650px;
margin: 40px auto;
}
body.table-body {
font-size: 14px;
line-height: 1.3;
}
body.table-body table {
border-collapse: collapse;
}
body.table-body td, body.table-body th {
padding: 0.3em;
border: 1px solid #999;
}
h1, h2, h3 {
line-height: 1.2
}