From 8826f42f5eda73b389669310c1392acc3e7773b2 Mon Sep 17 00:00:00 2001 From: crflynn Date: Thu, 6 Sep 2018 03:20:48 -0400 Subject: [PATCH] query-based fill charts --- pypistats/plots/data_base.json | 92 +++++++++----- pypistats/plots/plot_base.json | 226 ++++++++++++++++++++++----------- pypistats/run.py | 5 +- pypistats/views/general.py | 68 ++++++++-- 4 files changed, 279 insertions(+), 112 deletions(-) diff --git a/pypistats/plots/data_base.json b/pypistats/plots/data_base.json index bbf55bb..36c2793 100644 --- a/pypistats/plots/data_base.json +++ b/pypistats/plots/data_base.json @@ -1,32 +1,66 @@ { - "data": [ - { - "x": [ - "2017-05-01", - "2017-05-02", - "2017-05-03" - ], - "y": [ - "2", - "5", - "4" - ], - "name": "Retention", - "type": "scatter", - "mode": "lines+markers", - "connectgaps": true, - "marker": { - "symbol": "circle", - "line": { - "color": "#444", - "width": 1 - } - }, - "line": { - "shape": "linear", - "smoothing": 1, - "width": 2 - } + "downloads":{ + "data":[ + { + "x":[ + "2017-05-01", + "2017-05-02", + "2017-05-03" + ], + "y":[ + "2", + "5", + "4" + ], + "name":"Downloads", + "type":"scatter", + "mode":"lines+markers", + "connectgaps":true, + "marker":{ + "symbol":"circle", + "line":{ + "color":"#444", + "width":1 + } + }, + "line":{ + "shape":"linear", + "smoothing":1, + "width":2 + } + } + ] + }, + "percentages":{ + "data":[ + { + "x":[ + "2017-05-01", + "2017-05-02", + "2017-05-03" + ], + "y":[ + "2", + "5", + "4" + ], + "text":[ + "2", + "5", + "4" + ], + "hoverinfo": "x+text+name", + "name":"Downloads", + "type":"scatter", + "mode":"lines", + "connectgaps":false, + "line":{ + "shape":"linear", + "smoothing":1, + "width":2 + }, + "fill":"tonexty" + } + ] } - ] } diff --git a/pypistats/plots/plot_base.json b/pypistats/plots/plot_base.json index a279a42..80033e6 100644 --- a/pypistats/plots/plot_base.json +++ b/pypistats/plots/plot_base.json @@ -1,78 +1,156 @@ { - "layout": { - "autosize": true, - "height": 400, - "margin": { - "r": 100, - "t": 40, - "autoexpand": true, - "b": 80, - "l": 100, - "pad": 0 + "downloads":{ + "layout":{ + "autosize":true, + "height":400, + "margin":{ + "r":100, + "t":40, + "autoexpand":true, + "b":80, + "l":100, + "pad":0 + }, + "paper_bgcolor":"#fff", + "plot_bgcolor":"rgba(175, 175, 175, 0.2)", + "showlegend":true, + "legend":{ + "orientation":"v", + "bgcolor":"#e7e7e7", + "xanchor":"left", + "yanchor":"middle", + "x":0, + "y":0.5 + }, + "title":"Downloads", + "xaxis":{ + "tickformat":"%m-%d", + "dtick":604800000, + "tick0":"2017-08-07", + "gridcolor":"#FFF", + "gridwidth":2, + "anchor":"y", + "domain":[ + 0, + 1 + ], + "title":"Date", + "titlefont":{ + "family":"'Geneva', Verdana, Geneva, sans-serif", + "size":16, + "color":"#7f7f7f" + }, + "showline":true, + "linecolor":"rgba(148, 148, 148, 1)", + "linewidth":2, + "tickangle":-45 + }, + "yaxis":{ + "hoverformat":",.0", + "tickformat":",.0", + "gridcolor":"#FFF", + "gridwidth":2, + "autotick":true, + "rangemode":"tozero", + "showline":true, + "title":"Downloads", + "ticksuffix":"", + "tickmode":"auto", + "linecolor":"rgba(148, 148, 148, 1)", + "linewidth":2 + } + }, + "config":{ + "displaylogo":false, + "modeBarButtonsToRemove":[ + "toImage", + "sendDataToCloud", + "zoom2d", + "pan2d", + "select2d", + "lasso2d", + "zoomIn2d", + "zoomOut2d", + "toggleSpikelines" + ] + } }, - "paper_bgcolor": "#fff", - "plot_bgcolor": "rgba(175, 175, 175, 0.2)", - "showlegend": true, - "legend": { - "orientation": "v", - "bgcolor": "#e7e7e7", - "xanchor": "left", - "yanchor": "middle", - "x": 0, - "y": 0.5 - }, - "title": "Downloads", - "yaxis": { - }, - "xaxis": { - "tickformat": "%m-%d", - "dtick": 604800000, - "tick0": "2017-08-07", - "gridcolor": "#FFF", - "gridwidth": 2, - "anchor": "y", - "domain": [ - 0, - 1 - ], - "title": "Date", - "titlefont": { - "family": "'Geneva', Verdana, Geneva, sans-serif", - "size": 16, - "color": "#7f7f7f" - }, - "showline": true, - "linecolor": "rgba(148, 148, 148, 1)", - "linewidth": 2, - "tickangle": -45 - }, - "yaxis": { - "hoverformat": ",.0", - "tickformat": ",.0", - "gridcolor": "#FFF", - "gridwidth": 2, - "autotick": true, - "rangemode": "tozero", - "showline": true, - "title": "Downloads", - "ticksuffix": "", - "tickmode": "auto", - "linecolor": "rgba(148, 148, 148, 1)", - "linewidth": 2 + "percentages":{ + "layout":{ + "autosize":true, + "height":400, + "margin":{ + "r":100, + "t":40, + "autoexpand":true, + "b":80, + "l":100, + "pad":0 + }, + "paper_bgcolor":"#fff", + "plot_bgcolor":"rgba(175, 175, 175, 0.2)", + "showlegend":true, + "legend":{ + "orientation":"v", + "bgcolor":"#e7e7e7", + "xanchor":"left", + "yanchor":"middle", + "x":0, + "y":0.5 + }, + "title":"Proportional Downloads", + "xaxis":{ + "tickformat":"%m-%d", + "dtick":604800000, + "tick0":"2017-08-07", + "gridcolor":"#FFF", + "gridwidth":2, + "anchor":"y", + "domain":[ + 0, + 1 + ], + "title":"Date", + "titlefont":{ + "family":"'Geneva', Verdana, Geneva, sans-serif", + "size":16, + "color":"#7f7f7f" + }, + "showline":true, + "linecolor":"rgba(148, 148, 148, 1)", + "linewidth":2, + "tickangle":-45 + }, + "yaxis":{ + "range":[ + 0, + 100 + ], + "dtick":20, + "gridcolor":"#FFF", + "gridwidth":2, + "autotick":false, + "showline":true, + "title":"Proportional Downloads", + "ticksuffix":"%", + "tickmode":"auto", + "linecolor":"rgba(148, 148, 148, 1)", + "linewidth":2 + } + }, + "config":{ + "displaylogo":false, + "modeBarButtonsToRemove":[ + "toImage", + "sendDataToCloud", + "zoom2d", + "pan2d", + "select2d", + "lasso2d", + "zoomIn2d", + "zoomOut2d", + "toggleSpikelines" + ] + } } - }, - "config": { - "displaylogo": false, - "modeBarButtonsToRemove": [ - "toImage", - "sendDataToCloud", - "zoom2d", - "pan2d", - "select2d", - "lasso2d", - "zoomIn2d", - "zoomOut2d", - "toggleSpikelines" - ] - } } diff --git a/pypistats/run.py b/pypistats/run.py index 1d24b4c..f1e7792 100644 --- a/pypistats/run.py +++ b/pypistats/run.py @@ -7,6 +7,7 @@ from flask_sslify import SSLify from pypistats.application import create_app from pypistats.application import create_celery +from pypistats.extensions import db from pypistats.models.user import User from pypistats.settings import configs @@ -15,7 +16,7 @@ from pypistats.settings import configs env = os.environ.get("ENV", "dev") app = create_app(configs[env]) -sslify = SSLify(app) +# sslify = SSLify(app) celery = create_celery(app) app.logger.info(f"Environment: {env}") @@ -27,3 +28,5 @@ def before_request(): g.user = None if "user_id" in session: g.user = User.query.get(session["user_id"]) + if "db" not in g: + g.db = db diff --git a/pypistats/views/general.py b/pypistats/views/general.py index 3cbf5d6..185d901 100644 --- a/pypistats/views/general.py +++ b/pypistats/views/general.py @@ -1,4 +1,5 @@ """General pages.""" +from collections import defaultdict from copy import deepcopy import os import re @@ -12,6 +13,9 @@ from flask import redirect from flask import render_template from flask_wtf import FlaskForm import requests +from sqlalchemy import and_ +from sqlalchemy import func +from sqlalchemy.sql.expression import label from wtforms import StringField from wtforms.validators import DataRequired @@ -117,27 +121,35 @@ def package(package): # Get data from db model_data = [] for model in MODELS: - model_data.append({ - "name": model.__tablename__, - "data": get_download_data(package, model), - }) + if model == OverallDownloadCount: + metrics = ["downloads"] + else: + metrics = ["downloads", "percentages"] + + for metric in metrics: + model_data.append({ + "metric": metric, + "name": model.__tablename__, + "data": data_function[metric](package, model), + }) # Build the plots plots = [] for model in model_data: - plot = deepcopy(current_app.config["PLOT_BASE"]) + plot = deepcopy(current_app.config["PLOT_BASE"])[model["metric"]] data = [] for category, values in model["data"].items(): - base = deepcopy(current_app.config["DATA_BASE"]["data"][0]) + base = deepcopy(current_app.config["DATA_BASE"][model["metric"]]["data"][0]) base["x"] = values["x"] base["y"] = values["y"] + if model["metric"] == "percentages": + base["text"] = values["text"] base["name"] = category.title() data.append(base) plot["data"] = data plot["layout"]["title"] = \ f"Downloads of {package} package - {model['name'].title().replace('_', ' ')}" # noqa plots.append(plot) - return render_template( "package.html", package=package, @@ -147,7 +159,6 @@ def package(package): user=g.user ) - def get_download_data(package, model): """Get the download data for a package - model.""" records = model.query.filter_by(package=package).\ @@ -162,6 +173,47 @@ def get_download_data(package, model): data[category]["y"].append(record.downloads) return data +def get_proportion_data(package, model): + totals = g.db.session.query( + model.date, + model.package, + func.sum(model.downloads).label("totals") + ).filter_by(package=package).group_by(model.date, model.package).subquery() + + records = g.db.session.query( + model.date, + model.package, + model.category, + model.downloads, + label("percentages", 100.0 * model.downloads / totals.c.totals) + ).join( + totals, + and_( + model.date == totals.c.date, + model.package == totals.c.package + ) + ).order_by(model.category, model.date).all() + + data = {} + cumsum = defaultdict(float) + for record in records: + date = str(record.date) + category = record.category + if category not in data: + data[category] = {"x": [], "y": [], "text": []} + data[category]["x"].append(date) + value = getattr(record, "percentages") or 0 + cumsum[date] += value + data[category]["y"].append(cumsum[date]) + data[category]["text"].append("{0:.2f}%".format(value) + " = {:,}".format(record.downloads)) + return data + + +data_function = { + "downloads": get_download_data, + "percentages": get_proportion_data, +} + @blueprint.route("/top") def top():