LukeParke/pypistats.dev
1
0
Fork 0
mirror of https://github.com/LukeHagar/pypistats.dev.git synced 2025-12-06 12:47:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

single docker container using supervisord

Browse Source
This commit is contained in:
crflynn
2018-04-24 00:35:01 -04:00
parent 3d3de6cf9f
commit 9cd1ebb436
11 changed files with 225 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.dockerignore
View File

@@ -6,7 +6,7 @@ Dockerfile
Pipfile Pipfile
Pipfile.lock Pipfile.lock
migrations/ migrations/
run.sh README.rst
docs/ docs/
tests/ tests/

13
Dockerfile
View File

@@ -1,8 +1,15 @@
FROM python:3.6-slim FROM python:3.6-slim
RUN apt-get update && apt-get install -y supervisor redis-server
ADD requirements.txt requirements.txt WORKDIR /app
RUN pip install -r requirements.txt
ADD . . ADD requirements.txt /app/requirements.txt
RUN pip install -r /app/requirements.txt
ADD . /app
EXPOSE 5000 EXPOSE 5000
ENV C_FORCE_ROOT=1
CMD /usr/bin/supervisord

12
Pipfile
View File

@@ -8,17 +8,17 @@ name = "pypi"
[packages] [packages]
google-cloud-bigquery = "*" google-cloud-bigquery = "*"
pandas = "*"
"psycopg2" = "*" "psycopg2" = "*"
flask = "*" Flask = "*"
github-flask = "*" GitHub-Flask = "*"
flask-sqlalchemy = "*" flask-sqlalchemy = "*"
flask-migrate = "*" Flask-Migrate = "*"
flask-login = "*" Flask-Login = "*"
flask-wtf = "*" Flask-WTF = "*"
celery = "*" celery = "*"
redis = "*" redis = "*"
[dev-packages] [dev-packages]
awsebcli = "*"

194
Pipfile.lock generated
View File

@@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "8fbdf096ac477c3d82ab03aa50f0238201297c7572dc9d7b5f709675e7f0e6bd" "sha256": "d75592aa3ae64c8aef81f5033cf78a5e9914d90ef8e35d876b1b2729f2ff9cc4"
}, },
"host-environment-markers": { "host-environment-markers": {
"implementation_name": "cpython", "implementation_name": "cpython",
@@ -202,54 +202,6 @@
], ],
"version": "==1.0" "version": "==1.0"
}, },
"numpy": {
"hashes": [
"sha256:719d914f564f35cce4dc103808f8297c807c9f0297ac183ed81ae8b5650e698e",
"sha256:0f6a5ed0cd7ab1da11f5c07a8ecada73fc55a70ef7bb6311a4109891341d7277",
"sha256:d0928076d9bd8a98de44e79b1abe50c1456e7abbb40af7ef58092086f1a6c729",
"sha256:d858423f5ed444d494b15c4cc90a206e1b8c31354c781ac7584da0d21c09c1c3",
"sha256:20cac3123d791e4bf8482a580d98d6b5969ba348b9d5364df791ba3a666b660d",
"sha256:528ce59ded2008f9e8543e0146acb3a98a9890da00adf8904b1e18c82099418b",
"sha256:56e392b7c738bd70e6f46cf48c8194d3d1dd4c5a59fae4b30c58bb6ef86e5233",
"sha256:99051e03b445117b26028623f1a487112ddf61a09a27e2d25e6bc07d37d94f25",
"sha256:768e777cc1ffdbf97c507f65975c8686ebafe0f3dc8925d02ac117acc4669ce9",
"sha256:675e0f23967ce71067d12b6944add505d5f0a251f819cfb44bdf8ee7072c090d",
"sha256:a958bf9d4834c72dee4f91a0476e7837b8a2966dc6fcfc42c421405f98d0da51",
"sha256:bb370120de6d26004358611441e07acda26840e41dfedc259d7f8cc613f96495",
"sha256:f2b1378b63bdb581d5d7af2ec0373c8d40d651941d283a2afd7fc71184b3f570",
"sha256:a1413d06abfa942ca0553bf3bccaff5fdb36d55b84f2248e36228db871147dab",
"sha256:7f76d406c6b998d6410198dcb82688dcdaec7d846aa87e263ccf52efdcfeba30",
"sha256:a7157c9ac6bddd2908c35ef099e4b643bc0e0ebb4d653deb54891d29258dd329",
"sha256:0fd65cbbfdbf76bbf80c445d923b3accefea0fe2c2082049e0ce947c81fe1d3f",
"sha256:8c18ee4dddd5c6a811930c0a7c7947bf16387da3b394725f6063f1366311187d",
"sha256:0739146eaf4985962f07c62f7133aca89f3a600faac891ce6c7f3a1e2afe5272",
"sha256:07e21f14490324cc1160db101e9b6c1233c33985af4cb1d301dd02650fea1d7f",
"sha256:e6120d63b50e2248219f53302af7ec6fa2a42ed1f37e9cda2c76dbaca65036a7",
"sha256:6be6b0ca705321c178c9858e5ad5611af664bbdfae1df1541f938a840a103888",
"sha256:facc6f925c3099ac01a1f03758100772560a0b020fb9d70f210404be08006bcb"
],
"version": "==1.14.2"
},
"pandas": {
"hashes": [
"sha256:68ac484e857dcbbd07ea7c6f516cc67f7f143f5313d9bc661470e7f473528882",
"sha256:12f2a19d0b0adf31170d98d0e8bcbc59add0965a9b0c65d39e0665400491c0c5",
"sha256:68b121d13177f5128a4c118bb4f73ba40df28292c038389961aa55ea5a996427",
"sha256:06efae5c00b9f4c6e6d3fe1eb52e590ff0ea8e5cb58032c724e04d31c540de53",
"sha256:02541a4fdd31315f213a5c8e18708abad719ee03eda05f603c4fe973e9b9d770",
"sha256:2907f3fe91ca2119ac3c38de6891bbbc83333bfe0d98309768fee28de563ee7a",
"sha256:052a66f58783a59ea38fdfee25de083b107baa81fdbe38fabd169d0f9efce2bf",
"sha256:244ae0b9e998cfa88452a49b20e29bf582cc7c0e69093876d505aec4f8e1c7fe",
"sha256:66403162c8b45325a995493bdd78ad4d8be085e527d721dbfa773d56fbba9c88",
"sha256:af0dbac881f6f87acd325415adea0ce8cccf28f5d4ad7a54b6a1e176e2f7bf70",
"sha256:c2cd884794924687edbaad40d18ac984054d247bb877890932c4d41e3c3aba31",
"sha256:c372db80a5bcb143c9cb254d50f902772c3b093a4f965275197ec2d2184b1e61",
"sha256:97c8223d42d43d86ca359a57b4702ca0529c6553e83d736e93a5699951f0f8db",
"sha256:587a9816cc663c958fcff7907c553b73fe196604f990bc98e1b71ebf07e45b44",
"sha256:44a94091dd71f05922eec661638ec1a35f26d573c119aa2fad964f10a2880e6c"
],
"version": "==0.22.0"
},
"protobuf": { "protobuf": {
"hashes": [ "hashes": [
"sha256:ac0067e3c60737865ed72bb7416e02297d229d960902802d874c0e167128c809", "sha256:ac0067e3c60737865ed72bb7416e02297d229d960902802d874c0e167128c809",
@@ -423,5 +375,147 @@
"version": "==2.1" "version": "==2.1"
} }
}, },
"develop": {} "develop": {
"awsebcli": {
"hashes": [
"sha256:86ca33bad0d20f4fcd3d967ede405d040080849675cd26491ef42df17d7f0d89"
],
"version": "==3.12.4"
},
"blessed": {
"hashes": [
"sha256:713dd9466a469cf26d5a26159c572d69a25ab71c8cb8e1bcd35a301c803978c7",
"sha256:2342125fd4f27f00d2677798bd06be2e6a1178e77c0298080abe4f720070693b"
],
"version": "==1.14.2"
},
"botocore": {
"hashes": [
"sha256:cacece8bda3dc6191f0b339d677987232a896af306fcf6fcc1ce2340c27a3365",
"sha256:db90e17685c53867b2a11b2b562d692257fd7388833ff38df5efa43b5d3065f5"
],
"version": "==1.10.6"
},
"cement": {
"hashes": [
"sha256:8765ed052c061d74e4d0189addc33d268de544ca219b259d797741f725e422d2"
],
"version": "==2.8.2"
},
"colorama": {
"hashes": [
"sha256:a4c0f5bc358a62849653471e309dcc991223cf86abafbec17cd8f41327279e89",
"sha256:e043c8d32527607223652021ff648fbb394d5e19cba9f1a698670b338c9d782b",
"sha256:f4945bf52ae49da0728fe730a33c18744803752fc948f154f29dc0c4f9f2f9cc"
],
"version": "==0.3.7"
},
"dockerpty": {
"hashes": [
"sha256:69a9d69d573a0daa31bcd1c0774eeed5c15c295fe719c61aca550ed1393156ce"
],
"version": "==0.4.1"
},
"docopt": {
"hashes": [
"sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"
],
"version": "==0.6.2"
},
"docutils": {
"hashes": [
"sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6",
"sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6",
"sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274"
],
"version": "==0.14"
},
"jmespath": {
"hashes": [
"sha256:f11b4461f425740a1d908e9a3f7365c3d2e569f6ca68a2ff8bc5bcd9676edd63",
"sha256:6a81d4c9aa62caf061cb517b4d9ad1dd300374cd4706997aff9cd6aedd61fc64"
],
"version": "==0.9.3"
},
"pathspec": {
"hashes": [
"sha256:72c495d1bbe76674219e307f6d1c6062f2e1b0b483a5e4886435127d0df3d0d3"
],
"version": "==0.5.5"
},
"python-dateutil": {
"hashes": [
"sha256:3220490fb9741e2342e1cf29a503394fdac874bc39568288717ee67047ff29df",
"sha256:9d8074be4c993fbe4947878ce593052f71dac82932a677d49194d8ce9778002e"
],
"markers": "python_version >= '2.7'",
"version": "==2.7.2"
},
"pyyaml": {
"hashes": [
"sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f",
"sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736",
"sha256:e863072cdf4c72eebf179342c94e6989c67185842d9997960b3e69290b2fa269",
"sha256:bc6bced57f826ca7cb5125a10b23fd0f2fff3b7c4701d64c439a300ce665fff8",
"sha256:c01b880ec30b5a6e6aa67b09a2fe3fb30473008c85cd6a67359a1b15ed6d83a4",
"sha256:827dc04b8fa7d07c44de11fabbc888e627fa8293b695e0f99cb544fdfa1bf0d1",
"sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab",
"sha256:5f84523c076ad14ff5e6c037fe1c89a7f73a3e04cf0377cb4d017014976433f3",
"sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8",
"sha256:b4c423ab23291d3945ac61346feeb9a0dc4184999ede5e7c43e1ffb975130ae6",
"sha256:ca233c64c6e40eaa6c66ef97058cdc80e8d0157a443655baa1b2966e812807ca",
"sha256:4474f8ea030b5127225b8894d626bb66c01cda098d47a2b0d3429b6700af9fd8",
"sha256:326420cbb492172dec84b0f65c80942de6cedb5233c413dd824483989c000608",
"sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7"
],
"version": "==3.12"
},
"requests": {
"hashes": [
"sha256:113fbba5531a9e34945b7d36b33a084e8ba5d0664b703c81a7c572d91919a5b8",
"sha256:c577815dd00f1394203fc44eb979724b098f88264a9ef898ee45b8e5e9cf587f"
],
"version": "==2.9.1"
},
"semantic-version": {
"hashes": [
"sha256:75aa594dc3c8000102a35b92e0e21db7b88e083bcb95d0499dea8494b549c3f2",
"sha256:3baad35dcb074a49419539cea6a33b484706b6c2dd03f05b67763eba4c1bb65c"
],
"version": "==2.5.0"
},
"six": {
"hashes": [
"sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb",
"sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9"
],
"version": "==1.11.0"
},
"tabulate": {
"hashes": [
"sha256:9071aacbd97a9a915096c1aaf0dc684ac2672904cd876db5904085d6dac9810e"
],
"version": "==0.7.5"
},
"termcolor": {
"hashes": [
"sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"
],
"version": "==1.1.0"
},
"wcwidth": {
"hashes": [
"sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c",
"sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e"
],
"version": "==0.1.7"
},
"websocket-client": {
"hashes": [
"sha256:188b68b14fdb2d8eb1a111f21b9ffd2dbf1dbc4e4c1d28cf2c37cdbf1dd1cae6",
"sha256:a453dc4dfa6e0db3d8fd7738a308a88effe6240c59f3226eb93e8f020c216149"
],
"version": "==0.47.0"
}
}
} }

37
docker-compose.yaml
View File

@@ -1,37 +0,0 @@
version: '3'
services:
redis:
image: 'redis:alpine'
restart: always
command: redis-server
ports:
- '6379:6379'
flask:
build: .
restart: always
env_file:
- pypistats/secret/prod.env
command: 'flask run --host=0.0.0.0'
ports:
- '5000:5000'
celery:
build: .
restart: always
env_file:
- pypistats/secret/prod.env
command: 'celery -A pypistats.run.celery worker -l info'
user: nobody
depends_on:
- redis
beat:
build: .
restart: always
env_file:
- pypistats/secret/prod.env
command: 'celery beat -A pypistats.run.celery -l info --pidfile='
depends_on:
- redis

93
pypistats/tasks/pypi.py
View File

@@ -1,30 +1,17 @@
"""Get the download stats for a specific day.""" """Get the download stats for a specific day."""
import datetime import datetime
import json
import time import time
import os import os
from google.auth.crypt._python_rsa import RSASigner from google.auth.crypt._python_rsa import RSASigner
from google.cloud import bigquery from google.cloud import bigquery
from google.oauth2.service_account import Credentials from google.oauth2.service_account import Credentials
import pandas as pd
import psycopg2 import psycopg2
from psycopg2.extras import execute_values from psycopg2.extras import execute_values
from pypistats.run import celery from pypistats.run import celery
# For local use.
def load_env_vars(env="dev"):
"""Load environment variables."""
local_path = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"secret",
f"env_vars_{env}.json")
for key, value in json.load(open(local_path, 'r')).items():
os.environ[key] = value
# Mirrors to disregard when considering downloads # Mirrors to disregard when considering downloads
MIRRORS = ("bandersnatch", "z3c.pypimirror", "Artifactory", "devpi") MIRRORS = ("bandersnatch", "z3c.pypimirror", "Artifactory", "devpi")
@@ -64,10 +51,7 @@ def get_google_credentials():
def get_daily_download_stats(env="dev", date=None): def get_daily_download_stats(env="dev", date=None):
"""Get daily download stats for pypi packages from BigQuery.""" """Get daily download stats for pypi packages from BigQuery."""
start = time.time() start = time.time()
if os.environ.get("ENV", None) is None:
load_env_vars(env)
else:
env = os.environ.get("ENV")
job_config = bigquery.QueryJobConfig() job_config = bigquery.QueryJobConfig()
credentials = get_google_credentials() credentials = get_google_credentials()
bq_client = bigquery.Client( bq_client = bigquery.Client(
@@ -86,57 +70,45 @@ def get_daily_download_stats(env="dev", date=None):
rows = list(iterator) rows = list(iterator)
print(len(rows), "rows from gbq") print(len(rows), "rows from gbq")
data = [] data = {}
for row in rows: for row in rows:
data.append(( if row["category_label"] not in data:
data[row["category_label"]] = []
data[row["category_label"]].append([
date, date,
row['package'], row["package"],
row['category_label'], row["category"],
row['category'], row["downloads"],
row['downloads'] ])
))
df = pd.DataFrame(data, columns=[ results = update_db(data, env)
"date",
"package",
"category_label",
"category",
"downloads",
])
results = update_db(df, env)
print("Elapsed: " + str(time.time() - start)) print("Elapsed: " + str(time.time() - start))
results["elapsed"] = time.time() - start results["elapsed"] = time.time() - start
return results return results
def update_db(df, env="dev"): def update_db(data, env="dev"):
"""Update the db with new data by table.""" """Update the db with new data by table."""
load_env_vars(env)
connection, cursor = get_connection_cursor(env) connection, cursor = get_connection_cursor(env)
df_groups = df.groupby("category_label")
success = {} success = {}
for category_label, df_category in df_groups: for category_label, rows in data.items():
table = category_label table = category_label
df_category = df_category[[
"date",
"package",
"category",
"downloads",
]]
success[table] = update_table( success[table] = update_table(
connection, cursor, table, df_category, date connection, cursor, table, rows, date
) )
return success return success
def update_table(connection, cursor, table, df, date): def update_table(connection, cursor, table, rows, date):
"""Update a table.""" """Update a table."""
print(table) print(table)
df = df.fillna("null")
for row in rows:
for idx, item in enumerate(row):
if item is None:
row[idx] = "null"
delete_query = \ delete_query = \
f"""DELETE FROM {table} f"""DELETE FROM {table}
@@ -144,12 +116,12 @@ def update_table(connection, cursor, table, df, date):
insert_query = \ insert_query = \
f"""INSERT INTO {table} (date, package, category, downloads) f"""INSERT INTO {table} (date, package, category, downloads)
VALUES %s""" VALUES %s"""
values = list(df.itertuples(index=False, name=None))
try: try:
print(delete_query) print(delete_query)
cursor.execute(delete_query) cursor.execute(delete_query)
print(insert_query) print(insert_query)
execute_values(cursor, insert_query, values) execute_values(cursor, insert_query, rows)
connection.commit() connection.commit()
return True return True
except psycopg2.IntegrityError as e: except psycopg2.IntegrityError as e:
@@ -165,11 +137,6 @@ def update_all_package_stats(env="dev", date=None):
if date is None: if date is None:
date = str(datetime.date.today() - datetime.timedelta(days=1)) date = str(datetime.date.today() - datetime.timedelta(days=1))
if os.environ.get("ENV", None) is None:
load_env_vars(env)
else:
env = os.environ.get("ENV")
connection, cursor = get_connection_cursor(env) connection, cursor = get_connection_cursor(env)
success = {} success = {}
@@ -210,11 +177,6 @@ def update_recent_stats(env="dev", date=None):
if date is None: if date is None:
date = str(datetime.date.today() - datetime.timedelta(days=1)) date = str(datetime.date.today() - datetime.timedelta(days=1))
if os.environ.get("ENV", None) is None:
load_env_vars(env)
else:
env = os.environ.get("ENV")
connection, cursor = get_connection_cursor(env) connection, cursor = get_connection_cursor(env)
downloads_table = "overall" downloads_table = "overall"
@@ -285,11 +247,6 @@ def purge_old_data(env="dev", date=None):
if date is None: if date is None:
date = str(datetime.date.today() - datetime.timedelta(days=1)) date = str(datetime.date.today() - datetime.timedelta(days=1))
if os.environ.get("ENV", None) is None:
load_env_vars(env)
else:
env = os.environ.get("ENV")
connection, cursor = get_connection_cursor(env) connection, cursor = get_connection_cursor(env)
date = datetime.datetime.strptime(date, '%Y-%m-%d') date = datetime.datetime.strptime(date, '%Y-%m-%d')
@@ -416,14 +373,8 @@ def etl():
return results return results
@celery.task
def test():
"""Test task for celery/beat."""
print("hello test.")
if __name__ == "__main__": if __name__ == "__main__":
date = "2018-04-19" date = "2018-04-23"
env = "prod" env = "prod"
print(date, env) print(date, env)
print(get_daily_download_stats(env, date)) print(get_daily_download_stats(env, date))

2
requirements.txt
View File

@@ -24,8 +24,6 @@ jinja2==2.10 --hash=sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c139
kombu==4.1.0 --hash=sha256:01f0da9fe222a2183345004243d1518c0fbe5875955f1b24842f2d9c65709ade --hash=sha256:4249d9dd9dbf1fcec471d1c2def20653c9310dd1a217272d77e4844f9d5273cb kombu==4.1.0 --hash=sha256:01f0da9fe222a2183345004243d1518c0fbe5875955f1b24842f2d9c65709ade --hash=sha256:4249d9dd9dbf1fcec471d1c2def20653c9310dd1a217272d77e4844f9d5273cb
mako==1.0.7 --hash=sha256:4e02fde57bd4abb5ec400181e4c314f56ac3e49ba4fb8b0d50bba18cb27d25ae mako==1.0.7 --hash=sha256:4e02fde57bd4abb5ec400181e4c314f56ac3e49ba4fb8b0d50bba18cb27d25ae
markupsafe==1.0 --hash=sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665 markupsafe==1.0 --hash=sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665
numpy==1.14.2 --hash=sha256:719d914f564f35cce4dc103808f8297c807c9f0297ac183ed81ae8b5650e698e --hash=sha256:0f6a5ed0cd7ab1da11f5c07a8ecada73fc55a70ef7bb6311a4109891341d7277 --hash=sha256:d0928076d9bd8a98de44e79b1abe50c1456e7abbb40af7ef58092086f1a6c729 --hash=sha256:d858423f5ed444d494b15c4cc90a206e1b8c31354c781ac7584da0d21c09c1c3 --hash=sha256:20cac3123d791e4bf8482a580d98d6b5969ba348b9d5364df791ba3a666b660d --hash=sha256:528ce59ded2008f9e8543e0146acb3a98a9890da00adf8904b1e18c82099418b --hash=sha256:56e392b7c738bd70e6f46cf48c8194d3d1dd4c5a59fae4b30c58bb6ef86e5233 --hash=sha256:99051e03b445117b26028623f1a487112ddf61a09a27e2d25e6bc07d37d94f25 --hash=sha256:768e777cc1ffdbf97c507f65975c8686ebafe0f3dc8925d02ac117acc4669ce9 --hash=sha256:675e0f23967ce71067d12b6944add505d5f0a251f819cfb44bdf8ee7072c090d --hash=sha256:a958bf9d4834c72dee4f91a0476e7837b8a2966dc6fcfc42c421405f98d0da51 --hash=sha256:bb370120de6d26004358611441e07acda26840e41dfedc259d7f8cc613f96495 --hash=sha256:f2b1378b63bdb581d5d7af2ec0373c8d40d651941d283a2afd7fc71184b3f570 --hash=sha256:a1413d06abfa942ca0553bf3bccaff5fdb36d55b84f2248e36228db871147dab --hash=sha256:7f76d406c6b998d6410198dcb82688dcdaec7d846aa87e263ccf52efdcfeba30 --hash=sha256:a7157c9ac6bddd2908c35ef099e4b643bc0e0ebb4d653deb54891d29258dd329 --hash=sha256:0fd65cbbfdbf76bbf80c445d923b3accefea0fe2c2082049e0ce947c81fe1d3f --hash=sha256:8c18ee4dddd5c6a811930c0a7c7947bf16387da3b394725f6063f1366311187d --hash=sha256:0739146eaf4985962f07c62f7133aca89f3a600faac891ce6c7f3a1e2afe5272 --hash=sha256:07e21f14490324cc1160db101e9b6c1233c33985af4cb1d301dd02650fea1d7f --hash=sha256:e6120d63b50e2248219f53302af7ec6fa2a42ed1f37e9cda2c76dbaca65036a7 --hash=sha256:6be6b0ca705321c178c9858e5ad5611af664bbdfae1df1541f938a840a103888 --hash=sha256:facc6f925c3099ac01a1f03758100772560a0b020fb9d70f210404be08006bcb
pandas==0.22.0 --hash=sha256:68ac484e857dcbbd07ea7c6f516cc67f7f143f5313d9bc661470e7f473528882 --hash=sha256:12f2a19d0b0adf31170d98d0e8bcbc59add0965a9b0c65d39e0665400491c0c5 --hash=sha256:68b121d13177f5128a4c118bb4f73ba40df28292c038389961aa55ea5a996427 --hash=sha256:06efae5c00b9f4c6e6d3fe1eb52e590ff0ea8e5cb58032c724e04d31c540de53 --hash=sha256:02541a4fdd31315f213a5c8e18708abad719ee03eda05f603c4fe973e9b9d770 --hash=sha256:2907f3fe91ca2119ac3c38de6891bbbc83333bfe0d98309768fee28de563ee7a --hash=sha256:052a66f58783a59ea38fdfee25de083b107baa81fdbe38fabd169d0f9efce2bf --hash=sha256:244ae0b9e998cfa88452a49b20e29bf582cc7c0e69093876d505aec4f8e1c7fe --hash=sha256:66403162c8b45325a995493bdd78ad4d8be085e527d721dbfa773d56fbba9c88 --hash=sha256:af0dbac881f6f87acd325415adea0ce8cccf28f5d4ad7a54b6a1e176e2f7bf70 --hash=sha256:c2cd884794924687edbaad40d18ac984054d247bb877890932c4d41e3c3aba31 --hash=sha256:c372db80a5bcb143c9cb254d50f902772c3b093a4f965275197ec2d2184b1e61 --hash=sha256:97c8223d42d43d86ca359a57b4702ca0529c6553e83d736e93a5699951f0f8db --hash=sha256:587a9816cc663c958fcff7907c553b73fe196604f990bc98e1b71ebf07e45b44 --hash=sha256:44a94091dd71f05922eec661638ec1a35f26d573c119aa2fad964f10a2880e6c
protobuf==3.5.2.post1 --hash=sha256:ac0067e3c60737865ed72bb7416e02297d229d960902802d874c0e167128c809 --hash=sha256:5c1c8f6a0a68a874e3beff89255959dd80fad45870e96c88944a1b81a22dd5f5 --hash=sha256:7c193e6964e752bd056735594826c5b03274ceb8f07349d3ae47d9766250ba96 --hash=sha256:bcfa99f5a82f5eaaf6e5cee5bfdca5a1670f5740aec1d93dae170645ed1a16b0 --hash=sha256:e269ab7a50bf0fa6fe6a88ea7dcc7a1079ae9450d9ab9b7730ac32916d55508b --hash=sha256:01ccd6d03449ae75b779fb5bf4ed62177d61afe3c5e6465ccf3f8b2e1a84afbe --hash=sha256:628a3bf0794a8b3cabb18db11eb67cc10e0cc6e5525d557ae7b682bb73fa2018 --hash=sha256:242e4c7ae565267a8bc8b92d707177f915607ea4bd73244bec6cbf4a49b96661 --hash=sha256:e7fd33a3474cbe18fd5b5620784a0fa21fcae3e402b1806e29c6b450c7f61706 --hash=sha256:cc94079ae6cbcea5ae194464a30f3223f075e06a0446f52bca9ddbeb6e9f412a --hash=sha256:7222d6616108b33ad6cbeff8117062a73c43cdc8fa8f64f6a322ebeb663e710e --hash=sha256:3f655e1f99c3e14d56ca900af1b9a4715b691319a295cc38939d7f77eabd5e7c --hash=sha256:76ef6ca3c50e4cfd044861586d5f1b352e0fe7f17f883df6c165bad5b4d0e10a --hash=sha256:560a38e692a69957a70ba0e5839aa67430efd63072bf91b0539dac19055694cd --hash=sha256:d5d9edfdc5a3a01d06062d677b121081629782edf0e05ca1be14f15bb947eeee --hash=sha256:869e12bcfb5759e683f53ec1dd6155b7be034065431da289f0cb4510040a0799 --hash=sha256:905414e5ea6cdb78d8730f66335755152b46685fcb9fc2f2134024e3ea9e8dcc --hash=sha256:adf716a89c9cc1891ead79a861c427071ef59172f0e11967b00565a9547b3bd0 --hash=sha256:1d92cc30b0b46cced33adde5853d920179eb5ea8eecdee9552502a7f29cc3f21 --hash=sha256:3b60685732bd0cbdc802dfcb6071efbcf5d927ce3127c13c33ea1a8efae3aa76 protobuf==3.5.2.post1 --hash=sha256:ac0067e3c60737865ed72bb7416e02297d229d960902802d874c0e167128c809 --hash=sha256:5c1c8f6a0a68a874e3beff89255959dd80fad45870e96c88944a1b81a22dd5f5 --hash=sha256:7c193e6964e752bd056735594826c5b03274ceb8f07349d3ae47d9766250ba96 --hash=sha256:bcfa99f5a82f5eaaf6e5cee5bfdca5a1670f5740aec1d93dae170645ed1a16b0 --hash=sha256:e269ab7a50bf0fa6fe6a88ea7dcc7a1079ae9450d9ab9b7730ac32916d55508b --hash=sha256:01ccd6d03449ae75b779fb5bf4ed62177d61afe3c5e6465ccf3f8b2e1a84afbe --hash=sha256:628a3bf0794a8b3cabb18db11eb67cc10e0cc6e5525d557ae7b682bb73fa2018 --hash=sha256:242e4c7ae565267a8bc8b92d707177f915607ea4bd73244bec6cbf4a49b96661 --hash=sha256:e7fd33a3474cbe18fd5b5620784a0fa21fcae3e402b1806e29c6b450c7f61706 --hash=sha256:cc94079ae6cbcea5ae194464a30f3223f075e06a0446f52bca9ddbeb6e9f412a --hash=sha256:7222d6616108b33ad6cbeff8117062a73c43cdc8fa8f64f6a322ebeb663e710e --hash=sha256:3f655e1f99c3e14d56ca900af1b9a4715b691319a295cc38939d7f77eabd5e7c --hash=sha256:76ef6ca3c50e4cfd044861586d5f1b352e0fe7f17f883df6c165bad5b4d0e10a --hash=sha256:560a38e692a69957a70ba0e5839aa67430efd63072bf91b0539dac19055694cd --hash=sha256:d5d9edfdc5a3a01d06062d677b121081629782edf0e05ca1be14f15bb947eeee --hash=sha256:869e12bcfb5759e683f53ec1dd6155b7be034065431da289f0cb4510040a0799 --hash=sha256:905414e5ea6cdb78d8730f66335755152b46685fcb9fc2f2134024e3ea9e8dcc --hash=sha256:adf716a89c9cc1891ead79a861c427071ef59172f0e11967b00565a9547b3bd0 --hash=sha256:1d92cc30b0b46cced33adde5853d920179eb5ea8eecdee9552502a7f29cc3f21 --hash=sha256:3b60685732bd0cbdc802dfcb6071efbcf5d927ce3127c13c33ea1a8efae3aa76
psycopg2==2.7.4 --hash=sha256:aeaba399254ca79c299d9fe6aa811d3c3eac61458dee10270de7f4e71c624998 --hash=sha256:1d90379d01d0dc50ae9b40c863933d87ff82d51dd7d52cea5d1cb7019afd72cd --hash=sha256:36030ca7f4b4519ee4f52a74edc4ec73c75abfb6ea1d80ac7480953d1c0aa3c3 --hash=sha256:7cbc3b21ce2f681ca9ad2d8c0901090b23a30c955e980ebf1006d41f37068a95 --hash=sha256:b178e0923c93393e16646155794521e063ec17b7cc9f943f15b7d4b39776ea2c --hash=sha256:fe6a7f87356116f5ea840c65b032af17deef0e1a5c34013a2962dd6f99b860dd --hash=sha256:6f302c486132f8dd11f143e919e236ea4467d53bf18c451cac577e6988ecbd05 --hash=sha256:888bba7841116e529f407f15c6d28fe3ef0760df8c45257442ec2f14f161c871 --hash=sha256:932a4c101af007cb3132b1f8a9ffef23386acc53dad46536dc5ba43a3235ae02 --hash=sha256:179c52eb870110a8c1b460c86d4f696d58510ea025602cd3f81453746fccb94f --hash=sha256:33f9e1032095e1436fa9ec424abcbd4c170da934fb70e391c5d78275d0307c75 --hash=sha256:092a80da1b052a181b6e6c765849c9b32d46c5dac3b81bf8c9b83e697f3cdbe8 --hash=sha256:f3d3a88128f0c219bdc5b2d9ccd496517199660cea021c560a3252116df91cbd --hash=sha256:19983b77ec1fc2a210092aa0333ee48811fd9fb5f194c6cd5b927ed409aea5f8 --hash=sha256:027ae518d0e3b8fff41990e598bc7774c3d08a3a20e9ecc0b59fb2aaaf152f7f --hash=sha256:363fbbf4189722fc46779be1fad2597e2c40b3f577dc618f353a46391cf5d235 --hash=sha256:d74cf9234ba76426add5e123449be08993a9b13ff434c6efa3a07caa305a619f --hash=sha256:32702e3bd8bfe12b36226ba9846ed9e22336fc4bd710039d594b36bd432ae255 --hash=sha256:8eb94c0625c529215b53c08fb4e461546e2f3fc96a49c13d5474b5ad7aeab6cf --hash=sha256:8ebba5314c609a05c6955e5773c7e0e57b8dd817e4f751f30de729be58fa5e78 --hash=sha256:27467fd5af1dcc0a82d72927113b8f92da8f44b2efbdb8906bd76face95b596d --hash=sha256:b68e89bb086a9476fa85298caab43f92d0a6af135a5f433d1f6b6d82cafa7b55 --hash=sha256:0b9851e798bae024ed1a2a6377a8dab4b8a128a56ed406f572f9f06194e4b275 --hash=sha256:733166464598c239323142c071fa4c9b91c14359176e5ae7e202db6bcc1d2eb5 --hash=sha256:ad75fe10bea19ad2188c5cb5fc4cdf53ee808d9b44578c94a3cd1e9fc2beb656 --hash=sha256:8966829cb0d21a08a3c5ac971a2eb67c3927ae27c247300a8476554cc0ce2ae8 --hash=sha256:8bf51191d60f6987482ef0cfe8511bbf4877a5aa7f313d7b488b53189cf26209 psycopg2==2.7.4 --hash=sha256:aeaba399254ca79c299d9fe6aa811d3c3eac61458dee10270de7f4e71c624998 --hash=sha256:1d90379d01d0dc50ae9b40c863933d87ff82d51dd7d52cea5d1cb7019afd72cd --hash=sha256:36030ca7f4b4519ee4f52a74edc4ec73c75abfb6ea1d80ac7480953d1c0aa3c3 --hash=sha256:7cbc3b21ce2f681ca9ad2d8c0901090b23a30c955e980ebf1006d41f37068a95 --hash=sha256:b178e0923c93393e16646155794521e063ec17b7cc9f943f15b7d4b39776ea2c --hash=sha256:fe6a7f87356116f5ea840c65b032af17deef0e1a5c34013a2962dd6f99b860dd --hash=sha256:6f302c486132f8dd11f143e919e236ea4467d53bf18c451cac577e6988ecbd05 --hash=sha256:888bba7841116e529f407f15c6d28fe3ef0760df8c45257442ec2f14f161c871 --hash=sha256:932a4c101af007cb3132b1f8a9ffef23386acc53dad46536dc5ba43a3235ae02 --hash=sha256:179c52eb870110a8c1b460c86d4f696d58510ea025602cd3f81453746fccb94f --hash=sha256:33f9e1032095e1436fa9ec424abcbd4c170da934fb70e391c5d78275d0307c75 --hash=sha256:092a80da1b052a181b6e6c765849c9b32d46c5dac3b81bf8c9b83e697f3cdbe8 --hash=sha256:f3d3a88128f0c219bdc5b2d9ccd496517199660cea021c560a3252116df91cbd --hash=sha256:19983b77ec1fc2a210092aa0333ee48811fd9fb5f194c6cd5b927ed409aea5f8 --hash=sha256:027ae518d0e3b8fff41990e598bc7774c3d08a3a20e9ecc0b59fb2aaaf152f7f --hash=sha256:363fbbf4189722fc46779be1fad2597e2c40b3f577dc618f353a46391cf5d235 --hash=sha256:d74cf9234ba76426add5e123449be08993a9b13ff434c6efa3a07caa305a619f --hash=sha256:32702e3bd8bfe12b36226ba9846ed9e22336fc4bd710039d594b36bd432ae255 --hash=sha256:8eb94c0625c529215b53c08fb4e461546e2f3fc96a49c13d5474b5ad7aeab6cf --hash=sha256:8ebba5314c609a05c6955e5773c7e0e57b8dd817e4f751f30de729be58fa5e78 --hash=sha256:27467fd5af1dcc0a82d72927113b8f92da8f44b2efbdb8906bd76face95b596d --hash=sha256:b68e89bb086a9476fa85298caab43f92d0a6af135a5f433d1f6b6d82cafa7b55 --hash=sha256:0b9851e798bae024ed1a2a6377a8dab4b8a128a56ed406f572f9f06194e4b275 --hash=sha256:733166464598c239323142c071fa4c9b91c14359176e5ae7e202db6bcc1d2eb5 --hash=sha256:ad75fe10bea19ad2188c5cb5fc4cdf53ee808d9b44578c94a3cd1e9fc2beb656 --hash=sha256:8966829cb0d21a08a3c5ac971a2eb67c3927ae27c247300a8476554cc0ce2ae8 --hash=sha256:8bf51191d60f6987482ef0cfe8511bbf4877a5aa7f313d7b488b53189cf26209
pyasn1==0.4.2 --hash=sha256:f81c96761fca60d64b1c9b79ec2e40cf9495a745cf570613079ef324aeb9672b --hash=sha256:7d626683e3d792cccc608da02498aff37ab4f3dafd8905d6bf755d11f9b26b43 --hash=sha256:e85895087905c65b5b594eb91f7522664c85545b147d5f4d4e7b1b07da8dcbdc --hash=sha256:5a0db897b311d265cde49615cf783f1c78613138605cdd0f907ecfa5b2aba3ee --hash=sha256:d5cd6ed995dba16fad0c521cfe31cd2d68400b53fcc2bce93326829be73ab6d1 --hash=sha256:a7efe807c4b83a859e2735c692b92ed7b567cfddc4163763412920041d876c2b --hash=sha256:b5a9ca48055b9a20f6d1b3d68e38692e5431c86a0f99ea602e61294e891fee5b --hash=sha256:c07d6e587b2f928366b1f67c09bda026a3e6fcc99e80a744dc67f8fca3895626 --hash=sha256:d84c2aea3cf43780e9e6a19f4e4dddee9f6976519020e64e47c57e5c7a8c3dd2 --hash=sha256:758cb50abddc03e4563fd9e7f03db56e3e87b58c0bd01247360326e5c0c7ffa5 --hash=sha256:0d7f6e959fe53f3960a23d73f35e1fce61348b30915b6664309ca756de7c1f89 --hash=sha256:d258b0a71994f7770599835249cece1caef3c70def868c4915e6e5ca49b67d15 pyasn1==0.4.2 --hash=sha256:f81c96761fca60d64b1c9b79ec2e40cf9495a745cf570613079ef324aeb9672b --hash=sha256:7d626683e3d792cccc608da02498aff37ab4f3dafd8905d6bf755d11f9b26b43 --hash=sha256:e85895087905c65b5b594eb91f7522664c85545b147d5f4d4e7b1b07da8dcbdc --hash=sha256:5a0db897b311d265cde49615cf783f1c78613138605cdd0f907ecfa5b2aba3ee --hash=sha256:d5cd6ed995dba16fad0c521cfe31cd2d68400b53fcc2bce93326829be73ab6d1 --hash=sha256:a7efe807c4b83a859e2735c692b92ed7b567cfddc4163763412920041d876c2b --hash=sha256:b5a9ca48055b9a20f6d1b3d68e38692e5431c86a0f99ea602e61294e891fee5b --hash=sha256:c07d6e587b2f928366b1f67c09bda026a3e6fcc99e80a744dc67f8fca3895626 --hash=sha256:d84c2aea3cf43780e9e6a19f4e4dddee9f6976519020e64e47c57e5c7a8c3dd2 --hash=sha256:758cb50abddc03e4563fd9e7f03db56e3e87b58c0bd01247360326e5c0c7ffa5 --hash=sha256:0d7f6e959fe53f3960a23d73f35e1fce61348b30915b6664309ca756de7c1f89 --hash=sha256:d258b0a71994f7770599835249cece1caef3c70def868c4915e6e5ca49b67d15

5
run_beat.sh Executable file
View File

@@ -0,0 +1,5 @@
export ENV=prod
set -o allexport
source pypistats/secret/$ENV.env
set +o allexport
celery beat -A pypistats.run.celery -l info

5
run_celery.sh Executable file
View File

@@ -0,0 +1,5 @@
export ENV=prod
set -o allexport
source pypistats/secret/$ENV.env
set +o allexport
celery -A pypistats.run.celery worker -l info

1
run.sh → run_flask.sh
View File

@@ -2,4 +2,5 @@ export ENV=prod
set -o allexport set -o allexport
source pypistats/secret/$ENV.env source pypistats/secret/$ENV.env
set +o allexport set +o allexport
# python -m pypistats.tasks.pypi
flask run --host=0.0.0.0 flask run --host=0.0.0.0

31
supervisord.conf Normal file
View File

@@ -0,0 +1,31 @@
[supervisord]
nodaemon=true
[program:redis]
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
command=redis-server
[program:pypistats]
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
command=bash -c "./run_flask.sh"
[program:celery-worker]
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
user=nobody
command=bash -c "./run_celery.sh"
[program:celery-beat]
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
command=bash -c "./run_beat.sh"