LukeParke/pypistats.org
1
0
Fork 0
mirror of https://github.com/LukeHagar/pypistats.org.git synced 2025-12-06 04:21:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

180 retention

Browse Source
This commit is contained in:
crflynn
2018-11-19 23:20:21 -05:00
parent cc3d589fa5
commit 41e79b3818
1 changed files with 8 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
pypistats/tasks/pypi.py
View File

@@ -22,7 +22,7 @@ SYSTEMS = ("Windows", "Linux", "Darwin")
PSQL_TABLES = ["overall", "python_major", "python_minor", "system"] PSQL_TABLES = ["overall", "python_major", "python_minor", "system"]
# Number of days to retain records # Number of days to retain records
MAX_RECORD_AGE = 200 MAX_RECORD_AGE = 180
def get_google_credentials(): def get_google_credentials():
@@ -307,7 +307,6 @@ def vacuum_analyze(env="dev"):
return results return results
def get_query(date): def get_query(date):
"""Get the query to execute against pypistats on bigquery.""" """Get the query to execute against pypistats on bigquery."""
return f""" return f"""
@@ -402,20 +401,20 @@ def etl():
"""Perform the stats download.""" """Perform the stats download."""
env = os.environ.get("ENV") env = os.environ.get("ENV")
date = str(datetime.date.today() - datetime.timedelta(days=1)) date = str(datetime.date.today() - datetime.timedelta(days=1))
results = { results = dict()
"downloads": get_daily_download_stats(env, date), results["purge"] = purge_old_data(env, date)
"__all__": update_all_package_stats(env, date), results["downloads"] = get_daily_download_stats(env, date)
"recent": update_recent_stats(env, date), results["__all__"] = update_all_package_stats(env, date)
"purge": purge_old_data(env, date), results["recent"] = update_recent_stats(env, date)
}
results["cleanup"] = vacuum_analyze(env) results["cleanup"] = vacuum_analyze(env)
return results return results
if __name__ == "__main__": if __name__ == "__main__":
date = "2018-10-23" date = "2018-11-19"
env = "prod" env = "prod"
print(date, env) print(date, env)
print(purge_old_data(env, date))
print(get_daily_download_stats(env, date)) print(get_daily_download_stats(env, date))
print(update_all_package_stats(env, date)) print(update_all_package_stats(env, date))
print(update_recent_stats(env, date)) print(update_recent_stats(env, date))