diff --git a/Pipfile b/Pipfile index 07cfd2f..e92ac96 100644 --- a/Pipfile +++ b/Pipfile @@ -16,6 +16,7 @@ flask-sqlalchemy = "*" flask-migrate = "*" flask-login = "*" flask-wtf = "*" +zappa = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 01c09ce..f5c846a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f6427c3191e0004b972c855a3c74155aae118b1f0228e77b839ec4aa2c5a1141" + "sha256": "5330c2fa770e92623fdbbbc38ef22814315904c457993c6a34c993f34ab3fea8" }, "host-environment-markers": { "implementation_name": "cpython", @@ -33,6 +33,34 @@ ], "version": "==0.9.9" }, + "argcomplete": { + "hashes": [ + "sha256:74e34bbd5bcb902e67a39e2edf1b1ef3dcb504a6a1d4cd23ce3949f25c4aad55", + "sha256:d6ea272a93bb0387f758def836e73c36fff0c54170258c212de3e84f7db8d5ed" + ], + "version": "==1.9.2" + }, + "base58": { + "hashes": [ + "sha256:aafdd84adf1fbd073eb2fd7f7266d7bd9f2423ee1000ee61cd021b97ed359f39", + "sha256:97cb4dcbc7a81afb802f41033d5562b6c48633426a67bf41e4cad186f581158c" + ], + "version": "==0.2.4" + }, + "boto3": { + "hashes": [ + "sha256:4c746d55fb6294c11e78db76648c7f321f8068dc5ab402456b5e3e4ee52423b9", + "sha256:5c02372447e07bd67f7c19624642dcf5c7aabe126e1c44ff4ad241aeb97fc729" + ], + "version": "==1.7.2" + }, + "botocore": { + "hashes": [ + "sha256:8ded801591ef5df04244dc1ba2496dd04a9abbd165d0d2ad501b6cd4b34946d4", + "sha256:27945af4bfb2a1ff1f11c730d24b84da6e1f40465907029e8980903f3b984070" + ], + "version": "==1.10.2" + }, "cachetools": { "hashes": [ "sha256:4319bbb78172e7bcf99423e1ecd6914b32336ccfe97d2058ffe62e641a7f3abe", @@ -47,6 +75,12 @@ ], "version": "==2018.1.18" }, + "cfn-flip": { + "hashes": [ + "sha256:9c61039c71995ab204c005ec46d47d0f7a109e9f1b6d63569397f8bc648a8151" + ], + "version": "==1.0.3" + }, "chardet": { "hashes": [ "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691", @@ -61,6 +95,20 @@ ], "version": "==6.7" }, + "docutils": { + "hashes": [ + "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6", + "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", + "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274" + ], + "version": "==0.14" + }, + "durationpy": { + "hashes": [ + "sha256:5ef9416b527b50d722f34655becfb75e49228eb82f87b855ed1911b3314b5408" + ], + "version": "==0.5" + }, "flask": { "hashes": [ "sha256:0749df235e3ff61ac108f69ac178c9770caeaccad2509cb762ce1f65570a8856", @@ -95,6 +143,12 @@ ], "version": "==0.14.2" }, + "future": { + "hashes": [ + "sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb" + ], + "version": "==0.16.0" + }, "github-flask": { "hashes": [ "sha256:24600b720f698bac10667b76b136995ba7821d884e58b27e2a18ca0e4760c786" @@ -142,6 +196,12 @@ ], "version": "==1.5.3" }, + "hjson": { + "hashes": [ + "sha256:1d1727faa6aaef2973921877125a3ab7c5f6d34b93233179d01770f41fab51f9" + ], + "version": "==3.0.1" + }, "idna": { "hashes": [ "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4", @@ -162,6 +222,26 @@ ], "version": "==2.10" }, + "jmespath": { + "hashes": [ + "sha256:f11b4461f425740a1d908e9a3f7365c3d2e569f6ca68a2ff8bc5bcd9676edd63", + "sha256:6a81d4c9aa62caf061cb517b4d9ad1dd300374cd4706997aff9cd6aedd61fc64" + ], + "version": "==0.9.3" + }, + "kappa": { + "hashes": [ + "sha256:4d6b7b3accce4a0aaaac92b36237a6304f0f2fffbbe3caea3f7c9f52d12c9989", + "sha256:4b5b372872f25d619e427e04282551048dc975a107385b076b3ffc6406a15833" + ], + "version": "==0.6.0" + }, + "lambda-packages": { + "hashes": [ + "sha256:cbe35f0642206a4adfb4855f80d55e273bdacc083011eba5f9ed3beedf0879fa" + ], + "version": "==0.19.0" + }, "mako": { "hashes": [ "sha256:4e02fde57bd4abb5ec400181e4c314f56ac3e49ba4fb8b0d50bba18cb27d25ae" @@ -222,6 +302,12 @@ ], "version": "==0.22.0" }, + "placebo": { + "hashes": [ + "sha256:8aa848b892924786fa5e37e75524e8ec039b7d54860d35c51ffb4ed3e30590c5" + ], + "version": "==0.8.1" + }, "protobuf": { "hashes": [ "sha256:ac0067e3c60737865ed72bb7416e02297d229d960902802d874c0e167128c809", @@ -315,10 +401,10 @@ }, "python-dateutil": { "hashes": [ - "sha256:3220490fb9741e2342e1cf29a503394fdac874bc39568288717ee67047ff29df", - "sha256:9d8074be4c993fbe4947878ce593052f71dac82932a677d49194d8ce9778002e" + "sha256:95511bae634d69bc7329ba55e646499a842bc4ec342ad54a8cdb65645a0aad3c", + "sha256:891c38b2a02f5bb1be3e4793866c8df49c7d19baabf9c1bad62547e0b4866aca" ], - "version": "==2.7.2" + "version": "==2.6.1" }, "python-editor": { "hashes": [ @@ -326,6 +412,13 @@ ], "version": "==1.0.3" }, + "python-slugify": { + "hashes": [ + "sha256:c3733135d3b184196fdb8844f6a74bbfb9cf6720d1dcce3254bdc434353f938f", + "sha256:57a385df7a1c6dbd15f7666eaff0ff29d3f60363b228b1197c5308ed3ba5f824" + ], + "version": "==1.2.4" + }, "pytz": { "hashes": [ "sha256:ed6509d9af298b7995d69a440e2822288f2eca1681b8cce37673dbb10091e5fe", @@ -340,6 +433,25 @@ ], "version": "==2018.3" }, + "pyyaml": { + "hashes": [ + "sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f", + "sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736", + "sha256:e863072cdf4c72eebf179342c94e6989c67185842d9997960b3e69290b2fa269", + "sha256:bc6bced57f826ca7cb5125a10b23fd0f2fff3b7c4701d64c439a300ce665fff8", + "sha256:c01b880ec30b5a6e6aa67b09a2fe3fb30473008c85cd6a67359a1b15ed6d83a4", + "sha256:827dc04b8fa7d07c44de11fabbc888e627fa8293b695e0f99cb544fdfa1bf0d1", + "sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab", + "sha256:5f84523c076ad14ff5e6c037fe1c89a7f73a3e04cf0377cb4d017014976433f3", + "sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8", + "sha256:b4c423ab23291d3945ac61346feeb9a0dc4184999ede5e7c43e1ffb975130ae6", + "sha256:ca233c64c6e40eaa6c66ef97058cdc80e8d0157a443655baa1b2966e812807ca", + "sha256:4474f8ea030b5127225b8894d626bb66c01cda098d47a2b0d3429b6700af9fd8", + "sha256:326420cbb492172dec84b0f65c80942de6cedb5233c413dd824483989c000608", + "sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7" + ], + "version": "==3.12" + }, "requests": { "hashes": [ "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b", @@ -354,6 +466,13 @@ ], "version": "==3.4.2" }, + "s3transfer": { + "hashes": [ + "sha256:c7a9ec356982d5e9ab2d4b46391a7d6a950e2b04c472419f5fdec70cc0ada72f", + "sha256:90dc18e028989c609146e241ea153250be451e05ecc0c2832565231dacdf59c1" + ], + "version": "==0.1.13" + }, "six": { "hashes": [ "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb", @@ -367,6 +486,32 @@ ], "version": "==1.2.6" }, + "toml": { + "hashes": [ + "sha256:8e86bd6ce8cc11b9620cb637466453d94f5d57ad86f17e98a98d1f73e3baab2d" + ], + "version": "==0.9.4" + }, + "tqdm": { + "hashes": [ + "sha256:ba650e08b8b102923a05896bf9d7e1c9cdc20b484156df0511a4bbf1f6b6f89b", + "sha256:fa6d2ea6285f56e75d7efe9259805deadc450f16066a1f82ad0629ea9be2cd0f" + ], + "version": "==4.19.1" + }, + "troposphere": { + "hashes": [ + "sha256:e15e2470fe4f02a5c1b70ccd552b09ac664acfff0de74cd4464acf8e6039e118" + ], + "version": "==2.2.1" + }, + "unidecode": { + "hashes": [ + "sha256:72f49d3729f3d8f5799f710b97c1451c5163102e76d64d20e170aedbbd923582", + "sha256:8c33dd588e0c9bc22a76eaa0c715a5434851f726131bd44a6c26471746efabf5" + ], + "version": "==1.0.22" + }, "urllib3": { "hashes": [ "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b", @@ -376,16 +521,36 @@ }, "werkzeug": { "hashes": [ - "sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b", - "sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c" + "sha256:7fce5d96278e9e4132fd0e90ff17d2bcc6e74695d11e619ba4645c0341396810", + "sha256:f007848ed997101cb5c09a47e46c0b0b6f193d0f8a01cd2af920d77bf1ab4e68" ], - "version": "==0.14.1" + "version": "==0.12" + }, + "wheel": { + "hashes": [ + "sha256:9cdc8ab2cc9c3c2e2727a4b67c22881dbb0e1c503d592992594c5e131c867107", + "sha256:1ae8153bed701cb062913b72429bcf854ba824f973735427681882a688cb55ce" + ], + "version": "==0.31.0" + }, + "wsgi-request-logger": { + "hashes": [ + "sha256:445d7ec52799562f812006394d0b4a7064b37084c6ea6bd74ea7a2136c97ed83" + ], + "version": "==0.4.6" }, "wtforms": { "hashes": [ "sha256:ffdf10bd1fa565b8233380cb77a304cd36fd55c73023e91d4b803c96bc11d46f" ], "version": "==2.1" + }, + "zappa": { + "hashes": [ + "sha256:cf4654dfa7cd8777403ea2e379139b126c7e47943edb4e5441bb222a555a1aff", + "sha256:827c2c1cc84790674476a98773601b924a2b4f3b0b81684571cd034326231158" + ], + "version": "==0.45.1" } }, "develop": {} diff --git a/pypistats/tasks/pypi.py b/pypistats/tasks/pypi.py index 3fa1514..715fbf6 100644 --- a/pypistats/tasks/pypi.py +++ b/pypistats/tasks/pypi.py @@ -38,6 +38,9 @@ SCHEMA = [ # postgresql tables to update for __all__ PSQL_TABLES = ["overall", "python_major", "python_minor", "system"] +# Number of days to retain records +MAX_RECORD_AGE = 45 + def get_daily_download_stats(date, env="dev"): """Get daily download stats for pypi packages from BigQuery.""" @@ -243,6 +246,28 @@ def get_connection_cursor(env): return connection, cursor +def purge_old_data(date, env="dev", age=MAX_RECORD_AGE): + """Purge old data records.""" + connection, cursor = get_connection_cursor(env) + + date = datetime.datetime.strptime(date, '%Y-%m-%d') + purge_date = date - datetime.timedelta(days=age) + purge_date = purge_date.strftime('%Y-%m-%d') + + success = {} + for table in PSQL_TABLES: + delete_query = f"""DELETE FROM {table} where date < '{purge_date}'""" + try: + cursor.execute(delete_query) + connection.commit() + success[table] = True + except psycopg2.IntegrityError as e: + connection.rollback() + success[table] = False + + return success + + def get_query(date): """Get the query to execute against pypistats on bigquery.""" return f"""