usage-statistics/dist/collectors/pypi.js

/**
 * PyPI package statistics collector using external PyPI Stats API
 */
const PlatformSettings = {
    name: 'PyPI',
};
// (no BigQuery historical metrics; all data comes from the external API)
// External PyPI Stats API base URL
const PYPI_STATS_BASE_URL = process.env.PYPI_STATS_BASE_URL || 'https://pypistats.dev';
function normalizePackageName(name) {
    return name.replace(/[._]/g, '-').toLowerCase();
}
async function fetchJson(url) {
    const res = await fetch(url);
    if (!res.ok)
        throw new Error(`Request failed ${res.status}: ${url}`);
    return res.json();
}
export async function collectPypi(packageName) {
    const normalized = normalizePackageName(packageName);
    try {
        // Package metadata
        const packageDataPromise = fetchJson(`https://pypi.org/pypi/${normalized}/json`);
        const recentPromise = fetchJson(`${PYPI_STATS_BASE_URL}/api/packages/${normalized}/recent`);
        const summaryPromise = fetchJson(`${PYPI_STATS_BASE_URL}/api/packages/${normalized}/summary`);
        const overallPromise = fetchJson(`${PYPI_STATS_BASE_URL}/api/packages/${normalized}/overall`);
        const pythonMajorPromise = fetchJson(`${PYPI_STATS_BASE_URL}/api/packages/${normalized}/python_major`);
        const pythonMinorPromise = fetchJson(`${PYPI_STATS_BASE_URL}/api/packages/${normalized}/python_minor`);
        const systemPromise = fetchJson(`${PYPI_STATS_BASE_URL}/api/packages/${normalized}/system`);
        const installerPromise = fetchJson(`${PYPI_STATS_BASE_URL}/api/packages/${normalized}/installer`);
        const overallChartPromise = fetchJson(`${PYPI_STATS_BASE_URL}/api/packages/${normalized}/chart/overall?format=json`);
        const pythonMajorChartPromise = fetchJson(`${PYPI_STATS_BASE_URL}/api/packages/${normalized}/chart/python_major?format=json`);
        const pythonMinorChartPromise = fetchJson(`${PYPI_STATS_BASE_URL}/api/packages/${normalized}/chart/python_minor?format=json`);
        const systemChartPromise = fetchJson(`${PYPI_STATS_BASE_URL}/api/packages/${normalized}/chart/system?format=json`);
        const installerChartPromise = fetchJson(`${PYPI_STATS_BASE_URL}/api/packages/${normalized}/chart/installer?format=json`);
        const [packageData, recent, summary, overall, pythonMajor, pythonMinor, system, installer, overallChart, pythonMajorChart, pythonMinorChart, systemChart, installerChart] = await Promise.all([
            packageDataPromise,
            recentPromise,
            summaryPromise,
            overallPromise,
            pythonMajorPromise,
            pythonMinorPromise,
            systemPromise,
            installerPromise,
            overallChartPromise,
            pythonMajorChartPromise,
            pythonMinorChartPromise,
            systemChartPromise,
            installerChartPromise,
        ]);
        // All time-series and breakdowns are provided by the external API
        const overallSeries = (overall.data || []).filter(p => p.category === 'without_mirrors');
        const systemBreakdown = summary.totals?.system || null;
        const pythonVersionBreakdown = summary.totals?.python_major
            ? Object.fromEntries(Object.entries(summary.totals.python_major).filter(([k]) => /^\d+$/.test(k)).map(([k, v]) => [`python${k}`, v]))
            : null;
        const pythonMinorBreakdown = summary.totals?.python_minor
            ? Object.fromEntries(Object.entries(summary.totals.python_minor).filter(([k]) => /^\d+(?:\.\d+)?$/.test(k)).map(([k, v]) => [`python${k}`, v]))
            : null;
        // Derive popular system and installer from totals/series
        let popularSystem;
        if (systemBreakdown && Object.keys(systemBreakdown).length > 0) {
            popularSystem = Object.entries(systemBreakdown).sort((a, b) => b[1] - a[1])[0]?.[0];
        }
        else if (system.data && system.data.length > 0) {
            const totals = {};
            for (const p of system.data)
                totals[p.category] = (totals[p.category] || 0) + p.downloads;
            popularSystem = Object.entries(totals).sort((a, b) => b[1] - a[1])[0]?.[0];
        }
        let popularInstaller;
        if (installer.data && installer.data.length > 0) {
            const totals = {};
            for (const p of installer.data)
                totals[p.category] = (totals[p.category] || 0) + p.downloads;
            popularInstaller = Object.entries(totals).sort((a, b) => b[1] - a[1])[0]?.[0];
        }
        // Latest release date for current version
        let latestReleaseDate;
        try {
            const currentVersion = packageData.info?.version;
            const files = currentVersion ? (packageData.releases?.[currentVersion] || []) : [];
            const latestUpload = files.reduce((max, f) => {
                const t = f.upload_time;
                if (!t)
                    return max;
                if (!max)
                    return t;
                return new Date(t) > new Date(max) ? t : max;
            }, undefined);
            if (latestUpload) {
                const d = new Date(latestUpload);
                if (!isNaN(d.getTime()))
                    latestReleaseDate = d.toISOString().slice(0, 10);
            }
        }
        catch { }
        return {
            platform: PlatformSettings.name,
            name: packageName,
            timestamp: new Date().toISOString(),
            metrics: {
                downloadsTotal: summary.totals?.overall,
                downloadsMonthly: recent.data?.last_month,
                downloadsWeekly: recent.data?.last_week,
                downloadsDaily: recent.data?.last_day,
                version: packageData.info?.version,
                latestReleaseDate,
                description: packageData.info?.summary,
                homepage: packageData.info?.home_page,
                author: packageData.info?.author,
                license: packageData.info?.license,
                requiresPython: packageData.info?.requires_python,
                releases: Object.keys(packageData.releases || {}).length,
                downloadsRange: overallSeries.map(p => ({ day: p.date, downloads: p.downloads })),
                overallSeries,
                pythonMajorSeries: (pythonMajor.data || []).filter(p => p.category?.toLowerCase?.() !== 'unknown'),
                pythonMinorSeries: (pythonMinor.data || []).filter(p => p.category?.toLowerCase?.() !== 'unknown'),
                systemSeries: system.data || [],
                installerSeries: installer.data || [],
                popularSystem,
                popularInstaller,
                // Server-prepared chart JSON (preferred for rendering)
                overallChart,
                pythonMajorChart: { ...pythonMajorChart, datasets: (pythonMajorChart.datasets || []).filter(ds => !/unknown/i.test(ds.label)) },
                pythonMinorChart: { ...pythonMinorChart, datasets: (pythonMinorChart.datasets || []).filter(ds => !/unknown/i.test(ds.label)) },
                systemChart,
                installerChart,
                pythonVersionBreakdown,
                pythonMinorBreakdown,
                systemBreakdown,
            }
        };
    }
    catch (error) {
        return {
            platform: PlatformSettings.name,
            name: packageName,
            timestamp: new Date().toISOString(),
            metrics: {},
            error: error instanceof Error ? error.message : String(error)
        };
    }
}
export async function collectPypiBatch(packageNames) {
    const results = [];
    for (const packageName of packageNames) {
        results.push(collectPypi(packageName));
    }
    return Promise.all(results);
}