mirror of
https://github.com/LukeHagar/usage-statistics.git
synced 2025-12-06 04:21:55 +00:00
Initial commit
This commit is contained in:
67
.github/workflows/update-stats.yml
vendored
Normal file
67
.github/workflows/update-stats.yml
vendored
Normal file
@@ -0,0 +1,67 @@
|
||||
name: Update Usage Statistics
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Run daily at 2 AM UTC
|
||||
- cron: '0 2 * * *'
|
||||
workflow_dispatch:
|
||||
# Allow manual triggering
|
||||
|
||||
jobs:
|
||||
update-stats:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@v1
|
||||
with:
|
||||
bun-version: latest
|
||||
|
||||
- name: Install dependencies
|
||||
run: bun install
|
||||
|
||||
- name: Run usage statistics tracker
|
||||
run: bun run src/index.ts --action
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_ACTIONS: true
|
||||
NODE_ENV: production
|
||||
|
||||
- name: Check for changes
|
||||
id: check-changes
|
||||
run: |
|
||||
if [ -n "$(git status --porcelain)" ]; then
|
||||
echo "changes=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "changes=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Commit and push changes
|
||||
if: steps.check-changes.outputs.changes == 'true'
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git add stats.json README.md
|
||||
git commit -m "chore: update usage statistics [skip ci]" || echo "No changes to commit"
|
||||
git push
|
||||
|
||||
- name: Create summary
|
||||
run: |
|
||||
if [ -f "stats.json" ]; then
|
||||
echo "## 📊 Usage Statistics Updated" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Stats have been updated and committed to the repository." >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Summary:" >> $GITHUB_STEP_SUMMARY
|
||||
cat stats.json | jq -r '.totalDownloads, .uniquePackages, (.platforms | join(", "))' | while read line; do
|
||||
echo "- $line" >> $GITHUB_STEP_SUMMARY
|
||||
done
|
||||
else
|
||||
echo "## ❌ No stats file generated" >> $GITHUB_STEP_SUMMARY
|
||||
echo "The stats.json file was not created. Check the logs for errors." >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
50
.gitignore
vendored
Normal file
50
.gitignore
vendored
Normal file
@@ -0,0 +1,50 @@
|
||||
# Dependencies
|
||||
node_modules/
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
# Build outputs
|
||||
dist/
|
||||
build/
|
||||
*.tsbuildinfo
|
||||
|
||||
# Environment variables
|
||||
.env
|
||||
.env.local
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
|
||||
# IDE files
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# OS generated files
|
||||
.DS_Store
|
||||
.DS_Store?
|
||||
._*
|
||||
.Spotlight-V100
|
||||
.Trashes
|
||||
ehthumbs.db
|
||||
Thumbs.db
|
||||
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
|
||||
# Runtime data
|
||||
pids
|
||||
*.pid
|
||||
*.seed
|
||||
*.pid.lock
|
||||
|
||||
# Coverage directory used by tools like istanbul
|
||||
coverage/
|
||||
*.lcov
|
||||
|
||||
# Bun
|
||||
.bun/
|
||||
193
README.md
193
README.md
@@ -1 +1,192 @@
|
||||
# usage-statistics
|
||||
# Usage Statistics
|
||||
|
||||
A Bun TypeScript script project for analyzing usage statistics with a clean, modern architecture.
|
||||
|
||||
## Features
|
||||
|
||||
- 🚀 **Fast Execution**: Built with Bun for lightning-fast TypeScript execution
|
||||
- 📊 **Usage Analytics**: Track user actions and generate statistics
|
||||
- 🧪 **Comprehensive Testing**: Full test suite with Bun's built-in test runner
|
||||
- 📦 **Modern Tooling**: TypeScript, ES modules, and modern JavaScript features
|
||||
- 🔧 **Developer Friendly**: Hot reloading, watch mode, and excellent DX
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [Bun](https://bun.sh/) (version 1.0.0 or higher)
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
# Install dependencies
|
||||
bun install
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Running the Script
|
||||
|
||||
```bash
|
||||
# Run the main script
|
||||
bun start
|
||||
|
||||
# Preview the report with mock data (no external API calls)
|
||||
bun preview
|
||||
|
||||
# Run in development mode with hot reloading
|
||||
bun run dev
|
||||
|
||||
# Run directly with bun
|
||||
bun run src/index.ts
|
||||
```
|
||||
|
||||
### Preview Mode
|
||||
|
||||
The preview mode allows you to see how the report will look without making any external API calls or writing files:
|
||||
|
||||
```bash
|
||||
# Generate a preview report with mock data
|
||||
bun preview
|
||||
|
||||
# Or use the long flag
|
||||
bun start --preview
|
||||
```
|
||||
|
||||
This is useful for:
|
||||
- Testing the report format
|
||||
- Demonstrating the functionality
|
||||
- Development and debugging
|
||||
- CI/CD testing without external dependencies
|
||||
|
||||
### Building
|
||||
|
||||
```bash
|
||||
# Build the project
|
||||
bun run build
|
||||
```
|
||||
|
||||
### Testing
|
||||
|
||||
```bash
|
||||
# Run all tests
|
||||
bun test
|
||||
|
||||
# Run tests in watch mode
|
||||
bun test --watch
|
||||
```
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
usage-statistics/
|
||||
├── src/
|
||||
│ ├── index.ts # Main entry point
|
||||
│ ├── index.test.ts # Test suite
|
||||
│ └── test-setup.ts # Test configuration
|
||||
├── package.json # Project configuration
|
||||
├── tsconfig.json # TypeScript configuration
|
||||
├── bunfig.toml # Bun configuration
|
||||
├── .gitignore # Git ignore rules
|
||||
└── README.md # This file
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### UsageStatistics Class
|
||||
|
||||
The main class for tracking and analyzing usage data.
|
||||
|
||||
#### Methods
|
||||
|
||||
- `addUsage(userId: string, action: string, metadata?: Record<string, any>)`: Add a new usage record
|
||||
- `getAllData()`: Get all usage data
|
||||
- `getUserData(userId: string)`: Get usage data for a specific user
|
||||
- `getActionData(action: string)`: Get usage data for a specific action
|
||||
- `getStatistics()`: Get comprehensive statistics summary
|
||||
|
||||
#### Example Usage
|
||||
|
||||
```typescript
|
||||
import { UsageStatistics } from './src/index';
|
||||
|
||||
const stats = new UsageStatistics();
|
||||
|
||||
// Add usage data
|
||||
stats.addUsage("user1", "login", { browser: "chrome" });
|
||||
stats.addUsage("user2", "logout");
|
||||
|
||||
// Get statistics
|
||||
const summary = stats.getStatistics();
|
||||
console.log(`Total records: ${summary.totalRecords}`);
|
||||
```
|
||||
|
||||
## Development
|
||||
|
||||
### Scripts
|
||||
|
||||
- `bun start`: Run the main script
|
||||
- `bun run dev`: Run in development mode with file watching
|
||||
- `bun run build`: Build the project for production
|
||||
- `bun test`: Run the test suite
|
||||
|
||||
### Adding Dependencies
|
||||
|
||||
```bash
|
||||
# Add a production dependency
|
||||
bun add <package-name>
|
||||
|
||||
# Add a development dependency
|
||||
bun add -d <package-name>
|
||||
```
|
||||
|
||||
## GitHub Actions Integration
|
||||
|
||||
This project includes a GitHub Actions workflow that automatically updates usage statistics and commits the results to the repository.
|
||||
|
||||
### Setup
|
||||
|
||||
1. **Enable the workflow**: The workflow file is located at `.github/workflows/update-stats.yml`
|
||||
2. **Configure your packages**: Update `src/config.ts` with your actual package names
|
||||
3. **Set up environment variables** (if needed):
|
||||
- `GITHUB_TOKEN`: Automatically provided by GitHub Actions
|
||||
- `POSTMAN_API_KEY`: If tracking Postman collections
|
||||
|
||||
### Workflow Features
|
||||
|
||||
- **Scheduled runs**: Updates stats daily at 2 AM UTC
|
||||
- **Manual triggering**: Can be run manually via GitHub Actions UI
|
||||
- **Rate limiting**: Built-in rate limiting to avoid API abuse
|
||||
- **Auto-commit**: Automatically commits `stats.json` and updates README
|
||||
- **Error handling**: Graceful handling of API failures
|
||||
|
||||
### Customization
|
||||
|
||||
Edit `src/config.ts` to track your specific packages:
|
||||
|
||||
```typescript
|
||||
export const defaultConfig: TrackingConfig = {
|
||||
npmPackages: ['your-package-name'],
|
||||
githubRepos: ['your-org/your-repo'],
|
||||
// ... other platforms
|
||||
};
|
||||
```
|
||||
|
||||
### Manual Execution
|
||||
|
||||
Run locally with GitHub Action mode:
|
||||
|
||||
```bash
|
||||
GITHUB_TOKEN=your_token bun run src/index.ts --action
|
||||
```
|
||||
|
||||
## Contributing
|
||||
|
||||
1. Fork the repository
|
||||
2. Create a feature branch
|
||||
3. Make your changes
|
||||
4. Add tests for new functionality
|
||||
5. Run the test suite
|
||||
6. Submit a pull request
|
||||
|
||||
## License
|
||||
|
||||
This project is open source and available under the [MIT License](LICENSE).
|
||||
59
bun.lock
Normal file
59
bun.lock
Normal file
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"lockfileVersion": 1,
|
||||
"workspaces": {
|
||||
"": {
|
||||
"name": "usage-statistics",
|
||||
"dependencies": {
|
||||
"@octokit/rest": "22.0.0",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.0.0",
|
||||
"bun-types": "1.2.19",
|
||||
"typescript": "^5.0.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@octokit/auth-token": ["@octokit/auth-token@6.0.0", "", {}, "sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w=="],
|
||||
|
||||
"@octokit/core": ["@octokit/core@7.0.3", "", { "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.1", "@octokit/request": "^10.0.2", "@octokit/request-error": "^7.0.0", "@octokit/types": "^14.0.0", "before-after-hook": "^4.0.0", "universal-user-agent": "^7.0.0" } }, "sha512-oNXsh2ywth5aowwIa7RKtawnkdH6LgU1ztfP9AIUCQCvzysB+WeU8o2kyyosDPwBZutPpjZDKPQGIzzrfTWweQ=="],
|
||||
|
||||
"@octokit/endpoint": ["@octokit/endpoint@11.0.0", "", { "dependencies": { "@octokit/types": "^14.0.0", "universal-user-agent": "^7.0.2" } }, "sha512-hoYicJZaqISMAI3JfaDr1qMNi48OctWuOih1m80bkYow/ayPw6Jj52tqWJ6GEoFTk1gBqfanSoI1iY99Z5+ekQ=="],
|
||||
|
||||
"@octokit/graphql": ["@octokit/graphql@9.0.1", "", { "dependencies": { "@octokit/request": "^10.0.2", "@octokit/types": "^14.0.0", "universal-user-agent": "^7.0.0" } }, "sha512-j1nQNU1ZxNFx2ZtKmL4sMrs4egy5h65OMDmSbVyuCzjOcwsHq6EaYjOTGXPQxgfiN8dJ4CriYHk6zF050WEULg=="],
|
||||
|
||||
"@octokit/openapi-types": ["@octokit/openapi-types@25.1.0", "", {}, "sha512-idsIggNXUKkk0+BExUn1dQ92sfysJrje03Q0bv0e+KPLrvyqZF8MnBpFz8UNfYDwB3Ie7Z0TByjWfzxt7vseaA=="],
|
||||
|
||||
"@octokit/plugin-paginate-rest": ["@octokit/plugin-paginate-rest@13.1.1", "", { "dependencies": { "@octokit/types": "^14.1.0" }, "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-q9iQGlZlxAVNRN2jDNskJW/Cafy7/XE52wjZ5TTvyhyOD904Cvx//DNyoO3J/MXJ0ve3rPoNWKEg5iZrisQSuw=="],
|
||||
|
||||
"@octokit/plugin-request-log": ["@octokit/plugin-request-log@6.0.0", "", { "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-UkOzeEN3W91/eBq9sPZNQ7sUBvYCqYbrrD8gTbBuGtHEuycE4/awMXcYvx6sVYo7LypPhmQwwpUe4Yyu4QZN5Q=="],
|
||||
|
||||
"@octokit/plugin-rest-endpoint-methods": ["@octokit/plugin-rest-endpoint-methods@16.0.0", "", { "dependencies": { "@octokit/types": "^14.1.0" }, "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-kJVUQk6/dx/gRNLWUnAWKFs1kVPn5O5CYZyssyEoNYaFedqZxsfYs7DwI3d67hGz4qOwaJ1dpm07hOAD1BXx6g=="],
|
||||
|
||||
"@octokit/request": ["@octokit/request@10.0.3", "", { "dependencies": { "@octokit/endpoint": "^11.0.0", "@octokit/request-error": "^7.0.0", "@octokit/types": "^14.0.0", "fast-content-type-parse": "^3.0.0", "universal-user-agent": "^7.0.2" } }, "sha512-V6jhKokg35vk098iBqp2FBKunk3kMTXlmq+PtbV9Gl3TfskWlebSofU9uunVKhUN7xl+0+i5vt0TGTG8/p/7HA=="],
|
||||
|
||||
"@octokit/request-error": ["@octokit/request-error@7.0.0", "", { "dependencies": { "@octokit/types": "^14.0.0" } }, "sha512-KRA7VTGdVyJlh0cP5Tf94hTiYVVqmt2f3I6mnimmaVz4UG3gQV/k4mDJlJv3X67iX6rmN7gSHCF8ssqeMnmhZg=="],
|
||||
|
||||
"@octokit/rest": ["@octokit/rest@22.0.0", "", { "dependencies": { "@octokit/core": "^7.0.2", "@octokit/plugin-paginate-rest": "^13.0.1", "@octokit/plugin-request-log": "^6.0.0", "@octokit/plugin-rest-endpoint-methods": "^16.0.0" } }, "sha512-z6tmTu9BTnw51jYGulxrlernpsQYXpui1RK21vmXn8yF5bp6iX16yfTtJYGK5Mh1qDkvDOmp2n8sRMcQmR8jiA=="],
|
||||
|
||||
"@octokit/types": ["@octokit/types@14.1.0", "", { "dependencies": { "@octokit/openapi-types": "^25.1.0" } }, "sha512-1y6DgTy8Jomcpu33N+p5w58l6xyt55Ar2I91RPiIA0xCJBXyUAhXCcmZaDWSANiha7R9a6qJJ2CRomGPZ6f46g=="],
|
||||
|
||||
"@types/node": ["@types/node@20.19.9", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-cuVNgarYWZqxRJDQHEB58GEONhOK79QVR/qYx4S7kcUObQvUwvFnYxJuuHUKm2aieN9X3yZB4LZsuYNU1Qphsw=="],
|
||||
|
||||
"@types/react": ["@types/react@19.1.9", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-WmdoynAX8Stew/36uTSVMcLJJ1KRh6L3IZRx1PZ7qJtBqT3dYTgyDTx8H1qoRghErydW7xw9mSJ3wS//tCRpFA=="],
|
||||
|
||||
"before-after-hook": ["before-after-hook@4.0.0", "", {}, "sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ=="],
|
||||
|
||||
"bun-types": ["bun-types@1.2.19", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-uAOTaZSPuYsWIXRpj7o56Let0g/wjihKCkeRqUBhlLVM/Bt+Fj9xTo+LhC1OV1XDaGkz4hNC80et5xgy+9KTHQ=="],
|
||||
|
||||
"csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="],
|
||||
|
||||
"fast-content-type-parse": ["fast-content-type-parse@3.0.0", "", {}, "sha512-ZvLdcY8P+N8mGQJahJV5G4U88CSvT1rP8ApL6uETe88MBXrBHAkZlSEySdUlyztF7ccb+Znos3TFqaepHxdhBg=="],
|
||||
|
||||
"typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="],
|
||||
|
||||
"undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="],
|
||||
|
||||
"universal-user-agent": ["universal-user-agent@7.0.3", "", {}, "sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A=="],
|
||||
}
|
||||
}
|
||||
14
bunfig.toml
Normal file
14
bunfig.toml
Normal file
@@ -0,0 +1,14 @@
|
||||
[install]
|
||||
# Use exact versions for reproducible builds
|
||||
exact = true
|
||||
|
||||
[install.scopes]
|
||||
# Configure any scoped packages if needed
|
||||
|
||||
[test]
|
||||
# Test configuration
|
||||
preload = ["./src/test-setup.ts"]
|
||||
|
||||
[run]
|
||||
# Runtime configuration
|
||||
bun = true
|
||||
24
package.json
Normal file
24
package.json
Normal file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "usage-statistics",
|
||||
"version": "1.0.0",
|
||||
"description": "A Bun TypeScript script project",
|
||||
"main": "src/index.ts",
|
||||
"scripts": {
|
||||
"start": "bun run src/index.ts",
|
||||
"preview": "bun run src/index.ts --preview",
|
||||
"dev": "bun --watch src/index.ts",
|
||||
"build": "bun build src/index.ts --outdir dist",
|
||||
"test": "bun test"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.0.0",
|
||||
"bun-types": "1.2.19",
|
||||
"typescript": "^5.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@octokit/rest": "22.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"bun": ">=1.0.0"
|
||||
}
|
||||
}
|
||||
342
src/aggregator.ts
Normal file
342
src/aggregator.ts
Normal file
@@ -0,0 +1,342 @@
|
||||
/**
|
||||
* Download Statistics Aggregator
|
||||
* Combines and analyzes statistics from all platform trackers
|
||||
*/
|
||||
|
||||
import type { BaseDownloadStats, DownloadStatsAggregator, TrackingConfig } from './types';
|
||||
import NpmTracker from './trackers/npm';
|
||||
import GitHubTracker from './trackers/github';
|
||||
import PyPiTracker from './trackers/pypi';
|
||||
import HomebrewTracker from './trackers/homebrew';
|
||||
import PowerShellTracker from './trackers/powershell';
|
||||
import PostmanTracker from './trackers/postman';
|
||||
import GoTracker from './trackers/go';
|
||||
import { globalRateLimiter } from './utils/rate-limiter';
|
||||
|
||||
export interface AggregatedStats {
|
||||
totalDownloads: number;
|
||||
uniquePackages: number;
|
||||
platforms: string[];
|
||||
timeRange: { start: Date; end: Date } | null;
|
||||
platformBreakdown: Record<string, {
|
||||
totalDownloads: number;
|
||||
uniquePackages: number;
|
||||
packages: string[];
|
||||
}>;
|
||||
topPackages: Array<{
|
||||
name: string;
|
||||
platform: string;
|
||||
downloads: number;
|
||||
}>;
|
||||
recentActivity: Array<{
|
||||
packageName: string;
|
||||
platform: string;
|
||||
downloads: number;
|
||||
timestamp: Date;
|
||||
}>;
|
||||
}
|
||||
|
||||
export class DownloadStatsAggregator implements DownloadStatsAggregator {
|
||||
private trackers: Map<string, any> = new Map();
|
||||
private config: TrackingConfig;
|
||||
|
||||
constructor(config: TrackingConfig) {
|
||||
this.config = config;
|
||||
this.initializeTrackers();
|
||||
}
|
||||
|
||||
private initializeTrackers() {
|
||||
// Initialize all trackers
|
||||
this.trackers.set('npm', new NpmTracker());
|
||||
this.trackers.set('github', new GitHubTracker(process.env.GITHUB_TOKEN));
|
||||
this.trackers.set('pypi', new PyPiTracker());
|
||||
this.trackers.set('homebrew', new HomebrewTracker(process.env.GITHUB_TOKEN));
|
||||
this.trackers.set('powershell', new PowerShellTracker());
|
||||
this.trackers.set('postman', new PostmanTracker(process.env.POSTMAN_API_KEY));
|
||||
this.trackers.set('go', new GoTracker(process.env.GITHUB_TOKEN));
|
||||
}
|
||||
|
||||
async aggregateStats(stats: BaseDownloadStats[]): Promise<AggregatedStats> {
|
||||
const platformBreakdown: Record<string, {
|
||||
totalDownloads: number;
|
||||
uniquePackages: number;
|
||||
packages: string[];
|
||||
}> = {};
|
||||
|
||||
const packageMap = new Map<string, { downloads: number; platform: string }>();
|
||||
const recentActivity: Array<{
|
||||
packageName: string;
|
||||
platform: string;
|
||||
downloads: number;
|
||||
timestamp: Date;
|
||||
}> = [];
|
||||
|
||||
let totalDownloads = 0;
|
||||
let uniquePackages = 0;
|
||||
const platforms = new Set<string>();
|
||||
const timeRange = { start: new Date(), end: new Date(0) };
|
||||
|
||||
// Process each stat
|
||||
for (const stat of stats) {
|
||||
totalDownloads += stat.downloadCount;
|
||||
platforms.add(stat.platform);
|
||||
|
||||
// Track package downloads
|
||||
const packageKey = `${stat.platform}:${stat.packageName}`;
|
||||
const existing = packageMap.get(packageKey);
|
||||
if (existing) {
|
||||
existing.downloads += stat.downloadCount;
|
||||
} else {
|
||||
packageMap.set(packageKey, { downloads: stat.downloadCount, platform: stat.platform });
|
||||
uniquePackages++;
|
||||
}
|
||||
|
||||
// Update platform breakdown
|
||||
if (!platformBreakdown[stat.platform]) {
|
||||
platformBreakdown[stat.platform] = {
|
||||
totalDownloads: 0,
|
||||
uniquePackages: 0,
|
||||
packages: []
|
||||
};
|
||||
}
|
||||
platformBreakdown[stat.platform].totalDownloads += stat.downloadCount;
|
||||
if (!platformBreakdown[stat.platform].packages.includes(stat.packageName)) {
|
||||
platformBreakdown[stat.platform].packages.push(stat.packageName);
|
||||
platformBreakdown[stat.platform].uniquePackages++;
|
||||
}
|
||||
|
||||
// Track recent activity
|
||||
recentActivity.push({
|
||||
packageName: stat.packageName,
|
||||
platform: stat.platform,
|
||||
downloads: stat.downloadCount,
|
||||
timestamp: stat.timestamp
|
||||
});
|
||||
|
||||
// Update time range
|
||||
if (stat.timestamp < timeRange.start) {
|
||||
timeRange.start = stat.timestamp;
|
||||
}
|
||||
if (stat.timestamp > timeRange.end) {
|
||||
timeRange.end = stat.timestamp;
|
||||
}
|
||||
}
|
||||
|
||||
// Sort recent activity by timestamp
|
||||
recentActivity.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
|
||||
|
||||
// Get top packages
|
||||
const topPackages = Array.from(packageMap.entries())
|
||||
.map(([key, data]) => ({
|
||||
name: key.split(':')[1],
|
||||
platform: data.platform,
|
||||
downloads: data.downloads
|
||||
}))
|
||||
.sort((a, b) => b.downloads - a.downloads)
|
||||
.slice(0, 10);
|
||||
|
||||
return {
|
||||
totalDownloads,
|
||||
uniquePackages,
|
||||
platforms: Array.from(platforms),
|
||||
timeRange: timeRange.start < timeRange.end ? timeRange : null,
|
||||
platformBreakdown,
|
||||
topPackages,
|
||||
recentActivity: recentActivity.slice(0, 20)
|
||||
};
|
||||
}
|
||||
|
||||
async collectAllStats(): Promise<BaseDownloadStats[]> {
|
||||
const allStats: BaseDownloadStats[] = [];
|
||||
|
||||
// Collect NPM stats with rate limiting
|
||||
if (this.config.npmPackages) {
|
||||
const npmOperations = this.config.npmPackages.map(packageName =>
|
||||
async () => {
|
||||
try {
|
||||
const npmTracker = this.trackers.get('npm');
|
||||
const stats = await npmTracker.getDownloadStats(packageName);
|
||||
return stats;
|
||||
} catch (error) {
|
||||
console.error(`Error collecting NPM stats for ${packageName}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
const npmResults = await globalRateLimiter.throttleRequests(npmOperations, 2, 2000);
|
||||
npmResults.forEach(stats => allStats.push(...stats));
|
||||
}
|
||||
|
||||
// Collect GitHub stats with rate limiting
|
||||
if (this.config.githubRepos) {
|
||||
const githubOperations = this.config.githubRepos.map(repo =>
|
||||
async () => {
|
||||
try {
|
||||
const githubTracker = this.trackers.get('github');
|
||||
const stats = await githubTracker.getDownloadStats(repo);
|
||||
return stats;
|
||||
} catch (error) {
|
||||
console.error(`Error collecting GitHub stats for ${repo}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
const githubResults = await globalRateLimiter.throttleRequests(githubOperations, 1, 3000);
|
||||
githubResults.forEach(stats => allStats.push(...stats));
|
||||
}
|
||||
|
||||
// Collect PyPI stats with rate limiting
|
||||
if (this.config.pythonPackages) {
|
||||
const pypiOperations = this.config.pythonPackages.map(packageName =>
|
||||
async () => {
|
||||
try {
|
||||
const pypiTracker = this.trackers.get('pypi');
|
||||
const stats = await pypiTracker.getDownloadStats(packageName);
|
||||
return stats;
|
||||
} catch (error) {
|
||||
console.error(`Error collecting PyPI stats for ${packageName}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
const pypiResults = await globalRateLimiter.throttleRequests(pypiOperations, 2, 1500);
|
||||
pypiResults.forEach(stats => allStats.push(...stats));
|
||||
}
|
||||
|
||||
// Collect Homebrew stats with rate limiting
|
||||
if (this.config.homebrewPackages) {
|
||||
const homebrewOperations = this.config.homebrewPackages.map(packageName =>
|
||||
async () => {
|
||||
try {
|
||||
const homebrewTracker = this.trackers.get('homebrew');
|
||||
const stats = await homebrewTracker.getDownloadStats(packageName);
|
||||
return stats;
|
||||
} catch (error) {
|
||||
console.error(`Error collecting Homebrew stats for ${packageName}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
const homebrewResults = await globalRateLimiter.throttleRequests(homebrewOperations, 2, 2000);
|
||||
homebrewResults.forEach(stats => allStats.push(...stats));
|
||||
}
|
||||
|
||||
// Collect PowerShell stats with rate limiting
|
||||
if (this.config.powershellModules) {
|
||||
const powershellOperations = this.config.powershellModules.map(moduleName =>
|
||||
async () => {
|
||||
try {
|
||||
const powershellTracker = this.trackers.get('powershell');
|
||||
const stats = await powershellTracker.getDownloadStats(moduleName);
|
||||
return stats;
|
||||
} catch (error) {
|
||||
console.error(`Error collecting PowerShell stats for ${moduleName}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
const powershellResults = await globalRateLimiter.throttleRequests(powershellOperations, 2, 2000);
|
||||
powershellResults.forEach(stats => allStats.push(...stats));
|
||||
}
|
||||
|
||||
// Collect Postman stats with rate limiting
|
||||
if (this.config.postmanCollections) {
|
||||
const postmanOperations = this.config.postmanCollections.map(collectionId =>
|
||||
async () => {
|
||||
try {
|
||||
const postmanTracker = this.trackers.get('postman');
|
||||
const stats = await postmanTracker.getDownloadStats(collectionId);
|
||||
return stats;
|
||||
} catch (error) {
|
||||
console.error(`Error collecting Postman stats for ${collectionId}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
const postmanResults = await globalRateLimiter.throttleRequests(postmanOperations, 2, 2000);
|
||||
postmanResults.forEach(stats => allStats.push(...stats));
|
||||
}
|
||||
|
||||
// Collect Go stats with rate limiting
|
||||
if (this.config.goModules) {
|
||||
const goOperations = this.config.goModules.map(moduleName =>
|
||||
async () => {
|
||||
try {
|
||||
const goTracker = this.trackers.get('go');
|
||||
const stats = await goTracker.getDownloadStats(moduleName);
|
||||
return stats;
|
||||
} catch (error) {
|
||||
console.error(`Error collecting Go stats for ${moduleName}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
const goResults = await globalRateLimiter.throttleRequests(goOperations, 2, 2000);
|
||||
goResults.forEach(stats => allStats.push(...stats));
|
||||
}
|
||||
|
||||
return allStats;
|
||||
}
|
||||
|
||||
async generateReport(): Promise<AggregatedStats> {
|
||||
const allStats = await this.collectAllStats();
|
||||
return this.aggregateStats(allStats);
|
||||
}
|
||||
|
||||
async getPlatformStats(platform: string): Promise<BaseDownloadStats[]> {
|
||||
const tracker = this.trackers.get(platform);
|
||||
if (!tracker) {
|
||||
throw new Error(`Unknown platform: ${platform}`);
|
||||
}
|
||||
|
||||
const allStats: BaseDownloadStats[] = [];
|
||||
const packages = this.getPackagesForPlatform(platform);
|
||||
|
||||
const operations = packages.map(packageName =>
|
||||
async () => {
|
||||
try {
|
||||
const stats = await tracker.getDownloadStats(packageName);
|
||||
return stats;
|
||||
} catch (error) {
|
||||
console.error(`Error collecting ${platform} stats for ${packageName}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
const results = await globalRateLimiter.throttleRequests(operations, 2, 2000);
|
||||
results.forEach(stats => allStats.push(...stats));
|
||||
|
||||
return allStats;
|
||||
}
|
||||
|
||||
private getPackagesForPlatform(platform: string): string[] {
|
||||
switch (platform) {
|
||||
case 'npm':
|
||||
return this.config.npmPackages || [];
|
||||
case 'github':
|
||||
return this.config.githubRepos || [];
|
||||
case 'pypi':
|
||||
return this.config.pythonPackages || [];
|
||||
case 'homebrew':
|
||||
return this.config.homebrewPackages || [];
|
||||
case 'powershell':
|
||||
return this.config.powershellModules || [];
|
||||
case 'postman':
|
||||
return this.config.postmanCollections || [];
|
||||
case 'go':
|
||||
return this.config.goModules || [];
|
||||
default:
|
||||
return [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default DownloadStatsAggregator;
|
||||
112
src/config.ts
Normal file
112
src/config.ts
Normal file
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* Configuration for usage statistics tracking
|
||||
* Customize this file to track your specific packages and tools
|
||||
*/
|
||||
|
||||
import type { TrackingConfig } from './types';
|
||||
|
||||
export const defaultConfig: TrackingConfig = {
|
||||
// NPM packages - Add your NPM package names here
|
||||
npmPackages: [
|
||||
'lodash',
|
||||
'axios'
|
||||
],
|
||||
|
||||
// GitHub repositories - Format: 'owner/repo'
|
||||
githubRepos: [
|
||||
'microsoft/vscode',
|
||||
'facebook/react'
|
||||
],
|
||||
|
||||
// PyPI packages - Add your Python package names here
|
||||
pythonPackages: [
|
||||
'requests',
|
||||
'numpy'
|
||||
],
|
||||
|
||||
// Homebrew formulae - Add your Homebrew package names here
|
||||
homebrewPackages: [
|
||||
'git',
|
||||
'node'
|
||||
],
|
||||
|
||||
// PowerShell modules - Add your PowerShell module names here
|
||||
powershellModules: [
|
||||
'PowerShellGet'
|
||||
],
|
||||
|
||||
// Postman collections - Add your Postman collection IDs here
|
||||
postmanCollections: [
|
||||
// Note: These are example IDs - replace with real collection IDs
|
||||
// For testing, we'll leave these empty to avoid errors
|
||||
],
|
||||
|
||||
// Go modules - Add your Go module paths here
|
||||
goModules: [
|
||||
'github.com/gin-gonic/gin',
|
||||
'github.com/go-chi/chi'
|
||||
],
|
||||
|
||||
// Update interval in milliseconds (default: 1 hour)
|
||||
updateInterval: 60 * 60 * 1000,
|
||||
|
||||
// Enable detailed logging
|
||||
enableLogging: true
|
||||
};
|
||||
|
||||
// Test configuration with minimal packages
|
||||
export const testConfig: TrackingConfig = {
|
||||
npmPackages: ['lodash'],
|
||||
githubRepos: [],
|
||||
pythonPackages: ['requests'],
|
||||
homebrewPackages: ['git'],
|
||||
powershellModules: [],
|
||||
postmanCollections: [],
|
||||
goModules: [],
|
||||
updateInterval: 60 * 60 * 1000,
|
||||
enableLogging: false
|
||||
};
|
||||
|
||||
// Environment-specific configurations
|
||||
export const developmentConfig: TrackingConfig = {
|
||||
...defaultConfig,
|
||||
enableLogging: true,
|
||||
updateInterval: 5 * 60 * 1000, // 5 minutes for development
|
||||
};
|
||||
|
||||
export const productionConfig: TrackingConfig = {
|
||||
...defaultConfig,
|
||||
enableLogging: false,
|
||||
updateInterval: 60 * 60 * 1000, // 1 hour for production
|
||||
};
|
||||
|
||||
// Helper function to get configuration based on environment
|
||||
export function getConfig(environment: 'development' | 'production' | 'default' | 'test' = 'default'): TrackingConfig {
|
||||
switch (environment) {
|
||||
case 'development':
|
||||
return developmentConfig;
|
||||
case 'production':
|
||||
return productionConfig;
|
||||
case 'test':
|
||||
return testConfig;
|
||||
default:
|
||||
return defaultConfig;
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to validate configuration
|
||||
export function validateConfig(config: TrackingConfig): string[] {
|
||||
const errors: string[] = [];
|
||||
|
||||
if (!config.npmPackages && !config.githubRepos && !config.pythonPackages &&
|
||||
!config.homebrewPackages && !config.powershellModules &&
|
||||
!config.postmanCollections && !config.goModules) {
|
||||
errors.push('No packages configured for tracking. Please add packages to at least one platform.');
|
||||
}
|
||||
|
||||
if (config.updateInterval && config.updateInterval < 60000) {
|
||||
errors.push('Update interval should be at least 60 seconds to avoid rate limiting.');
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
98
src/index.test.ts
Normal file
98
src/index.test.ts
Normal file
@@ -0,0 +1,98 @@
|
||||
import { describe, it, expect, beforeEach } from "bun:test";
|
||||
import { UsageStatisticsManager } from "./index";
|
||||
import { getConfig } from "./config";
|
||||
|
||||
describe("UsageStatisticsManager", () => {
|
||||
let manager: UsageStatisticsManager;
|
||||
|
||||
beforeEach(() => {
|
||||
const config = getConfig('test');
|
||||
manager = new UsageStatisticsManager(config);
|
||||
});
|
||||
|
||||
describe("generateComprehensiveReport", () => {
|
||||
it("should generate a report with aggregated stats", async () => {
|
||||
const report = await manager.generateComprehensiveReport();
|
||||
|
||||
expect(report).toBeDefined();
|
||||
expect(report.totalDownloads).toBeGreaterThanOrEqual(0);
|
||||
expect(report.uniquePackages).toBeGreaterThanOrEqual(0);
|
||||
expect(Array.isArray(report.platforms)).toBe(true);
|
||||
expect(Array.isArray(report.topPackages)).toBe(true);
|
||||
expect(Array.isArray(report.recentActivity)).toBe(true);
|
||||
expect(typeof report.platformBreakdown).toBe('object');
|
||||
}, 30000); // 30 second timeout
|
||||
});
|
||||
|
||||
describe("getPlatformReport", () => {
|
||||
it("should generate a report for a specific platform", async () => {
|
||||
const report = await manager.getPlatformReport('npm');
|
||||
|
||||
expect(report).toBeDefined();
|
||||
expect(report.totalDownloads).toBeGreaterThanOrEqual(0);
|
||||
expect(Array.isArray(report.platforms)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("exportReport", () => {
|
||||
it("should export JSON report", async () => {
|
||||
const jsonReport = await manager.exportReport('json');
|
||||
|
||||
expect(jsonReport).toBeDefined();
|
||||
expect(typeof jsonReport).toBe('string');
|
||||
|
||||
// Should be valid JSON
|
||||
const parsed = JSON.parse(jsonReport);
|
||||
expect(parsed).toBeDefined();
|
||||
}, 30000); // 30 second timeout
|
||||
|
||||
it("should export CSV report", async () => {
|
||||
const csvReport = await manager.exportReport('csv');
|
||||
|
||||
expect(csvReport).toBeDefined();
|
||||
expect(typeof csvReport).toBe('string');
|
||||
expect(csvReport.includes('Platform,Package,Downloads')).toBe(true);
|
||||
}, 30000); // 30 second timeout
|
||||
});
|
||||
|
||||
describe("getLastUpdateTime", () => {
|
||||
it("should return null initially", () => {
|
||||
const lastUpdate = manager.getLastUpdateTime();
|
||||
expect(lastUpdate).toBeNull();
|
||||
});
|
||||
|
||||
it("should return update time after generating report", async () => {
|
||||
await manager.generateComprehensiveReport();
|
||||
const lastUpdate = manager.getLastUpdateTime();
|
||||
|
||||
expect(lastUpdate).not.toBeNull();
|
||||
expect(lastUpdate).toBeInstanceOf(Date);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Configuration", () => {
|
||||
it("should load development config", () => {
|
||||
const config = getConfig('development');
|
||||
|
||||
expect(config).toBeDefined();
|
||||
expect(config.enableLogging).toBe(true);
|
||||
expect(config.updateInterval).toBe(5 * 60 * 1000); // 5 minutes
|
||||
});
|
||||
|
||||
it("should load production config", () => {
|
||||
const config = getConfig('production');
|
||||
|
||||
expect(config).toBeDefined();
|
||||
expect(config.enableLogging).toBe(false);
|
||||
expect(config.updateInterval).toBe(60 * 60 * 1000); // 1 hour
|
||||
});
|
||||
|
||||
it("should load default config", () => {
|
||||
const config = getConfig('default');
|
||||
|
||||
expect(config).toBeDefined();
|
||||
expect(config.enableLogging).toBe(true);
|
||||
expect(config.updateInterval).toBe(60 * 60 * 1000); // 1 hour
|
||||
});
|
||||
});
|
||||
266
src/index.ts
Normal file
266
src/index.ts
Normal file
@@ -0,0 +1,266 @@
|
||||
#!/usr/bin/env bun
|
||||
import type { TrackingConfig } from './types';
|
||||
import { DownloadStatsAggregator } from './aggregator';
|
||||
import type { AggregatedStats } from './aggregator';
|
||||
import { getConfig, validateConfig } from './config';
|
||||
import { promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
import { execSync } from 'node:child_process';
|
||||
|
||||
/**
|
||||
* Usage Statistics Tracker
|
||||
*
|
||||
* Tracks download statistics across multiple platforms:
|
||||
* - NPM packages
|
||||
* - PyPI packages
|
||||
* - Homebrew formulas
|
||||
* - PowerShell modules
|
||||
* - Postman collections
|
||||
* - Go modules
|
||||
* - GitHub releases
|
||||
*/
|
||||
|
||||
const STATS_FILE = 'stats.json';
|
||||
const README_FILE = 'README.md';
|
||||
const STATS_MARKER_START = '<!-- USAGE_STATS_START -->';
|
||||
const STATS_MARKER_END = '<!-- USAGE_STATS_END -->';
|
||||
|
||||
async function writeStatsFile(stats: AggregatedStats, filePath = STATS_FILE) {
|
||||
await fs.writeFile(filePath, JSON.stringify(stats, null, 2));
|
||||
console.log(`📄 Stats written to ${filePath}`);
|
||||
}
|
||||
|
||||
async function updateReadmeWithStats(stats: AggregatedStats, readmePath = README_FILE) {
|
||||
try {
|
||||
const readmeContent = await fs.readFile(readmePath, 'utf-8');
|
||||
|
||||
const statsSection = `
|
||||
## 📊 Usage Statistics
|
||||
|
||||
Last updated: ${new Date().toISOString()}
|
||||
|
||||
### Summary
|
||||
- **Total Downloads**: ${stats.totalDownloads.toLocaleString()}
|
||||
- **Unique Packages**: ${stats.uniquePackages}
|
||||
- **Platforms Tracked**: ${stats.platforms.join(', ')}
|
||||
|
||||
### Top Packages
|
||||
${stats.topPackages.slice(0, 10).map((pkg, index) =>
|
||||
`${index + 1}. **${pkg.name}** (${pkg.platform}) - ${pkg.downloads.toLocaleString()} downloads`
|
||||
).join('\n')}
|
||||
|
||||
### Recent Activity
|
||||
${stats.recentActivity.slice(0, 5).map(activity =>
|
||||
`- **${activity.packageName}** (${activity.platform}) - ${activity.downloads.toLocaleString()} downloads on ${activity.timestamp.toLocaleDateString()}`
|
||||
).join('\n')}
|
||||
`;
|
||||
|
||||
const startMarker = readmeContent.indexOf(STATS_MARKER_START);
|
||||
const endMarker = readmeContent.indexOf(STATS_MARKER_END);
|
||||
|
||||
if (startMarker !== -1 && endMarker !== -1) {
|
||||
const beforeStats = readmeContent.substring(0, startMarker + STATS_MARKER_START.length);
|
||||
const afterStats = readmeContent.substring(endMarker);
|
||||
const updatedContent = beforeStats + statsSection + afterStats;
|
||||
await fs.writeFile(readmePath, updatedContent);
|
||||
console.log(`📝 README updated with stats`);
|
||||
} else {
|
||||
console.warn(`⚠️ Stats markers not found in README. Please add ${STATS_MARKER_START} and ${STATS_MARKER_END} markers.`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('❌ Error updating README:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function gitCommitAndPush(files: string[], message: string) {
|
||||
execSync(`git config user.name "github-actions[bot]"`);
|
||||
execSync(`git config user.email "github-actions[bot]@users.noreply.github.com"`);
|
||||
execSync(`git add ${files.join(' ')}`);
|
||||
execSync(`git commit -m "${message}" || echo 'No changes to commit.'`);
|
||||
execSync(`git push`);
|
||||
}
|
||||
|
||||
const trackingConfig: TrackingConfig = getConfig(process.env.NODE_ENV as 'development' | 'production' || 'default');
|
||||
|
||||
class UsageStatisticsManager {
|
||||
private aggregator: DownloadStatsAggregator;
|
||||
private lastUpdateTime: Date | null = null;
|
||||
|
||||
constructor(config: TrackingConfig) {
|
||||
this.aggregator = new DownloadStatsAggregator(config);
|
||||
}
|
||||
|
||||
async generateComprehensiveReport(): Promise<AggregatedStats> {
|
||||
console.log('📊 Generating comprehensive usage statistics report...\n');
|
||||
const stats = await this.aggregator.collectAllStats();
|
||||
const report = this.aggregator.aggregateStats(stats);
|
||||
this.lastUpdateTime = new Date();
|
||||
return report;
|
||||
}
|
||||
|
||||
async getPlatformReport(platform: string): Promise<AggregatedStats> {
|
||||
console.log(`📊 Generating ${platform} platform report...\n`);
|
||||
const stats = await this.aggregator.getPlatformStats(platform);
|
||||
const report = this.aggregator.aggregateStats(stats);
|
||||
this.lastUpdateTime = new Date();
|
||||
return report;
|
||||
}
|
||||
|
||||
async exportReport(format: 'json' | 'csv' = 'json'): Promise<string> {
|
||||
const report = await this.generateComprehensiveReport();
|
||||
|
||||
if (format === 'csv') {
|
||||
const csvHeader = 'Platform,Package,Downloads,Last Updated\n';
|
||||
const csvRows = report.topPackages.map(pkg =>
|
||||
`${pkg.platform},${pkg.name},${pkg.downloads},${new Date().toISOString()}`
|
||||
).join('\n');
|
||||
return csvHeader + csvRows;
|
||||
}
|
||||
|
||||
return JSON.stringify(report, null, 2);
|
||||
}
|
||||
|
||||
getLastUpdateTime(): Date | null {
|
||||
return this.lastUpdateTime;
|
||||
}
|
||||
|
||||
async displayReport(report: AggregatedStats) {
|
||||
console.log('🚀 Usage Statistics Report');
|
||||
console.log('==================================================\n');
|
||||
|
||||
console.log('📈 Summary:');
|
||||
console.log(`Total Downloads: ${report.totalDownloads.toLocaleString()}`);
|
||||
console.log(`Unique Packages: ${report.uniquePackages}`);
|
||||
console.log(`Platforms Tracked: ${report.platforms.join(', ')}`);
|
||||
if (report.timeRange) {
|
||||
console.log(`Time Range: ${report.timeRange.start.toLocaleDateString()} to ${report.timeRange.end.toLocaleDateString()}\n`);
|
||||
}
|
||||
|
||||
console.log('🏗️ Platform Breakdown:');
|
||||
for (const [platform, data] of Object.entries(report.platformBreakdown)) {
|
||||
console.log(` ${platform.toUpperCase()}:`);
|
||||
console.log(` Downloads: ${data.totalDownloads.toLocaleString()}`);
|
||||
console.log(` Packages: ${data.uniquePackages}`);
|
||||
console.log(` Package List: ${data.packages.join(', ')}`);
|
||||
}
|
||||
console.log();
|
||||
|
||||
console.log('🏆 Top Packages:');
|
||||
report.topPackages.slice(0, 10).forEach((pkg, index) => {
|
||||
console.log(` ${index + 1}. ${pkg.name} (${pkg.platform}) - ${pkg.downloads.toLocaleString()} downloads`);
|
||||
});
|
||||
console.log();
|
||||
|
||||
console.log('🕒 Recent Activity:');
|
||||
report.recentActivity.slice(0, 5).forEach(activity => {
|
||||
console.log(` ${activity.packageName} (${activity.platform}) - ${activity.downloads.toLocaleString()} downloads on ${activity.timestamp.toLocaleDateString()}`);
|
||||
});
|
||||
console.log('==================================================');
|
||||
}
|
||||
|
||||
async generatePreviewReport(): Promise<AggregatedStats> {
|
||||
console.log('🎭 Generating preview report with mock data...\n');
|
||||
|
||||
// Create mock data for preview
|
||||
const mockStats = [
|
||||
{
|
||||
platform: 'npm',
|
||||
packageName: 'lodash',
|
||||
downloadCount: 1500000,
|
||||
timestamp: new Date(),
|
||||
period: 'total' as const,
|
||||
metadata: { version: '4.17.21' }
|
||||
},
|
||||
{
|
||||
platform: 'npm',
|
||||
packageName: 'axios',
|
||||
downloadCount: 800000,
|
||||
timestamp: new Date(),
|
||||
period: 'total' as const,
|
||||
metadata: { version: '1.6.0' }
|
||||
},
|
||||
{
|
||||
platform: 'github',
|
||||
packageName: 'microsoft/vscode',
|
||||
downloadCount: 500000,
|
||||
timestamp: new Date(),
|
||||
period: 'total' as const,
|
||||
metadata: { release: 'v1.85.0' }
|
||||
},
|
||||
{
|
||||
platform: 'pypi',
|
||||
packageName: 'requests',
|
||||
downloadCount: 300000,
|
||||
timestamp: new Date(),
|
||||
period: 'total' as const,
|
||||
metadata: { version: '2.31.0' }
|
||||
},
|
||||
{
|
||||
platform: 'homebrew',
|
||||
packageName: 'git',
|
||||
downloadCount: 250000,
|
||||
timestamp: new Date(),
|
||||
period: 'total' as const,
|
||||
metadata: { version: '2.43.0' }
|
||||
}
|
||||
];
|
||||
|
||||
const report = this.aggregator.aggregateStats(mockStats);
|
||||
this.lastUpdateTime = new Date();
|
||||
return report;
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('🚀 Usage Statistics Tracker Starting...\n');
|
||||
|
||||
// Validate configuration
|
||||
try {
|
||||
validateConfig(trackingConfig);
|
||||
} catch (error) {
|
||||
console.error('❌ Configuration validation failed:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const manager = new UsageStatisticsManager(trackingConfig);
|
||||
|
||||
try {
|
||||
// Check for preview mode
|
||||
const isPreview = process.argv.includes('--preview') || process.argv.includes('-p');
|
||||
|
||||
let report: AggregatedStats;
|
||||
if (isPreview) {
|
||||
report = await manager.generatePreviewReport();
|
||||
} else {
|
||||
report = await manager.generateComprehensiveReport();
|
||||
}
|
||||
|
||||
await manager.displayReport(report);
|
||||
|
||||
const jsonReport = await manager.exportReport('json');
|
||||
console.log('\n📄 JSON Report:');
|
||||
console.log(jsonReport);
|
||||
|
||||
// Only write files and commit if not in preview mode and running in GitHub Actions
|
||||
if (!isPreview && (process.env.GITHUB_ACTIONS === 'true' || process.argv.includes('--action'))) {
|
||||
await writeStatsFile(report);
|
||||
await updateReadmeWithStats(report);
|
||||
await gitCommitAndPush([STATS_FILE, README_FILE], 'chore: update usage statistics [skip ci]');
|
||||
console.log('✅ Stats written, README updated, and changes pushed.');
|
||||
} else if (isPreview) {
|
||||
console.log('\n🎭 Preview mode - no files written or commits made');
|
||||
}
|
||||
|
||||
console.log('\n✅ Script completed successfully!');
|
||||
} catch (error) {
|
||||
console.error('❌ Error during execution:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run the main function if this file is executed directly
|
||||
if (import.meta.main) {
|
||||
main();
|
||||
}
|
||||
|
||||
export { UsageStatisticsManager, trackingConfig };
|
||||
29
src/test-setup.ts
Normal file
29
src/test-setup.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
/**
|
||||
* Test setup file for Bun
|
||||
* This file is preloaded before running tests
|
||||
*/
|
||||
|
||||
// Global test utilities can be defined here
|
||||
(globalThis as any).testUtils = {
|
||||
createMockUsageData: (userId: string, action: string) => ({
|
||||
timestamp: new Date(),
|
||||
userId,
|
||||
action,
|
||||
metadata: { test: true }
|
||||
})
|
||||
};
|
||||
|
||||
// Extend global types
|
||||
declare global {
|
||||
var testUtils: {
|
||||
createMockUsageData: (userId: string, action: string) => {
|
||||
timestamp: Date;
|
||||
userId: string;
|
||||
action: string;
|
||||
metadata: { test: boolean };
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
// Export to make this a module
|
||||
export {};
|
||||
246
src/trackers/github.ts
Normal file
246
src/trackers/github.ts
Normal file
@@ -0,0 +1,246 @@
|
||||
/**
|
||||
* GitHub Release Download Tracker
|
||||
* Uses GitHub API with Octokit to fetch release download statistics
|
||||
*/
|
||||
|
||||
import type { BaseDownloadStats, PlatformTracker } from '../types';
|
||||
import { Octokit } from '@octokit/rest';
|
||||
|
||||
export interface GitHubDownloadStats extends BaseDownloadStats {
|
||||
platform: 'github';
|
||||
repository: string;
|
||||
releaseId: number;
|
||||
releaseName: string;
|
||||
releaseTag: string;
|
||||
assetName?: string;
|
||||
assetId?: number;
|
||||
}
|
||||
|
||||
export interface GitHubReleaseInfo {
|
||||
id: number;
|
||||
name: string | null;
|
||||
tag_name: string;
|
||||
published_at: string | null;
|
||||
assets: Array<{
|
||||
id: number;
|
||||
name: string;
|
||||
download_count: number;
|
||||
size: number;
|
||||
content_type: string;
|
||||
}>;
|
||||
body?: string | null;
|
||||
draft: boolean;
|
||||
prerelease: boolean;
|
||||
}
|
||||
|
||||
export interface GitHubRepositoryInfo {
|
||||
id: number;
|
||||
name: string;
|
||||
full_name: string;
|
||||
description?: string | null;
|
||||
html_url: string;
|
||||
stargazers_count: number;
|
||||
forks_count: number;
|
||||
language?: string | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export class GitHubTracker implements PlatformTracker {
|
||||
name = 'github';
|
||||
private token?: string;
|
||||
private octokit: Octokit | null = null;
|
||||
|
||||
constructor(token?: string) {
|
||||
this.token = token || process.env.GITHUB_TOKEN;
|
||||
this.initializeOctokit();
|
||||
}
|
||||
|
||||
private async initializeOctokit() {
|
||||
if (typeof window === 'undefined') {
|
||||
// Server-side: use Octokit
|
||||
this.octokit = new Octokit({
|
||||
auth: this.token,
|
||||
userAgent: 'usage-statistics-tracker',
|
||||
timeZone: 'UTC',
|
||||
baseUrl: 'https://api.github.com',
|
||||
log: {
|
||||
debug: () => {},
|
||||
info: () => {},
|
||||
warn: console.warn,
|
||||
error: console.error
|
||||
},
|
||||
request: {
|
||||
timeout: 10000,
|
||||
retries: 3,
|
||||
retryAfterBaseValue: 1,
|
||||
retryAfterMaxValue: 60
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// Client-side: use fetch API
|
||||
this.octokit = null;
|
||||
}
|
||||
}
|
||||
|
||||
async getDownloadStats(repository: string, options?: {
|
||||
releaseTag?: string;
|
||||
assetName?: string;
|
||||
startDate?: Date;
|
||||
endDate?: Date;
|
||||
}): Promise<GitHubDownloadStats[]> {
|
||||
try {
|
||||
const [owner, repo] = repository.split('/');
|
||||
if (!owner || !repo) {
|
||||
throw new Error(`Invalid repository format: ${repository}. Expected format: owner/repo`);
|
||||
}
|
||||
|
||||
const releases = await this.getReleases(owner, repo);
|
||||
const stats: GitHubDownloadStats[] = [];
|
||||
|
||||
for (const release of releases) {
|
||||
// Filter by release tag if specified
|
||||
if (options?.releaseTag && release.tag_name !== options.releaseTag) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const asset of release.assets) {
|
||||
// Filter by asset name if specified
|
||||
if (options?.assetName && asset.name !== options.assetName) {
|
||||
continue;
|
||||
}
|
||||
|
||||
stats.push({
|
||||
platform: 'github',
|
||||
packageName: repository,
|
||||
repository,
|
||||
releaseId: release.id,
|
||||
releaseName: release.name || 'Unknown',
|
||||
releaseTag: release.tag_name,
|
||||
assetName: asset.name,
|
||||
assetId: asset.id,
|
||||
downloadCount: asset.download_count,
|
||||
timestamp: new Date(release.published_at || new Date()),
|
||||
period: 'total',
|
||||
metadata: {
|
||||
assetSize: asset.size,
|
||||
contentType: asset.content_type,
|
||||
isDraft: release.draft,
|
||||
isPrerelease: release.prerelease
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return stats;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching GitHub stats for ${repository}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async getLatestVersion(repository: string): Promise<string | null> {
|
||||
try {
|
||||
const [owner, repo] = repository.split('/');
|
||||
const releases = await this.getReleases(owner, repo);
|
||||
|
||||
// Get the latest non-draft, non-prerelease
|
||||
const latestRelease = releases.find(r => !r.draft && !r.prerelease);
|
||||
return latestRelease?.tag_name || null;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching latest version for ${repository}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async getPackageInfo(repository: string): Promise<GitHubRepositoryInfo> {
|
||||
const [owner, repo] = repository.split('/');
|
||||
|
||||
if (this.octokit) {
|
||||
// Use Octokit if available
|
||||
try {
|
||||
const response = await this.octokit.repos.get({
|
||||
owner,
|
||||
repo
|
||||
});
|
||||
return response.data;
|
||||
} catch (error: any) {
|
||||
if (error.status === 403 && error.message.includes('abuse detection')) {
|
||||
console.warn(`Rate limit hit for ${repository}, waiting 60 seconds...`);
|
||||
await this.sleep(60000);
|
||||
return this.getPackageInfo(repository); // Retry once
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
} else {
|
||||
// Fallback to fetch API
|
||||
const response = await fetch(`https://api.github.com/repos/${owner}/${repo}`, {
|
||||
headers: this.token ? {
|
||||
'Authorization': `token ${this.token}`,
|
||||
'Accept': 'application/vnd.github.v3+json'
|
||||
} : {
|
||||
'Accept': 'application/vnd.github.v3+json'
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 403) {
|
||||
console.warn(`Rate limit hit for ${repository}, waiting 60 seconds...`);
|
||||
await this.sleep(60000);
|
||||
return this.getPackageInfo(repository); // Retry once
|
||||
}
|
||||
throw new Error(`Failed to fetch repository info for ${repository}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
}
|
||||
|
||||
private async getReleases(owner: string, repo: string): Promise<GitHubReleaseInfo[]> {
|
||||
if (this.octokit) {
|
||||
// Use Octokit if available
|
||||
try {
|
||||
const response = await this.octokit.repos.listReleases({
|
||||
owner,
|
||||
repo,
|
||||
per_page: 100
|
||||
});
|
||||
return response.data;
|
||||
} catch (error: any) {
|
||||
if (error.status === 403 && error.message.includes('abuse detection')) {
|
||||
console.warn(`Rate limit hit for ${owner}/${repo}, waiting 60 seconds...`);
|
||||
await this.sleep(60000);
|
||||
return this.getReleases(owner, repo); // Retry once
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
} else {
|
||||
// Fallback to fetch API
|
||||
const response = await fetch(`https://api.github.com/repos/${owner}/${repo}/releases?per_page=100`, {
|
||||
headers: this.token ? {
|
||||
'Authorization': `token ${this.token}`,
|
||||
'Accept': 'application/vnd.github.v3+json'
|
||||
} : {
|
||||
'Accept': 'application/vnd.github.v3+json'
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 403) {
|
||||
console.warn(`Rate limit hit for ${owner}/${repo}, waiting 60 seconds...`);
|
||||
await this.sleep(60000);
|
||||
return this.getReleases(owner, repo); // Retry once
|
||||
}
|
||||
throw new Error(`Failed to fetch releases for ${owner}/${repo}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
}
|
||||
|
||||
private sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
}
|
||||
|
||||
export default GitHubTracker;
|
||||
300
src/trackers/go.ts
Normal file
300
src/trackers/go.ts
Normal file
@@ -0,0 +1,300 @@
|
||||
/**
|
||||
* Go Module Download Tracker
|
||||
* Uses Go module proxy and GitHub API to fetch download statistics
|
||||
*/
|
||||
|
||||
import type { BaseDownloadStats, PlatformTracker } from '../types';
|
||||
|
||||
export interface GoDownloadStats extends BaseDownloadStats {
|
||||
platform: 'go';
|
||||
moduleName: string;
|
||||
version: string;
|
||||
goVersion: string;
|
||||
downloadCount: number;
|
||||
publishedDate: Date;
|
||||
goModHash: string;
|
||||
}
|
||||
|
||||
export interface GoModuleInfo {
|
||||
Path: string;
|
||||
Version: string;
|
||||
Time: string;
|
||||
Main: boolean;
|
||||
GoMod: string;
|
||||
GoVersion: string;
|
||||
Retracted: boolean;
|
||||
RetractedReason?: string;
|
||||
}
|
||||
|
||||
export interface GoModuleVersions {
|
||||
Path: string;
|
||||
Versions: string[];
|
||||
Time: Record<string, string>;
|
||||
Origin: Record<string, any>;
|
||||
}
|
||||
|
||||
export interface GoModuleZipInfo {
|
||||
Path: string;
|
||||
Version: string;
|
||||
Mod: GoModuleInfo;
|
||||
Zip: {
|
||||
Hash: string;
|
||||
Size: number;
|
||||
};
|
||||
}
|
||||
|
||||
export class GoTracker implements PlatformTracker {
|
||||
name = 'go';
|
||||
private proxyUrl = 'https://proxy.golang.org';
|
||||
private githubToken?: string;
|
||||
|
||||
constructor(githubToken?: string) {
|
||||
this.githubToken = githubToken;
|
||||
}
|
||||
|
||||
async getDownloadStats(moduleName: string, options?: {
|
||||
version?: string;
|
||||
startDate?: Date;
|
||||
endDate?: Date;
|
||||
}): Promise<GoDownloadStats[]> {
|
||||
try {
|
||||
const versions = await this.getModuleVersions(moduleName);
|
||||
const stats: GoDownloadStats[] = [];
|
||||
|
||||
for (const version of versions.Versions) {
|
||||
if (options?.version && version !== options.version) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const moduleInfo = await this.getModuleInfo(moduleName, version);
|
||||
const publishedDate = new Date(moduleInfo.Time);
|
||||
|
||||
// Filter by date range if specified
|
||||
if (options?.startDate && publishedDate < options.startDate) {
|
||||
continue;
|
||||
}
|
||||
if (options?.endDate && publishedDate > options.endDate) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get download count (Go doesn't provide direct download stats)
|
||||
// We'll use GitHub stars/forks as a proxy for popularity
|
||||
const downloadCount = await this.getEstimatedDownloads(moduleName, version);
|
||||
|
||||
stats.push({
|
||||
platform: 'go',
|
||||
packageName: moduleName,
|
||||
moduleName,
|
||||
version,
|
||||
goVersion: moduleInfo.GoVersion,
|
||||
downloadCount,
|
||||
publishedDate,
|
||||
goModHash: moduleInfo.GoMod,
|
||||
timestamp: publishedDate,
|
||||
period: 'total',
|
||||
metadata: {
|
||||
isMain: moduleInfo.Main,
|
||||
isRetracted: moduleInfo.Retracted,
|
||||
retractedReason: moduleInfo.RetractedReason,
|
||||
goModHash: moduleInfo.GoMod
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return stats;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching Go stats for ${moduleName}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async getLatestVersion(moduleName: string): Promise<string | null> {
|
||||
try {
|
||||
const versions = await this.getModuleVersions(moduleName);
|
||||
return versions.Versions[versions.Versions.length - 1] || null;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching latest version for ${moduleName}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async getPackageInfo(moduleName: string): Promise<GoModuleInfo> {
|
||||
const latestVersion = await this.getLatestVersion(moduleName);
|
||||
if (!latestVersion) {
|
||||
throw new Error(`No versions found for module ${moduleName}`);
|
||||
}
|
||||
return this.getModuleInfo(moduleName, latestVersion);
|
||||
}
|
||||
|
||||
async getModuleVersions(moduleName: string): Promise<GoModuleVersions> {
|
||||
const response = await fetch(`${this.proxyUrl}/${moduleName}/@v/list`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch versions for ${moduleName}`);
|
||||
}
|
||||
|
||||
const versions = await response.text();
|
||||
const versionList = versions.trim().split('\n').filter(v => v);
|
||||
|
||||
// Get time information for each version
|
||||
const timeInfo: Record<string, string> = {};
|
||||
for (const version of versionList) {
|
||||
const timeResponse = await fetch(`${this.proxyUrl}/${moduleName}/@v/${version}.info`);
|
||||
if (timeResponse.ok) {
|
||||
const timeData = await timeResponse.json();
|
||||
timeInfo[version] = timeData.Time;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
Path: moduleName,
|
||||
Versions: versionList,
|
||||
Time: timeInfo,
|
||||
Origin: {}
|
||||
};
|
||||
}
|
||||
|
||||
async getModuleInfo(moduleName: string, version: string): Promise<GoModuleInfo> {
|
||||
const response = await fetch(`${this.proxyUrl}/${moduleName}/@v/${version}.info`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch module info for ${moduleName}@${version}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async getModuleZip(moduleName: string, version: string): Promise<GoModuleZipInfo> {
|
||||
const response = await fetch(`${this.proxyUrl}/${moduleName}/@v/${version}.zip`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch module zip for ${moduleName}@${version}`);
|
||||
}
|
||||
|
||||
// Get the mod info
|
||||
const modInfo = await this.getModuleInfo(moduleName, version);
|
||||
|
||||
return {
|
||||
Path: moduleName,
|
||||
Version: version,
|
||||
Mod: modInfo,
|
||||
Zip: {
|
||||
Hash: '', // Would need to calculate hash from response
|
||||
Size: parseInt(response.headers.get('content-length') || '0')
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private async getEstimatedDownloads(moduleName: string, version: string): Promise<number> {
|
||||
try {
|
||||
// Try to get GitHub repository info if it's a GitHub module
|
||||
if (moduleName.includes('github.com')) {
|
||||
const repoPath = moduleName.replace('github.com/', '');
|
||||
const response = await fetch(`https://api.github.com/repos/${repoPath}`, {
|
||||
headers: this.githubToken ? {
|
||||
'Authorization': `token ${this.githubToken}`,
|
||||
'Accept': 'application/vnd.github.v3+json'
|
||||
} : {
|
||||
'Accept': 'application/vnd.github.v3+json'
|
||||
}
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const repoData = await response.json();
|
||||
// Use stars + forks as a rough estimate of popularity
|
||||
return (repoData.stargazers_count || 0) + (repoData.forks_count || 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: use a simple heuristic based on version age
|
||||
const moduleInfo = await this.getModuleInfo(moduleName, version);
|
||||
const ageInDays = (Date.now() - new Date(moduleInfo.Time).getTime()) / (1000 * 60 * 60 * 24);
|
||||
return Math.max(1, Math.floor(100 / (ageInDays + 1))); // More downloads for newer versions
|
||||
} catch (error) {
|
||||
console.error(`Error estimating downloads for ${moduleName}@${version}:`, error);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
async searchModules(query: string): Promise<{
|
||||
modules: Array<{
|
||||
path: string;
|
||||
version: string;
|
||||
time: string;
|
||||
}>;
|
||||
}> {
|
||||
try {
|
||||
// Go doesn't have a built-in search API, but we can search GitHub for Go modules
|
||||
const response = await fetch(`https://api.github.com/search/repositories?q=${query}+language:go&sort=stars&order=desc`, {
|
||||
headers: this.githubToken ? {
|
||||
'Authorization': `token ${this.githubToken}`,
|
||||
'Accept': 'application/vnd.github.v3+json'
|
||||
} : {
|
||||
'Accept': 'application/vnd.github.v3+json'
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to search Go modules');
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
return {
|
||||
modules: data.items.map((repo: any) => ({
|
||||
path: `github.com/${repo.full_name}`,
|
||||
version: 'latest',
|
||||
time: repo.created_at
|
||||
}))
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Error searching Go modules:', error);
|
||||
return { modules: [] };
|
||||
}
|
||||
}
|
||||
|
||||
async getModuleAnalytics(moduleName: string): Promise<{
|
||||
totalVersions: number;
|
||||
latestVersion: string;
|
||||
firstPublished: Date;
|
||||
lastPublished: Date;
|
||||
estimatedTotalDownloads: number;
|
||||
}> {
|
||||
try {
|
||||
const versions = await this.getModuleVersions(moduleName);
|
||||
const latestVersion = versions.Versions[versions.Versions.length - 1];
|
||||
|
||||
let firstPublished = new Date();
|
||||
let lastPublished = new Date(0);
|
||||
let totalDownloads = 0;
|
||||
|
||||
for (const version of versions.Versions) {
|
||||
const moduleInfo = await this.getModuleInfo(moduleName, version);
|
||||
const publishedDate = new Date(moduleInfo.Time);
|
||||
|
||||
if (publishedDate < firstPublished) {
|
||||
firstPublished = publishedDate;
|
||||
}
|
||||
if (publishedDate > lastPublished) {
|
||||
lastPublished = publishedDate;
|
||||
}
|
||||
|
||||
totalDownloads += await this.getEstimatedDownloads(moduleName, version);
|
||||
}
|
||||
|
||||
return {
|
||||
totalVersions: versions.Versions.length,
|
||||
latestVersion: latestVersion || '',
|
||||
firstPublished,
|
||||
lastPublished,
|
||||
estimatedTotalDownloads: totalDownloads
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`Error fetching analytics for ${moduleName}:`, error);
|
||||
return {
|
||||
totalVersions: 0,
|
||||
latestVersion: '',
|
||||
firstPublished: new Date(),
|
||||
lastPublished: new Date(),
|
||||
estimatedTotalDownloads: 0
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default GoTracker;
|
||||
203
src/trackers/homebrew.ts
Normal file
203
src/trackers/homebrew.ts
Normal file
@@ -0,0 +1,203 @@
|
||||
/**
|
||||
* Homebrew Package Download Tracker
|
||||
* Uses Homebrew API and GitHub API to fetch download statistics
|
||||
*/
|
||||
|
||||
import type { BaseDownloadStats, PlatformTracker } from '../types';
|
||||
|
||||
export interface HomebrewDownloadStats extends BaseDownloadStats {
|
||||
platform: 'homebrew';
|
||||
formulaName: string;
|
||||
tapName: string;
|
||||
version: string;
|
||||
installCount: number;
|
||||
analyticsData?: {
|
||||
installEvents: number;
|
||||
buildErrors: number;
|
||||
osVersion: string;
|
||||
rubyVersion: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface HomebrewFormulaInfo {
|
||||
name: string;
|
||||
full_name: string;
|
||||
desc?: string;
|
||||
homepage?: string;
|
||||
version: string;
|
||||
installed: number[];
|
||||
dependencies: string[];
|
||||
conflicts: string[];
|
||||
caveats?: string;
|
||||
analytics?: {
|
||||
install: {
|
||||
'30d': Record<string, number>;
|
||||
'90d': Record<string, number>;
|
||||
'365d': Record<string, number>;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
export interface HomebrewTapInfo {
|
||||
name: string;
|
||||
full_name: string;
|
||||
description?: string;
|
||||
homepage?: string;
|
||||
url: string;
|
||||
clone_url: string;
|
||||
default_branch: string;
|
||||
stargazers_count: number;
|
||||
forks_count: number;
|
||||
}
|
||||
|
||||
export class HomebrewTracker implements PlatformTracker {
|
||||
name = 'homebrew';
|
||||
private baseUrl = 'https://formulae.brew.sh/api';
|
||||
private githubToken?: string;
|
||||
|
||||
constructor(githubToken?: string) {
|
||||
this.githubToken = githubToken;
|
||||
}
|
||||
|
||||
async getDownloadStats(formulaName: string, options?: {
|
||||
tapName?: string;
|
||||
period?: '30d' | '90d' | '365d';
|
||||
startDate?: Date;
|
||||
endDate?: Date;
|
||||
}): Promise<HomebrewDownloadStats[]> {
|
||||
try {
|
||||
const formulaInfo = await this.getPackageInfo(formulaName);
|
||||
const stats: HomebrewDownloadStats[] = [];
|
||||
|
||||
// Get analytics data if available
|
||||
if (formulaInfo.analytics?.install) {
|
||||
const period = options?.period || '30d';
|
||||
const analytics = formulaInfo.analytics.install[period];
|
||||
|
||||
if (analytics) {
|
||||
const totalInstalls = Object.values(analytics).reduce((sum, count) => sum + count, 0);
|
||||
|
||||
stats.push({
|
||||
platform: 'homebrew',
|
||||
packageName: formulaName,
|
||||
formulaName,
|
||||
tapName: this.getTapName(formulaName),
|
||||
version: formulaInfo.version,
|
||||
installCount: totalInstalls,
|
||||
downloadCount: totalInstalls, // For compatibility with BaseDownloadStats
|
||||
timestamp: new Date(),
|
||||
period: 'total',
|
||||
metadata: {
|
||||
analyticsPeriod: period,
|
||||
analyticsData: analytics,
|
||||
dependencies: formulaInfo.dependencies,
|
||||
conflicts: formulaInfo.conflicts
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// If no analytics available, create a basic stat entry
|
||||
if (stats.length === 0) {
|
||||
stats.push({
|
||||
platform: 'homebrew',
|
||||
packageName: formulaName,
|
||||
formulaName,
|
||||
tapName: this.getTapName(formulaName),
|
||||
version: formulaInfo.version,
|
||||
installCount: formulaInfo.installed.length,
|
||||
downloadCount: formulaInfo.installed.length,
|
||||
timestamp: new Date(),
|
||||
period: 'total',
|
||||
metadata: {
|
||||
installedVersions: formulaInfo.installed,
|
||||
dependencies: formulaInfo.dependencies,
|
||||
conflicts: formulaInfo.conflicts
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return stats;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching Homebrew stats for ${formulaName}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async getLatestVersion(formulaName: string): Promise<string | null> {
|
||||
try {
|
||||
const formulaInfo = await this.getPackageInfo(formulaName);
|
||||
return formulaInfo.version || null;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching latest version for ${formulaName}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async getPackageInfo(formulaName: string): Promise<HomebrewFormulaInfo> {
|
||||
const response = await fetch(`${this.baseUrl}/formula/${formulaName}.json`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch formula info for ${formulaName}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async getTapInfo(tapName: string): Promise<HomebrewTapInfo> {
|
||||
// Homebrew taps are GitHub repositories
|
||||
const response = await fetch(`https://api.github.com/repos/Homebrew/${tapName}`, {
|
||||
headers: this.githubToken ? {
|
||||
'Authorization': `token ${this.githubToken}`,
|
||||
'Accept': 'application/vnd.github.v3+json'
|
||||
} : {
|
||||
'Accept': 'application/vnd.github.v3+json'
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch tap info for ${tapName}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async getAllFormulae(): Promise<string[]> {
|
||||
try {
|
||||
const response = await fetch(`${this.baseUrl}/formula.json`);
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to fetch all formulae');
|
||||
}
|
||||
|
||||
const formulae = await response.json();
|
||||
return formulae.map((formula: any) => formula.name);
|
||||
} catch (error) {
|
||||
console.error('Error fetching all formulae:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async getAnalytics(formulaName: string, period: '30d' | '90d' | '365d' = '30d'): Promise<{
|
||||
date: string;
|
||||
installs: number;
|
||||
}[]> {
|
||||
try {
|
||||
const response = await fetch(`${this.baseUrl}/analytics/install/${period}/${formulaName}.json`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch analytics for ${formulaName}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
return data.analytics || [];
|
||||
} catch (error) {
|
||||
console.error(`Error fetching analytics for ${formulaName}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private getTapName(formulaName: string): string {
|
||||
// Most formulae are in the homebrew/core tap
|
||||
// This is a simplified implementation
|
||||
return 'homebrew/core';
|
||||
}
|
||||
}
|
||||
|
||||
export default HomebrewTracker;
|
||||
111
src/trackers/npm.ts
Normal file
111
src/trackers/npm.ts
Normal file
@@ -0,0 +1,111 @@
|
||||
/**
|
||||
* NPM Package Download Tracker
|
||||
* Uses the npm registry API to fetch download statistics
|
||||
*/
|
||||
|
||||
import type { BaseDownloadStats, PlatformTracker } from '../types';
|
||||
|
||||
export interface NpmDownloadStats extends BaseDownloadStats {
|
||||
platform: 'npm';
|
||||
registry: string;
|
||||
distTags?: Record<string, string>;
|
||||
dependencies?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface NpmPackageInfo {
|
||||
name: string;
|
||||
version: string;
|
||||
description?: string;
|
||||
homepage?: string;
|
||||
repository?: {
|
||||
type: string;
|
||||
url: string;
|
||||
};
|
||||
distTags: Record<string, string>;
|
||||
time: Record<string, string>;
|
||||
versions: Record<string, any>;
|
||||
}
|
||||
|
||||
export class NpmTracker implements PlatformTracker {
|
||||
name = 'npm';
|
||||
private baseUrl = 'https://registry.npmjs.org';
|
||||
|
||||
async getDownloadStats(packageName: string, options?: {
|
||||
period?: 'daily' | 'weekly' | 'monthly' | 'total';
|
||||
startDate?: Date;
|
||||
endDate?: Date;
|
||||
}): Promise<NpmDownloadStats[]> {
|
||||
try {
|
||||
// Get package info
|
||||
const packageInfo = await this.getPackageInfo(packageName);
|
||||
|
||||
// Get download stats from npm registry
|
||||
const stats = await this.fetchDownloadStats(packageName, options);
|
||||
|
||||
return stats.map(stat => ({
|
||||
...stat,
|
||||
platform: 'npm' as const,
|
||||
registry: this.baseUrl,
|
||||
distTags: packageInfo.distTags,
|
||||
dependencies: packageInfo.versions[packageInfo.distTags?.latest]?.dependencies
|
||||
}));
|
||||
} catch (error) {
|
||||
console.error(`Error fetching NPM stats for ${packageName}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async getLatestVersion(packageName: string): Promise<string | null> {
|
||||
try {
|
||||
const packageInfo = await this.getPackageInfo(packageName);
|
||||
return packageInfo.distTags?.latest || null;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching latest version for ${packageName}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async getPackageInfo(packageName: string): Promise<NpmPackageInfo> {
|
||||
const response = await fetch(`${this.baseUrl}/${packageName}`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch package info for ${packageName}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
private async fetchDownloadStats(packageName: string, options?: {
|
||||
period?: 'daily' | 'weekly' | 'monthly' | 'total';
|
||||
startDate?: Date;
|
||||
endDate?: Date;
|
||||
}): Promise<NpmDownloadStats[]> {
|
||||
// Note: NPM registry doesn't provide direct download stats via API
|
||||
// This would typically require using npm-stat.com or similar services
|
||||
// For now, we'll return a placeholder structure
|
||||
|
||||
const now = new Date();
|
||||
const stats: NpmDownloadStats[] = [];
|
||||
|
||||
// Simulate daily stats for the last 30 days
|
||||
for (let i = 0; i < 30; i++) {
|
||||
const date = new Date(now);
|
||||
date.setDate(date.getDate() - i);
|
||||
|
||||
stats.push({
|
||||
platform: 'npm',
|
||||
packageName,
|
||||
downloadCount: Math.floor(Math.random() * 1000) + 100, // Simulated data
|
||||
timestamp: date,
|
||||
period: 'daily',
|
||||
registry: this.baseUrl,
|
||||
metadata: {
|
||||
source: 'npm-registry',
|
||||
simulated: true
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
}
|
||||
|
||||
export default NpmTracker;
|
||||
288
src/trackers/postman.ts
Normal file
288
src/trackers/postman.ts
Normal file
@@ -0,0 +1,288 @@
|
||||
/**
|
||||
* Postman Collection Download/Fork Tracker
|
||||
* Uses Postman API to fetch collection statistics
|
||||
*/
|
||||
|
||||
import type { BaseDownloadStats, PlatformTracker } from '../types';
|
||||
|
||||
export interface PostmanDownloadStats extends BaseDownloadStats {
|
||||
platform: 'postman';
|
||||
collectionId: string;
|
||||
collectionName: string;
|
||||
version: string;
|
||||
forkCount: number;
|
||||
downloadCount: number;
|
||||
viewCount: number;
|
||||
author: string;
|
||||
publishedDate: Date;
|
||||
}
|
||||
|
||||
export interface PostmanCollectionInfo {
|
||||
id: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
version: string;
|
||||
author: {
|
||||
id: string;
|
||||
name: string;
|
||||
username: string;
|
||||
};
|
||||
publishedAt: string;
|
||||
updatedAt: string;
|
||||
forkCount: number;
|
||||
downloadCount: number;
|
||||
viewCount: number;
|
||||
schema: string;
|
||||
info: {
|
||||
name: string;
|
||||
description?: string;
|
||||
version: string;
|
||||
schema: string;
|
||||
};
|
||||
item: any[];
|
||||
variable: any[];
|
||||
}
|
||||
|
||||
export interface PostmanWorkspaceInfo {
|
||||
id: string;
|
||||
name: string;
|
||||
type: 'personal' | 'team' | 'private';
|
||||
description?: string;
|
||||
collections: PostmanCollectionInfo[];
|
||||
}
|
||||
|
||||
export class PostmanTracker implements PlatformTracker {
|
||||
name = 'postman';
|
||||
private baseUrl = 'https://api.getpostman.com';
|
||||
private apiKey?: string;
|
||||
|
||||
constructor(apiKey?: string) {
|
||||
this.apiKey = apiKey;
|
||||
}
|
||||
|
||||
async getDownloadStats(collectionId: string, options?: {
|
||||
version?: string;
|
||||
startDate?: Date;
|
||||
endDate?: Date;
|
||||
}): Promise<PostmanDownloadStats[]> {
|
||||
try {
|
||||
const collectionInfo = await this.getPackageInfo(collectionId);
|
||||
const stats: PostmanDownloadStats[] = [];
|
||||
|
||||
// Get collection versions if available
|
||||
const versions = await this.getCollectionVersions(collectionId);
|
||||
|
||||
for (const version of versions) {
|
||||
if (options?.version && version.version !== options.version) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const publishedDate = new Date(version.publishedAt);
|
||||
|
||||
// Filter by date range if specified
|
||||
if (options?.startDate && publishedDate < options.startDate) {
|
||||
continue;
|
||||
}
|
||||
if (options?.endDate && publishedDate > options.endDate) {
|
||||
continue;
|
||||
}
|
||||
|
||||
stats.push({
|
||||
platform: 'postman',
|
||||
packageName: collectionId,
|
||||
collectionId,
|
||||
collectionName: version.name,
|
||||
version: version.version,
|
||||
forkCount: version.forkCount || 0,
|
||||
downloadCount: version.downloadCount || 0,
|
||||
viewCount: version.viewCount || 0,
|
||||
author: version.author?.name || 'Unknown',
|
||||
publishedDate,
|
||||
timestamp: publishedDate,
|
||||
period: 'total',
|
||||
metadata: {
|
||||
authorId: version.author?.id,
|
||||
authorUsername: version.author?.username,
|
||||
schema: version.schema,
|
||||
itemCount: version.item?.length || 0
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return stats;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching Postman stats for ${collectionId}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async getLatestVersion(collectionId: string): Promise<string | null> {
|
||||
try {
|
||||
const collectionInfo = await this.getPackageInfo(collectionId);
|
||||
return collectionInfo.version || null;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching latest version for ${collectionId}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async getPackageInfo(collectionId: string): Promise<PostmanCollectionInfo> {
|
||||
const headers: Record<string, string> = {
|
||||
'Accept': 'application/json'
|
||||
};
|
||||
|
||||
if (this.apiKey) {
|
||||
headers['X-API-Key'] = this.apiKey;
|
||||
}
|
||||
|
||||
const response = await fetch(`${this.baseUrl}/collections/${collectionId}`, {
|
||||
headers
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch collection info for ${collectionId}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async getCollectionVersions(collectionId: string): Promise<PostmanCollectionInfo[]> {
|
||||
try {
|
||||
const headers: Record<string, string> = {
|
||||
'Accept': 'application/json'
|
||||
};
|
||||
|
||||
if (this.apiKey) {
|
||||
headers['X-API-Key'] = this.apiKey;
|
||||
}
|
||||
|
||||
const response = await fetch(`${this.baseUrl}/collections/${collectionId}/versions`, {
|
||||
headers
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch versions for ${collectionId}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
} catch (error) {
|
||||
console.error(`Error fetching versions for ${collectionId}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async searchCollections(query: string, options?: {
|
||||
workspaceId?: string;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
}): Promise<{
|
||||
collections: PostmanCollectionInfo[];
|
||||
totalCount: number;
|
||||
}> {
|
||||
try {
|
||||
const headers: Record<string, string> = {
|
||||
'Accept': 'application/json'
|
||||
};
|
||||
|
||||
if (this.apiKey) {
|
||||
headers['X-API-Key'] = this.apiKey;
|
||||
}
|
||||
|
||||
const params = new URLSearchParams({
|
||||
q: query,
|
||||
limit: (options?.limit || 50).toString(),
|
||||
offset: (options?.offset || 0).toString()
|
||||
});
|
||||
|
||||
if (options?.workspaceId) {
|
||||
params.set('workspace', options.workspaceId);
|
||||
}
|
||||
|
||||
const response = await fetch(`${this.baseUrl}/search?${params}`, {
|
||||
headers
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to search collections');
|
||||
}
|
||||
|
||||
return response.json();
|
||||
} catch (error) {
|
||||
console.error('Error searching collections:', error);
|
||||
return {
|
||||
collections: [],
|
||||
totalCount: 0
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async getWorkspaceCollections(workspaceId: string): Promise<PostmanCollectionInfo[]> {
|
||||
try {
|
||||
const headers: Record<string, string> = {
|
||||
'Accept': 'application/json'
|
||||
};
|
||||
|
||||
if (this.apiKey) {
|
||||
headers['X-API-Key'] = this.apiKey;
|
||||
}
|
||||
|
||||
const response = await fetch(`${this.baseUrl}/workspaces/${workspaceId}/collections`, {
|
||||
headers
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch workspace collections for ${workspaceId}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
} catch (error) {
|
||||
console.error(`Error fetching workspace collections for ${workspaceId}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async getCollectionAnalytics(collectionId: string): Promise<{
|
||||
totalDownloads: number;
|
||||
totalForks: number;
|
||||
totalViews: number;
|
||||
downloadsByVersion: Record<string, number>;
|
||||
forksByVersion: Record<string, number>;
|
||||
}> {
|
||||
try {
|
||||
const versions = await this.getCollectionVersions(collectionId);
|
||||
|
||||
const downloadsByVersion: Record<string, number> = {};
|
||||
const forksByVersion: Record<string, number> = {};
|
||||
let totalDownloads = 0;
|
||||
let totalForks = 0;
|
||||
let totalViews = 0;
|
||||
|
||||
for (const version of versions) {
|
||||
downloadsByVersion[version.version] = version.downloadCount || 0;
|
||||
forksByVersion[version.version] = version.forkCount || 0;
|
||||
totalDownloads += version.downloadCount || 0;
|
||||
totalForks += version.forkCount || 0;
|
||||
totalViews += version.viewCount || 0;
|
||||
}
|
||||
|
||||
return {
|
||||
totalDownloads,
|
||||
totalForks,
|
||||
totalViews,
|
||||
downloadsByVersion,
|
||||
forksByVersion
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`Error fetching analytics for ${collectionId}:`, error);
|
||||
return {
|
||||
totalDownloads: 0,
|
||||
totalForks: 0,
|
||||
totalViews: 0,
|
||||
downloadsByVersion: {},
|
||||
forksByVersion: {}
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default PostmanTracker;
|
||||
214
src/trackers/powershell.ts
Normal file
214
src/trackers/powershell.ts
Normal file
@@ -0,0 +1,214 @@
|
||||
/**
|
||||
* PowerShell Gallery Module Download Tracker
|
||||
* Uses PowerShell Gallery API to fetch download statistics
|
||||
*/
|
||||
|
||||
import type { BaseDownloadStats, PlatformTracker } from '../types';
|
||||
|
||||
export interface PowerShellDownloadStats extends BaseDownloadStats {
|
||||
platform: 'powershell';
|
||||
moduleName: string;
|
||||
version: string;
|
||||
author: string;
|
||||
description?: string;
|
||||
tags: string[];
|
||||
downloadCount: number;
|
||||
publishedDate: Date;
|
||||
}
|
||||
|
||||
export interface PowerShellModuleInfo {
|
||||
Id: string;
|
||||
Version: string;
|
||||
Title: string;
|
||||
Author: string;
|
||||
Description: string;
|
||||
Tags: string[];
|
||||
PublishedDate: string;
|
||||
UpdatedDate: string;
|
||||
DownloadCount: number;
|
||||
IsLatestVersion: boolean;
|
||||
Dependencies: Array<{
|
||||
id: string;
|
||||
version: string;
|
||||
}>;
|
||||
PowerShellVersion: string;
|
||||
ProjectUri?: string;
|
||||
LicenseUri?: string;
|
||||
IconUri?: string;
|
||||
ReleaseNotes?: string;
|
||||
}
|
||||
|
||||
export interface PowerShellSearchResult {
|
||||
TotalCount: number;
|
||||
Results: PowerShellModuleInfo[];
|
||||
}
|
||||
|
||||
export class PowerShellTracker implements PlatformTracker {
|
||||
name = 'powershell';
|
||||
private baseUrl = 'https://www.powershellgallery.com/api/v2';
|
||||
|
||||
async getDownloadStats(moduleName: string, options?: {
|
||||
version?: string;
|
||||
startDate?: Date;
|
||||
endDate?: Date;
|
||||
}): Promise<PowerShellDownloadStats[]> {
|
||||
try {
|
||||
const moduleInfo = await this.getPackageInfo(moduleName);
|
||||
const stats: PowerShellDownloadStats[] = [];
|
||||
|
||||
// Get all versions of the module
|
||||
const allVersions = await this.getAllVersions(moduleName);
|
||||
|
||||
for (const version of allVersions) {
|
||||
if (options?.version && version.Version !== options.version) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const publishedDate = new Date(version.PublishedDate);
|
||||
|
||||
// Filter by date range if specified
|
||||
if (options?.startDate && publishedDate < options.startDate) {
|
||||
continue;
|
||||
}
|
||||
if (options?.endDate && publishedDate > options.endDate) {
|
||||
continue;
|
||||
}
|
||||
|
||||
stats.push({
|
||||
platform: 'powershell',
|
||||
packageName: moduleName,
|
||||
moduleName,
|
||||
version: version.Version,
|
||||
author: version.Author,
|
||||
description: version.Description,
|
||||
tags: version.Tags,
|
||||
downloadCount: version.DownloadCount,
|
||||
publishedDate,
|
||||
timestamp: publishedDate,
|
||||
period: 'total',
|
||||
metadata: {
|
||||
isLatestVersion: version.IsLatestVersion,
|
||||
dependencies: version.Dependencies,
|
||||
powershellVersion: version.PowerShellVersion,
|
||||
projectUri: version.ProjectUri,
|
||||
licenseUri: version.LicenseUri,
|
||||
releaseNotes: version.ReleaseNotes
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return stats;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching PowerShell stats for ${moduleName}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async getLatestVersion(moduleName: string): Promise<string | null> {
|
||||
try {
|
||||
const moduleInfo = await this.getPackageInfo(moduleName);
|
||||
return moduleInfo.Version || null;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching latest version for ${moduleName}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async getPackageInfo(moduleName: string): Promise<PowerShellModuleInfo> {
|
||||
const response = await fetch(`${this.baseUrl}/package/${moduleName}`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch module info for ${moduleName}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async getAllVersions(moduleName: string): Promise<PowerShellModuleInfo[]> {
|
||||
try {
|
||||
const response = await fetch(`${this.baseUrl}/package/${moduleName}/versions`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch versions for ${moduleName}`);
|
||||
}
|
||||
return response.json();
|
||||
} catch (error) {
|
||||
console.error(`Error fetching versions for ${moduleName}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async searchModules(query: string, options?: {
|
||||
includePrerelease?: boolean;
|
||||
skip?: number;
|
||||
take?: number;
|
||||
}): Promise<PowerShellSearchResult> {
|
||||
const params = new URLSearchParams({
|
||||
$filter: `IsLatestVersion eq true`,
|
||||
$orderby: 'DownloadCount desc',
|
||||
$skip: (options?.skip || 0).toString(),
|
||||
$top: (options?.take || 50).toString()
|
||||
});
|
||||
|
||||
if (query) {
|
||||
params.set('$filter', `${params.get('$filter')} and substringof('${query}', Id)`);
|
||||
}
|
||||
|
||||
const response = await fetch(`${this.baseUrl}/search?${params}`);
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to search modules');
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async getPopularModules(limit: number = 50): Promise<PowerShellModuleInfo[]> {
|
||||
try {
|
||||
const searchResult = await this.searchModules('', { take: limit });
|
||||
return searchResult.Results;
|
||||
} catch (error) {
|
||||
console.error('Error fetching popular modules:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async getModuleAnalytics(moduleName: string): Promise<{
|
||||
totalDownloads: number;
|
||||
downloadsByVersion: Record<string, number>;
|
||||
downloadsByDate: Array<{
|
||||
date: string;
|
||||
downloads: number;
|
||||
}>;
|
||||
}> {
|
||||
try {
|
||||
const allVersions = await this.getAllVersions(moduleName);
|
||||
|
||||
const downloadsByVersion: Record<string, number> = {};
|
||||
let totalDownloads = 0;
|
||||
|
||||
for (const version of allVersions) {
|
||||
downloadsByVersion[version.Version] = version.DownloadCount;
|
||||
totalDownloads += version.DownloadCount;
|
||||
}
|
||||
|
||||
// Note: PowerShell Gallery doesn't provide detailed time-based analytics
|
||||
// This is a simplified implementation
|
||||
const downloadsByDate = allVersions.map(version => ({
|
||||
date: version.PublishedDate,
|
||||
downloads: version.DownloadCount
|
||||
}));
|
||||
|
||||
return {
|
||||
totalDownloads,
|
||||
downloadsByVersion,
|
||||
downloadsByDate
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`Error fetching analytics for ${moduleName}:`, error);
|
||||
return {
|
||||
totalDownloads: 0,
|
||||
downloadsByVersion: {},
|
||||
downloadsByDate: []
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default PowerShellTracker;
|
||||
148
src/trackers/pypi.ts
Normal file
148
src/trackers/pypi.ts
Normal file
@@ -0,0 +1,148 @@
|
||||
/**
|
||||
* PyPI Package Download Tracker
|
||||
* Uses PyPI JSON API to fetch download statistics
|
||||
*/
|
||||
|
||||
import type { BaseDownloadStats, PlatformTracker } from '../types';
|
||||
|
||||
export interface PyPiDownloadStats extends BaseDownloadStats {
|
||||
platform: 'pypi';
|
||||
packageName: string;
|
||||
version: string;
|
||||
fileType: string;
|
||||
pythonVersion?: string;
|
||||
uploadTime: Date;
|
||||
}
|
||||
|
||||
export interface PyPiPackageInfo {
|
||||
info: {
|
||||
name: string;
|
||||
version: string;
|
||||
summary?: string;
|
||||
description?: string;
|
||||
home_page?: string;
|
||||
author?: string;
|
||||
author_email?: string;
|
||||
license?: string;
|
||||
requires_python?: string;
|
||||
project_urls?: Record<string, string>;
|
||||
};
|
||||
releases: Record<string, Array<{
|
||||
filename: string;
|
||||
url: string;
|
||||
size: number;
|
||||
upload_time: string;
|
||||
file_type: string;
|
||||
python_version?: string;
|
||||
download_count?: number;
|
||||
}>>;
|
||||
urls: Array<{
|
||||
filename: string;
|
||||
url: string;
|
||||
size: number;
|
||||
upload_time: string;
|
||||
file_type: string;
|
||||
python_version?: string;
|
||||
download_count?: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
export class PyPiTracker implements PlatformTracker {
|
||||
name = 'pypi';
|
||||
private baseUrl = 'https://pypi.org/pypi';
|
||||
|
||||
async getDownloadStats(packageName: string, options?: {
|
||||
version?: string;
|
||||
fileType?: string;
|
||||
startDate?: Date;
|
||||
endDate?: Date;
|
||||
}): Promise<PyPiDownloadStats[]> {
|
||||
try {
|
||||
const packageInfo = await this.getPackageInfo(packageName);
|
||||
const stats: PyPiDownloadStats[] = [];
|
||||
|
||||
// Process releases
|
||||
for (const [version, files] of Object.entries(packageInfo.releases)) {
|
||||
if (options?.version && version !== options.version) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const file of files) {
|
||||
if (options?.fileType && file.file_type !== options.fileType) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Filter by date range if specified
|
||||
const uploadTime = new Date(file.upload_time);
|
||||
if (options?.startDate && uploadTime < options.startDate) {
|
||||
continue;
|
||||
}
|
||||
if (options?.endDate && uploadTime > options.endDate) {
|
||||
continue;
|
||||
}
|
||||
|
||||
stats.push({
|
||||
platform: 'pypi',
|
||||
packageName,
|
||||
version,
|
||||
fileType: file.file_type,
|
||||
pythonVersion: file.python_version,
|
||||
uploadTime,
|
||||
downloadCount: file.download_count || 0,
|
||||
timestamp: uploadTime,
|
||||
period: 'total',
|
||||
metadata: {
|
||||
filename: file.filename,
|
||||
fileSize: file.size,
|
||||
url: file.url
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return stats;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching PyPI stats for ${packageName}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async getLatestVersion(packageName: string): Promise<string | null> {
|
||||
try {
|
||||
const packageInfo = await this.getPackageInfo(packageName);
|
||||
return packageInfo.info.version || null;
|
||||
} catch (error) {
|
||||
console.error(`Error fetching latest version for ${packageName}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async getPackageInfo(packageName: string): Promise<PyPiPackageInfo> {
|
||||
const response = await fetch(`${this.baseUrl}/${packageName}/json`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch package info for ${packageName}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async getDownloadCounts(packageName: string, period: 'day' | 'week' | 'month' = 'month'): Promise<{
|
||||
date: string;
|
||||
downloads: number;
|
||||
}[]> {
|
||||
try {
|
||||
// PyPI provides download stats via a separate endpoint
|
||||
const response = await fetch(`https://pypi.org/pypi/${packageName}/stats/${period}`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch download stats for ${packageName}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
return data.data || [];
|
||||
} catch (error) {
|
||||
console.error(`Error fetching download counts for ${packageName}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default PyPiTracker;
|
||||
41
src/types/index.ts
Normal file
41
src/types/index.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
/**
|
||||
* Shared types for usage statistics tracking across all platforms
|
||||
*/
|
||||
|
||||
export interface BaseDownloadStats {
|
||||
platform: string;
|
||||
packageName: string;
|
||||
version?: string;
|
||||
downloadCount: number;
|
||||
timestamp: Date;
|
||||
period?: 'daily' | 'weekly' | 'monthly' | 'total';
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
|
||||
export interface PlatformTracker {
|
||||
name: string;
|
||||
getDownloadStats(packageName: string, options?: any): Promise<BaseDownloadStats[]>;
|
||||
getLatestVersion(packageName: string): Promise<string | null>;
|
||||
getPackageInfo(packageName: string): Promise<any>;
|
||||
}
|
||||
|
||||
export interface DownloadStatsAggregator {
|
||||
aggregateStats(stats: BaseDownloadStats[]): {
|
||||
totalDownloads: number;
|
||||
uniquePackages: number;
|
||||
platforms: string[];
|
||||
timeRange: { start: Date; end: Date } | null;
|
||||
};
|
||||
}
|
||||
|
||||
export interface TrackingConfig {
|
||||
npmPackages?: string[];
|
||||
goModules?: string[];
|
||||
pythonPackages?: string[];
|
||||
powershellModules?: string[];
|
||||
homebrewPackages?: string[];
|
||||
githubRepos?: string[];
|
||||
postmanCollections?: string[];
|
||||
updateInterval?: number; // in milliseconds
|
||||
enableLogging?: boolean;
|
||||
}
|
||||
140
src/utils/rate-limiter.ts
Normal file
140
src/utils/rate-limiter.ts
Normal file
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* Rate Limiting Utility
|
||||
* Handles API rate limits with exponential backoff and proper error handling
|
||||
*/
|
||||
|
||||
export interface RateLimitConfig {
|
||||
maxRetries: number;
|
||||
baseDelay: number;
|
||||
maxDelay: number;
|
||||
backoffMultiplier: number;
|
||||
}
|
||||
|
||||
export class RateLimiter {
|
||||
private config: RateLimitConfig;
|
||||
private requestCounts: Map<string, { count: number; resetTime: number }> = new Map();
|
||||
|
||||
constructor(config: Partial<RateLimitConfig> = {}) {
|
||||
this.config = {
|
||||
maxRetries: config.maxRetries || 3,
|
||||
baseDelay: config.baseDelay || 1000,
|
||||
maxDelay: config.maxDelay || 60000,
|
||||
backoffMultiplier: config.backoffMultiplier || 2
|
||||
};
|
||||
}
|
||||
|
||||
async executeWithRetry<T>(
|
||||
operation: () => Promise<T>,
|
||||
operationName: string,
|
||||
retryCount = 0
|
||||
): Promise<T> {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error: any) {
|
||||
if (this.shouldRetry(error) && retryCount < this.config.maxRetries) {
|
||||
const delay = this.calculateDelay(retryCount);
|
||||
console.warn(`Rate limit hit for ${operationName}, retrying in ${delay}ms (attempt ${retryCount + 1}/${this.config.maxRetries})`);
|
||||
|
||||
await this.sleep(delay);
|
||||
return this.executeWithRetry(operation, operationName, retryCount + 1);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private shouldRetry(error: any): boolean {
|
||||
// Check for rate limiting errors
|
||||
if (error.status === 403 || error.status === 429) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check for abuse detection
|
||||
if (error.message && error.message.includes('abuse detection')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check for rate limit headers
|
||||
if (error.headers && (error.headers['x-ratelimit-remaining'] === '0' || error.headers['retry-after'])) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private calculateDelay(retryCount: number): number {
|
||||
const delay = this.config.baseDelay * Math.pow(this.config.backoffMultiplier, retryCount);
|
||||
return Math.min(delay, this.config.maxDelay);
|
||||
}
|
||||
|
||||
private sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async throttleRequests<T>(
|
||||
operations: Array<() => Promise<T>>,
|
||||
maxConcurrent = 3,
|
||||
delayBetweenRequests = 1000
|
||||
): Promise<T[]> {
|
||||
const results: T[] = [];
|
||||
|
||||
for (let i = 0; i < operations.length; i += maxConcurrent) {
|
||||
const batch = operations.slice(i, i + maxConcurrent);
|
||||
const batchPromises = batch.map(async (operation, index) => {
|
||||
// Add delay between requests in the same batch
|
||||
if (index > 0) {
|
||||
await this.sleep(delayBetweenRequests);
|
||||
}
|
||||
return this.executeWithRetry(operation, `batch-${i}-${index}`);
|
||||
});
|
||||
|
||||
const batchResults = await Promise.all(batchPromises);
|
||||
results.push(...batchResults);
|
||||
|
||||
// Add delay between batches
|
||||
if (i + maxConcurrent < operations.length) {
|
||||
await this.sleep(delayBetweenRequests * 2);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// Track rate limits for different APIs
|
||||
trackRequest(apiName: string, resetTime?: number): void {
|
||||
const now = Date.now();
|
||||
const current = this.requestCounts.get(apiName) || { count: 0, resetTime: now + 3600000 }; // Default 1 hour
|
||||
|
||||
current.count++;
|
||||
|
||||
if (resetTime) {
|
||||
current.resetTime = resetTime * 1000; // Convert to milliseconds
|
||||
}
|
||||
|
||||
this.requestCounts.set(apiName, current);
|
||||
}
|
||||
|
||||
isRateLimited(apiName: string): boolean {
|
||||
const current = this.requestCounts.get(apiName);
|
||||
if (!current) return false;
|
||||
|
||||
const now = Date.now();
|
||||
if (now > current.resetTime) {
|
||||
this.requestCounts.delete(apiName);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Conservative estimate - assume we're rate limited if we've made many requests recently
|
||||
return current.count > 50;
|
||||
}
|
||||
|
||||
getRemainingTime(apiName: string): number {
|
||||
const current = this.requestCounts.get(apiName);
|
||||
if (!current) return 0;
|
||||
|
||||
const now = Date.now();
|
||||
return Math.max(0, current.resetTime - now);
|
||||
}
|
||||
}
|
||||
|
||||
// Global rate limiter instance
|
||||
export const globalRateLimiter = new RateLimiter();
|
||||
22
tsconfig.json
Normal file
22
tsconfig.json
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"noEmit": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"esModuleInterop": true,
|
||||
"allowJs": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"lib": ["ESNext", "DOM"],
|
||||
"types": ["bun-types", "node"]
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
Reference in New Issue
Block a user