agent-park/scripts/scrape-huggingface.js

// Simple Hugging Face scraper
const https = require('https');
const { HttpsProxyAgent } = require('https-proxy-agent');

async function scrapeHuggingFace() {
  try {
    console.log('Fetching Hugging Face models page...');

    const proxyUrl = process.env.HTTPS_PROXY || process.env.HTTP_PROXY;
    const agent = proxyUrl ? new HttpsProxyAgent(proxyUrl) : undefined;

    const html = await new Promise((resolve, reject) => {
      const options = {
        headers: {
          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
          'Accept-Language': 'en-US,en;q=0.9',
        }
      };

      if (agent) {
        options.agent = agent;
      }

      const req = https.get('https://huggingface.co/models', options, (res) => {
        let data = '';
        res.on('data', chunk => data += chunk);
        res.on('end', () => {
          if (res.statusCode === 200) {
            resolve(data);
          } else {
            reject(new Error(`HTTP ${res.statusCode}: ${res.statusMessage}`));
          }
        });
      });

      req.on('error', reject);
      req.setTimeout(30000, () => {
        req.destroy();
        reject(new Error('Request timeout'));
      });
    });

    console.log(`Page fetched successfully, size: ${html.length} bytes`);

    // Extract model information
    const models = [];
    const modelLinkRegex = /<a\s+href="\/models\/([^"]+)"[^>]*>/gi;
    const seenModels = new Set();

    let match;
    while ((match = modelLinkRegex.exec(html)) !== null) {
      const modelId = decodeURIComponent(match[1]);

      // Filter: must have org/model format
      if (!modelId.includes('/')) continue;
      if (modelId.includes('/discussions')) continue;
      if (modelId.includes('/blob')) continue;
      if (modelId.includes('/tree')) continue;
      if (modelId.includes('/commit')) continue;
      if (seenModels.has(modelId)) continue;

      seenModels.add(modelId);

      models.push({
        source: 'huggingface',
        name: modelId,
        url: `https://huggingface.co/${modelId}`,
        description: modelId,
        metadata: {
          likes: 0,
          downloads: 0,
          pipeline: ''
        }
      });

      if (models.length >= 100) break;
    }

    console.log(`Extracted ${models.length} unique models`);

    // Take first 25 (they should be roughly ordered by popularity on the page)
    const topModels = models.slice(0, 25);

    const fs = require('fs');
    const outputPath = 'D:\\Code\\AI\\agent-park-v2\\.trending-workspace\\20260106-1226\\scraped-huggingface-projects.json';
    fs.writeFileSync(outputPath, JSON.stringify(topModels, null, 2), 'utf8');
    console.log(`\nData saved to: ${outputPath}`);

    console.log('\nTop 25 Models:');
    topModels.forEach((m, i) => {
      console.log(`${i + 1}. ${m.name}`);
    });

  } catch (error) {
    console.error('Error:', error.message);

    // Output empty result on error
    const fs = require('fs');
    const outputPath = 'D:\\Code\\AI\\agent-park-v2\\.trending-workspace\\20260106-1226\\scraped-huggingface-projects.json';

    const errorResult = {
      error: error.message,
      projects: [],
      timestamp: new Date().toISOString()
    };

    fs.writeFileSync(outputPath, JSON.stringify(errorResult, null, 2), 'utf8');
    console.log(`Error result saved to: ${outputPath}`);
  }
}

scrapeHuggingFace();