Market Research
& Intelligence

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import random

# Proxy configuration
PROXIES = {
    'http': 'http://username:[email protected]:8000',
    'https': 'https://username:[email protected]:8000'
}

class MarketResearcher:
    def __init__(self, proxies):
        self.session = requests.Session()
        self.session.proxies.update(proxies)
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        })
    
    def get_product_price(self, url, price_selector):
        try:
            response = self.session.get(url, timeout=30)
            soup = BeautifulSoup(response.content, 'html.parser')
            
            price_element = soup.select_one(price_selector)
            if price_element:
                price_text = price_element.get_text().strip()
                # Extract numeric price
                price = float(''.join(filter(str.isdigit, price_text.replace('.', ''))) / 100)
                return price
            return None
        except Exception as e:
            print(f"Error fetching price: {e}")
            return None
    
    def monitor_competitors(self, competitors):
        results = []
        for competitor in competitors:
            price = self.get_product_price(competitor['url'], competitor['selector'])
            results.append({
                'competitor': competitor['name'],
                'price': price,
                'timestamp': time.time()
            })
            # Random delay to avoid detection
            time.sleep(random.uniform(2, 5))
        return results

# Usage
researcher = MarketResearcher(PROXIES)
competitors = [
    {'name': 'Competitor A', 'url': 'https://example1.com/product', 'selector': '.price'},
    {'name': 'Competitor B', 'url': 'https://example2.com/product', 'selector': '.cost'},
]

prices = researcher.monitor_competitors(competitors)
df = pd.DataFrame(prices)
print(df)

const axios = require('axios');
const cheerio = require('cheerio');
const HttpsProxyAgent = require('https-proxy-agent');

// Proxy configuration
const proxyAgent = new HttpsProxyAgent(
  'http://username:[email protected]:8000'
);

class MarketAnalyzer {
  constructor() {
    this.client = axios.create({
      httpsAgent: proxyAgent,
      timeout: 30000,
      headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
      }
    });
  }

  async collectMarketData(sources) {
    const results = [];
    
    for (const source of sources) {
      try {
        const response = await this.client.get(source.url);
        const $ = cheerio.load(response.data);
        
        const data = {
          source: source.name,
          title: $(source.selectors.title).text().trim(),
          price: this.extractPrice($(source.selectors.price).text()),
          availability: $(source.selectors.stock).text().trim(),
          rating: $(source.selectors.rating).text().trim(),
          timestamp: new Date().toISOString()
        };
        
        results.push(data);
        console.log(`Collected data from ${source.name}`);
        
        // Rate limiting
        await this.delay(Math.random() * 3000 + 1000);
        
      } catch (error) {
        console.error(`Error collecting from ${source.name}:`, error.message);
      }
    }
    
    return results;
  }
  
  extractPrice(priceText) {
    const match = priceText.match(/[d,]+.?d*/);
    return match ? parseFloat(match[0].replace(',', '')) : null;
  }
  
  delay(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
  }
  
  async generateReport(data) {
    const report = {
      totalSources: data.length,
      averagePrice: data.reduce((sum, item) => sum + (item.price || 0), 0) / data.length,
      priceRange: {
        min: Math.min(...data.map(item => item.price || Infinity)),
        max: Math.max(...data.map(item => item.price || 0))
      },
      timestamp: new Date().toISOString()
    };
    
    console.log('Market Research Report:', report);
    return report;
  }
}

// Usage
const analyzer = new MarketAnalyzer();
const sources = [
  {
    name: 'Amazon',
    url: 'https://amazon.com/product',
    selectors: {
      title: 'h1',
      price: '.a-price-whole',
      stock: '.availability span',
      rating: '.a-icon-alt'
    }
  }
];

analyzer.collectMarketData(sources)
  .then(data => analyzer.generateReport(data))
  .catch(console.error);

<?php
require_once 'vendor/autoload.php';

use GuzzleHttp\Client;
use Symfony\Component\DomCrawler\Crawler;

class CompetitiveIntelligence {
    private $client;
    
    public function __construct($proxyUrl) {
        $this->client = new Client([
            'proxy' => $proxyUrl,
            'timeout' => 30,
            'headers' => [
                'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
            ]
        ]);
    }
    
    public function analyzeCompetitor($url, $selectors) {
        try {
            $response = $this->client->get($url);
            $crawler = new Crawler($response->getBody()->getContents());
            
            $data = [
                'url' => $url,
                'title' => $this->extractText($crawler, $selectors['title']),
                'price' => $this->extractPrice($crawler, $selectors['price']),
                'features' => $this->extractFeatures($crawler, $selectors['features']),
                'reviews' => $this->extractReviews($crawler, $selectors['reviews']),
                'timestamp' => date('Y-m-d H:i:s')
            ];
            
            return $data;
            
        } catch (Exception $e) {
            error_log("Error analyzing competitor: " . $e->getMessage());
            return null;
        }
    }
    
    private function extractText($crawler, $selector) {
        try {
            return $crawler->filter($selector)->first()->text();
        } catch (Exception $e) {
            return null;
        }
    }
    
    private function extractPrice($crawler, $selector) {
        $priceText = $this->extractText($crawler, $selector);
        if ($priceText) {
            preg_match('/[d,]+.?d*/', $priceText, $matches);
            return $matches ? (float)str_replace(',', '', $matches[0]) : null;
        }
        return null;
    }
    
    private function extractFeatures($crawler, $selector) {
        try {
            return $crawler->filter($selector)->each(function ($node) {
                return trim($node->text());
            });
        } catch (Exception $e) {
            return [];
        }
    }
    
    private function extractReviews($crawler, $selector) {
        try {
            $reviewText = $this->extractText($crawler, $selector);
            preg_match('/(d+.?d*)/', $reviewText, $matches);
            return $matches ? (float)$matches[1] : null;
        } catch (Exception $e) {
            return null;
        }
    }
    
    public function compareCompetitors($competitors) {
        $results = [];
        
        foreach ($competitors as $competitor) {
            $data = $this->analyzeCompetitor($competitor['url'], $competitor['selectors']);
            if ($data) {
                $results[] = array_merge($data, ['name' => $competitor['name']]);
            }
            
            // Rate limiting
            sleep(rand(2, 5));
        }
        
        return $this->generateComparisonReport($results);
    }
    
    private function generateComparisonReport($results) {
        $prices = array_filter(array_column($results, 'price'));
        
        return [
            'total_competitors' => count($results),
            'average_price' => $prices ? array_sum($prices) / count($prices) : 0,
            'price_range' => [
                'min' => $prices ? min($prices) : 0,
                'max' => $prices ? max($prices) : 0
            ],
            'competitors' => $results,
            'analysis_date' => date('Y-m-d H:i:s')
        ];
    }
}

// Usage
$proxyUrl = 'http://username:[email protected]:8000';
$intelligence = new CompetitiveIntelligence($proxyUrl);

$competitors = [
    [
        'name' => 'Competitor 1',
        'url' => 'https://example1.com/product',
        'selectors' => [
            'title' => 'h1.product-title',
            'price' => '.price-current',
            'features' => '.feature-list li',
            'reviews' => '.rating-average'
        ]
    ]
];

$report = $intelligence->compareCompetitors($competitors);
echo json_encode($report, JSON_PRETTY_PRINT);
?>