feat(автоподбор): DeepStudyCollector — движок глобального сбора (близнецы порознь)

2026-07-03 23:32:23 +03:00
parent 92e128d5bd
commit 535bb7d3af
2 changed files with 234 additions and 0 deletions
@@ -0,0 +1,188 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Services\Autopodbor\Agent\Study;
+
+use App\Services\Autopodbor\Agent\Extract\DirectoryParser;
+use App\Services\Autopodbor\Agent\Extract\PhoneType;
+use App\Services\Autopodbor\Agent\Fetch\PlaywrightPageFetcher;
+use App\Services\Autopodbor\Agent\Fetch\RoutingBatchFetcher;
+use App\Services\Autopodbor\Agent\Fetch\XfetchClient;
+use App\Services\Autopodbor\Agent\Fetch\XfetchDirectoryFetcher;
+use App\Services\Autopodbor\AutopodborNormalizer;
+
+/**
+ * Движок «глобального шага 2»: от отпечатка фирмы собирает сайты (оба близнеца порознь) и номера
+ * с пометками — 1:1 с песочницей sandbox-step2.php. Возвращает строки контракта StudyCompetitorResult.
+ */
+final class DeepStudyCollector
+{
+    private const SRC_CHIP = ['code' => 'в коде сайта', 'displayed' => 'коллтрекинг',
+        'pool' => 'коллтрекинг (пул)', 'contacts' => 'страница «Контакты»', 'uncertain' => 'на сайте (код города?)'];
+
+    /** @var array<string,array{sub:bool,src:array<string,?string>}> */
+    private array $phones = [];
+
+    public function __construct(
+        private XfetchClient $xfetch,
+        private SiteOpener $opener,
+        private FootprintSearch $exa = new FootprintSearch,
+        private FootprintQueryBuilder $queries = new FootprintQueryBuilder,
+        private PlatformClassifier $classifier = new PlatformClassifier,
+        private AnchorGate $gate = new AnchorGate(new DomainKinship),
+        private DirectoryParser $dirParser = new DirectoryParser,
+        private AutopodborNormalizer $norm = new AutopodborNormalizer,
+        private ?string $nodeBin = null,
+    ) {}
+
+    /** @return array<int,array{signal_type:string,identifier:string,phone_kind:?string,phone_type:?string,provenance_url:?string,provenance_label:?string,where_found:array<int,array{label:string,url:?string}>,office:?string,confirmations:int}> */
+    public function collect(Fingerprint $fp): array
+    {
+        $this->phones = [];
+        $urls = $this->exa->search($this->queries->build($fp));
+        $buckets = $this->classifier->classify($urls);
+
+        $seedAscii = $fp->seedDomain !== '' ? $this->idn($fp->seedDomain) : '';
+        $order = array_keys($buckets['own']);
+        if ($seedAscii !== '') {
+            array_unshift($order, $seedAscii);
+            $order = array_values(array_unique($order));
+        }
+
+        // 1) сайты + якоря
+        $anchorDigits = [];
+        $siteRows = [];
+        foreach (array_slice($order, 0, 8) as $domain) {
+            $r = $this->opener->open($domain, $fp->areaCode);
+            $verified = $this->gate->siteVerified($domain, $seedAscii, $fp->seedDomain, array_map('strval', array_keys($anchorDigits)), $r->htmlLower);
+            if (! $verified) {
+                continue; // чужой — не наша фирма
+            }
+            foreach ($r->codeDigits as $d) {
+                $anchorDigits[$d] = true;
+            }
+            $siteUrl = 'https://'.$this->idn($domain).'/';
+            foreach ($r->numbers as $c) {
+                $sub = in_array($c['kind'], ['displayed', 'pool'], true);
+                $this->addPhone($c['number'], [(self::SRC_CHIP[$c['kind']] ?? $c['kind']) => $siteUrl, $this->idnUtf($domain) => $siteUrl], $sub);
+            }
+            // отдельная строка-источник на КАЖДЫЙ сайт (близнецы — порознь)
+            $siteRows[] = [
+                'signal_type' => 'site', 'identifier' => $this->idnUtf($domain),
+                'phone_kind' => null, 'phone_type' => null,
+                'provenance_url' => $siteUrl, 'provenance_label' => 'сайт конкурента',
+                'where_found' => [['label' => 'сайт конкурента', 'url' => $siteUrl]],
+                'office' => null, 'confirmations' => 1,
+            ];
+        }
+
+        // 2) 2ГИС по имени + EXA-ссылки
+        $this->collect2gis($fp, $buckets['gis'], $anchorDigits);
+        // 3) Яндекс через Playwright
+        $this->collectYandex($fp, $buckets['yandex'], $anchorDigits);
+
+        // 4) маппинг номеров
+        $callRows = [];
+        foreach ($this->phones as $digits => $info) {
+            $digits = (string) $digits;
+            $src = [];
+            foreach ($info['src'] as $label => $url) {
+                $src[] = ['label' => (string) $label, 'url' => $url];
+            }
+            $callRows[] = [
+                'signal_type' => 'call', 'identifier' => $digits,
+                'phone_kind' => $info['sub'] ? 'substitute' : 'real',
+                'phone_type' => PhoneType::of($digits),
+                'provenance_url' => $src[0]['url'] ?? null,
+                'provenance_label' => $src[0]['label'] ?? null,
+                'where_found' => $src, 'office' => null, 'confirmations' => count($src),
+            ];
+        }
+
+        return array_merge($siteRows, $callRows);
+    }
+
+    /** @param list<string> $gisFromExa @param array<string,bool> $anchorDigits */
+    private function collect2gis(Fingerprint $fp, array $gisFromExa, array $anchorDigits): void
+    {
+        $firmUrls = $gisFromExa;
+        if ($fp->gisSlug !== '') {
+            $searchUrl = 'https://2gis.ru/'.$fp->gisSlug.'/search/'.rawurlencode($fp->name);
+            $found = [];
+            for ($try = 1; $try <= 4 && $found === []; $try++) {
+                $found = $this->dirParser->parseBranchList($this->xfetch->html($searchUrl));
+            }
+            foreach ($found as $f) {
+                $firmUrls[] = str_starts_with($f, 'http') ? $f : 'https://2gis.ru'.$f;
+            }
+        }
+        $firmUrls = array_slice(array_values(array_filter(array_unique($firmUrls), fn ($u) => str_contains((string) $u, '/firm/'))), 0, 12);
+        foreach ($firmUrls as $furl) {
+            $fh = $this->xfetch->html($furl);
+            if ($fh === '') {
+                continue;
+            }
+            foreach ($this->dirParser->parse2gisCard($fh, $furl) as $card) {
+                $this->pushCard('2ГИС', $card->number ?: null, $card->siteUrl, $furl, $fp->seedDomain, $anchorDigits);
+            }
+        }
+    }
+
+    /** @param list<string> $yandexUrls @param array<string,bool> $anchorDigits */
+    private function collectYandex(Fingerprint $fp, array $yandexUrls, array $anchorDigits): void
+    {
+        if ($yandexUrls === []) {
+            return;
+        }
+        $node = $this->nodeBin ?? (string) config('autopodbor.node_bin', 'node');
+        $pages = new RoutingBatchFetcher($this->xfetch, ['yandex' => new PlaywrightPageFetcher($node)]);
+        $dir = new XfetchDirectoryFetcher($pages, $this->dirParser);
+        foreach (array_slice($yandexUrls, 0, 5) as $yurl) {
+            try {
+                foreach ($dir->directory($yurl) as $card) {
+                    $this->pushCard($card->source, $card->number ?: null, $card->siteUrl, $card->url, $fp->seedDomain, $anchorDigits);
+                }
+            } catch (\Throwable) {
+                // одна карточка упала — не роняем сбор
+            }
+        }
+    }
+
+    /** @param array<string,bool> $anchorDigits */
+    private function pushCard(string $source, ?string $number, ?string $site, string $url, string $seedDomain, array $anchorDigits): void
+    {
+        $cd = $number ? $this->norm->phone($number) : '';
+        if ($cd !== '' && strlen($cd) !== 11) {
+            $cd = '';
+        }
+        if ($this->gate->cardKept($site, $cd, $seedDomain, $anchorDigits) && $cd !== '') {
+            $this->addPhone($cd, [$source => $url]);
+        }
+    }
+
+    /** @param array<string,?string> $chips */
+    private function addPhone(string $number, array $chips, bool $sub = false): void
+    {
+        $d = $this->norm->phone($number);
+        if (strlen($d) !== 11) {
+            return;
+        }
+        $this->phones[$d]['sub'] = ($this->phones[$d]['sub'] ?? false) || $sub;
+        foreach ($chips as $label => $url) {
+            if ((string) $label !== '') {
+                $this->phones[$d]['src'][$label] = $url ?: ($this->phones[$d]['src'][$label] ?? null);
+            }
+        }
+    }
+
+    private function idn(string $d): string
+    {
+        return function_exists('idn_to_ascii') ? (idn_to_ascii($d, IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46) ?: $d) : $d;
+    }
+
+    private function idnUtf(string $h): string
+    {
+        return function_exists('idn_to_utf8') ? (idn_to_utf8($h, IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46) ?: $h) : $h;
+    }
+}
@@ -0,0 +1,46 @@
+<?php
+
+use App\Services\Autopodbor\Agent\Fetch\FetchedSite;
+use App\Services\Autopodbor\Agent\Fetch\Fetcher;
+use App\Services\Autopodbor\Agent\Fetch\XfetchClient;
+use App\Services\Autopodbor\Agent\Study\DeepStudyCollector;
+use App\Services\Autopodbor\Agent\Study\Fingerprint;
+use App\Services\Autopodbor\Agent\Study\SiteOpener;
+use Illuminate\Support\Facades\Http;
+
+it('оба сайта-близнеца — две отдельные строки-источника (НЕ слиты)', function () {
+    config()->set('services.exa', ['key' => 't', 'base_url' => 'https://api.exa.ai']);
+    Http::fake([
+        'api.exa.ai/*' => Http::response(['results' => [
+            ['url' => 'https://kraslombard24.ru/'],
+            ['url' => 'https://xn--80aacn0ahhppgf.xn--p1ai/'],   // красломбард.рф в punycode
+        ]]),
+        '*' => Http::response(['response_body_base64' => '']),
+    ]);
+    $siteFetcher = new class implements Fetcher
+    {
+        public function site(string $url): FetchedSite
+        {
+            return new FetchedSite($url, '<a href="tel:+73912920000">'.$url.'</a>');
+        }
+
+        public function directory(string $url): array
+        {
+            return [];
+        }
+    };
+    $collector = new DeepStudyCollector(new XfetchClient(null), new SiteOpener($siteFetcher));
+    $fp = new Fingerprint('КрасЛомбард', 'kraslombard24.ru', 'Красноярск', '391', 'krasnoyarsk', false);
+
+    $rows = $collector->collect($fp);
+    $sites = array_values(array_filter($rows, fn ($r) => $r['signal_type'] === 'site'));
+    $domains = array_map(fn ($r) => $r['identifier'], $sites);
+
+    expect(count($sites))->toBe(2)
+        ->and($domains)->toContain('kraslombard24.ru')
+        ->and($domains)->toContain('красломбард.рф');
+
+    $call = array_values(array_filter($rows, fn ($r) => $r['signal_type'] === 'call' && $r['identifier'] === '73912920000'))[0];
+    expect(collect($call['where_found'])->pluck('label'))->toContain('в коде сайта')
+        ->and($call['phone_kind'])->toBe('real');
+});