From 535bb7d3afc2cc883adfa0e6dff0155920bf46b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9?= Date: Fri, 3 Jul 2026 23:32:23 +0300 Subject: [PATCH] =?UTF-8?q?feat(=D0=B0=D0=B2=D1=82=D0=BE=D0=BF=D0=BE=D0=B4?= =?UTF-8?q?=D0=B1=D0=BE=D1=80):=20DeepStudyCollector=20=E2=80=94=20=D0=B4?= =?UTF-8?q?=D0=B2=D0=B8=D0=B6=D0=BE=D0=BA=20=D0=B3=D0=BB=D0=BE=D0=B1=D0=B0?= =?UTF-8?q?=D0=BB=D1=8C=D0=BD=D0=BE=D0=B3=D0=BE=20=D1=81=D0=B1=D0=BE=D1=80?= =?UTF-8?q?=D0=B0=20(=D0=B1=D0=BB=D0=B8=D0=B7=D0=BD=D0=B5=D1=86=D1=8B=20?= =?UTF-8?q?=D0=BF=D0=BE=D1=80=D0=BE=D0=B7=D0=BD=D1=8C)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Agent/Study/DeepStudyCollector.php | 188 ++++++++++++++++++ .../Autopodbor/DeepStudyCollectorTest.php | 46 +++++ 2 files changed, 234 insertions(+) create mode 100644 app/app/Services/Autopodbor/Agent/Study/DeepStudyCollector.php create mode 100644 app/tests/Feature/Autopodbor/DeepStudyCollectorTest.php diff --git a/app/app/Services/Autopodbor/Agent/Study/DeepStudyCollector.php b/app/app/Services/Autopodbor/Agent/Study/DeepStudyCollector.php new file mode 100644 index 00000000..072a49b7 --- /dev/null +++ b/app/app/Services/Autopodbor/Agent/Study/DeepStudyCollector.php @@ -0,0 +1,188 @@ + 'в коде сайта', 'displayed' => 'коллтрекинг', + 'pool' => 'коллтрекинг (пул)', 'contacts' => 'страница «Контакты»', 'uncertain' => 'на сайте (код города?)']; + + /** @var array}> */ + private array $phones = []; + + public function __construct( + private XfetchClient $xfetch, + private SiteOpener $opener, + private FootprintSearch $exa = new FootprintSearch, + private FootprintQueryBuilder $queries = new FootprintQueryBuilder, + private PlatformClassifier $classifier = new PlatformClassifier, + private AnchorGate $gate = new AnchorGate(new DomainKinship), + private DirectoryParser $dirParser = new DirectoryParser, + private AutopodborNormalizer $norm = new AutopodborNormalizer, + private ?string $nodeBin = null, + ) {} + + /** @return array,office:?string,confirmations:int}> */ + public function collect(Fingerprint $fp): array + { + $this->phones = []; + $urls = $this->exa->search($this->queries->build($fp)); + $buckets = $this->classifier->classify($urls); + + $seedAscii = $fp->seedDomain !== '' ? $this->idn($fp->seedDomain) : ''; + $order = array_keys($buckets['own']); + if ($seedAscii !== '') { + array_unshift($order, $seedAscii); + $order = array_values(array_unique($order)); + } + + // 1) сайты + якоря + $anchorDigits = []; + $siteRows = []; + foreach (array_slice($order, 0, 8) as $domain) { + $r = $this->opener->open($domain, $fp->areaCode); + $verified = $this->gate->siteVerified($domain, $seedAscii, $fp->seedDomain, array_map('strval', array_keys($anchorDigits)), $r->htmlLower); + if (! $verified) { + continue; // чужой — не наша фирма + } + foreach ($r->codeDigits as $d) { + $anchorDigits[$d] = true; + } + $siteUrl = 'https://'.$this->idn($domain).'/'; + foreach ($r->numbers as $c) { + $sub = in_array($c['kind'], ['displayed', 'pool'], true); + $this->addPhone($c['number'], [(self::SRC_CHIP[$c['kind']] ?? $c['kind']) => $siteUrl, $this->idnUtf($domain) => $siteUrl], $sub); + } + // отдельная строка-источник на КАЖДЫЙ сайт (близнецы — порознь) + $siteRows[] = [ + 'signal_type' => 'site', 'identifier' => $this->idnUtf($domain), + 'phone_kind' => null, 'phone_type' => null, + 'provenance_url' => $siteUrl, 'provenance_label' => 'сайт конкурента', + 'where_found' => [['label' => 'сайт конкурента', 'url' => $siteUrl]], + 'office' => null, 'confirmations' => 1, + ]; + } + + // 2) 2ГИС по имени + EXA-ссылки + $this->collect2gis($fp, $buckets['gis'], $anchorDigits); + // 3) Яндекс через Playwright + $this->collectYandex($fp, $buckets['yandex'], $anchorDigits); + + // 4) маппинг номеров + $callRows = []; + foreach ($this->phones as $digits => $info) { + $digits = (string) $digits; + $src = []; + foreach ($info['src'] as $label => $url) { + $src[] = ['label' => (string) $label, 'url' => $url]; + } + $callRows[] = [ + 'signal_type' => 'call', 'identifier' => $digits, + 'phone_kind' => $info['sub'] ? 'substitute' : 'real', + 'phone_type' => PhoneType::of($digits), + 'provenance_url' => $src[0]['url'] ?? null, + 'provenance_label' => $src[0]['label'] ?? null, + 'where_found' => $src, 'office' => null, 'confirmations' => count($src), + ]; + } + + return array_merge($siteRows, $callRows); + } + + /** @param list $gisFromExa @param array $anchorDigits */ + private function collect2gis(Fingerprint $fp, array $gisFromExa, array $anchorDigits): void + { + $firmUrls = $gisFromExa; + if ($fp->gisSlug !== '') { + $searchUrl = 'https://2gis.ru/'.$fp->gisSlug.'/search/'.rawurlencode($fp->name); + $found = []; + for ($try = 1; $try <= 4 && $found === []; $try++) { + $found = $this->dirParser->parseBranchList($this->xfetch->html($searchUrl)); + } + foreach ($found as $f) { + $firmUrls[] = str_starts_with($f, 'http') ? $f : 'https://2gis.ru'.$f; + } + } + $firmUrls = array_slice(array_values(array_filter(array_unique($firmUrls), fn ($u) => str_contains((string) $u, '/firm/'))), 0, 12); + foreach ($firmUrls as $furl) { + $fh = $this->xfetch->html($furl); + if ($fh === '') { + continue; + } + foreach ($this->dirParser->parse2gisCard($fh, $furl) as $card) { + $this->pushCard('2ГИС', $card->number ?: null, $card->siteUrl, $furl, $fp->seedDomain, $anchorDigits); + } + } + } + + /** @param list $yandexUrls @param array $anchorDigits */ + private function collectYandex(Fingerprint $fp, array $yandexUrls, array $anchorDigits): void + { + if ($yandexUrls === []) { + return; + } + $node = $this->nodeBin ?? (string) config('autopodbor.node_bin', 'node'); + $pages = new RoutingBatchFetcher($this->xfetch, ['yandex' => new PlaywrightPageFetcher($node)]); + $dir = new XfetchDirectoryFetcher($pages, $this->dirParser); + foreach (array_slice($yandexUrls, 0, 5) as $yurl) { + try { + foreach ($dir->directory($yurl) as $card) { + $this->pushCard($card->source, $card->number ?: null, $card->siteUrl, $card->url, $fp->seedDomain, $anchorDigits); + } + } catch (\Throwable) { + // одна карточка упала — не роняем сбор + } + } + } + + /** @param array $anchorDigits */ + private function pushCard(string $source, ?string $number, ?string $site, string $url, string $seedDomain, array $anchorDigits): void + { + $cd = $number ? $this->norm->phone($number) : ''; + if ($cd !== '' && strlen($cd) !== 11) { + $cd = ''; + } + if ($this->gate->cardKept($site, $cd, $seedDomain, $anchorDigits) && $cd !== '') { + $this->addPhone($cd, [$source => $url]); + } + } + + /** @param array $chips */ + private function addPhone(string $number, array $chips, bool $sub = false): void + { + $d = $this->norm->phone($number); + if (strlen($d) !== 11) { + return; + } + $this->phones[$d]['sub'] = ($this->phones[$d]['sub'] ?? false) || $sub; + foreach ($chips as $label => $url) { + if ((string) $label !== '') { + $this->phones[$d]['src'][$label] = $url ?: ($this->phones[$d]['src'][$label] ?? null); + } + } + } + + private function idn(string $d): string + { + return function_exists('idn_to_ascii') ? (idn_to_ascii($d, IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46) ?: $d) : $d; + } + + private function idnUtf(string $h): string + { + return function_exists('idn_to_utf8') ? (idn_to_utf8($h, IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46) ?: $h) : $h; + } +} diff --git a/app/tests/Feature/Autopodbor/DeepStudyCollectorTest.php b/app/tests/Feature/Autopodbor/DeepStudyCollectorTest.php new file mode 100644 index 00000000..a75fa5c9 --- /dev/null +++ b/app/tests/Feature/Autopodbor/DeepStudyCollectorTest.php @@ -0,0 +1,46 @@ +set('services.exa', ['key' => 't', 'base_url' => 'https://api.exa.ai']); + Http::fake([ + 'api.exa.ai/*' => Http::response(['results' => [ + ['url' => 'https://kraslombard24.ru/'], + ['url' => 'https://xn--80aacn0ahhppgf.xn--p1ai/'], // красломбард.рф в punycode + ]]), + '*' => Http::response(['response_body_base64' => '']), + ]); + $siteFetcher = new class implements Fetcher + { + public function site(string $url): FetchedSite + { + return new FetchedSite($url, ''.$url.''); + } + + public function directory(string $url): array + { + return []; + } + }; + $collector = new DeepStudyCollector(new XfetchClient(null), new SiteOpener($siteFetcher)); + $fp = new Fingerprint('КрасЛомбард', 'kraslombard24.ru', 'Красноярск', '391', 'krasnoyarsk', false); + + $rows = $collector->collect($fp); + $sites = array_values(array_filter($rows, fn ($r) => $r['signal_type'] === 'site')); + $domains = array_map(fn ($r) => $r['identifier'], $sites); + + expect(count($sites))->toBe(2) + ->and($domains)->toContain('kraslombard24.ru') + ->and($domains)->toContain('красломбард.рф'); + + $call = array_values(array_filter($rows, fn ($r) => $r['signal_type'] === 'call' && $r['identifier'] === '73912920000'))[0]; + expect(collect($call['where_found'])->pluck('label'))->toContain('в коде сайта') + ->and($call['phone_kind'])->toBe('real'); +});