From 78d654ee7b07c34afa741d630429c0404b15158c Mon Sep 17 00:00:00 2001
From: Alexey Kolpakov <al.kolpak@gmail.com>
Date: Wed, 31 Oct 2018 01:21:00 +0300
Subject: [PATCH] impr yandex

---
 .gitignore                |  1 +
 lib/ax.js                 |  2 ++
 lib/provider/avito.js     |  4 ++--
 lib/provider/thelocals.js |  2 +-
 lib/provider/yandex.js    | 31 ++++++++++---------------------
 package.json              |  4 +++-
 6 files changed, 19 insertions(+), 25 deletions(-)

diff --git a/.gitignore b/.gitignore
index d01bc36..4fd4130 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,5 @@ node_modules/*
 config/*
 db/*
 yarn.lock
+yarn-error.log
 .vscode/*
diff --git a/lib/ax.js b/lib/ax.js
index 2609814..88050bb 100644
--- a/lib/ax.js
+++ b/lib/ax.js
@@ -1,4 +1,5 @@
 const axios = require('axios');
+const qs = require('qs');
 
 module.exports.create = (headers) => {
     const ax = axios.create();
@@ -7,5 +8,6 @@ module.exports.create = (headers) => {
     if (headers) {
         Object.entries(headers).forEach(([k, v]) => ax.defaults.headers.common[k] = v);
     }
+    ax.defaults.paramsSerializer = (params) => qs.stringify(params, {arrayFormat: 'repeat'});
     return ax;
 };
diff --git a/lib/provider/avito.js b/lib/provider/avito.js
index b4ebd0d..45a0f4e 100644
--- a/lib/provider/avito.js
+++ b/lib/provider/avito.js
@@ -20,14 +20,14 @@ module.exports.getOffers = async () => {
     let points = path(['data', 'coords'], rawListRes) || {};
     points = Object.entries(points).map(([k, v]) => Object.assign({id: k, lng: v.lon}, v))
                 .filter(({id}) => db.data.indexOf(id) === -1);
-    
+
     let offers = await queue.addAll(points.map((x) => () => {
         return ax.get(urlGetItems, {params: x});
     }));
     offers = offers.map((x) => path(['data', 'items'], x) || []);
     offers = Object.assign({}, ...offers);
     offers = Object.entries(offers).map(([k, v]) => Object.assign({id: k}, v))
-                .filter((x) => pathOr(0, ['ext', 'rooms'], x) >= 2 
+                .filter((x) => pathOr(0, ['ext', 'rooms'], x) >= 2
                                 && db.data.indexOf(x.id) === -1
                                 && geoFilter(path(['ext', 'address'], x))
                 )
diff --git a/lib/provider/thelocals.js b/lib/provider/thelocals.js
index e4ca1ce..63e5531 100644
--- a/lib/provider/thelocals.js
+++ b/lib/provider/thelocals.js
@@ -16,7 +16,7 @@ module.exports.getOffers = async () => {
     const rawListRes = (await ax.post(urlGetIdx, query));
     const points = path(['data', 'ads'], rawListRes) || [];
 
-    const offers = points.filter((x) => pathOr(0, ['rooms'], x) >= 2 
+    const offers = points.filter((x) => pathOr(0, ['rooms'], x) >= 2
                                 && db.data.indexOf(x.id) === -1
                                 && geoFilter(x.address)
                     )
diff --git a/lib/provider/yandex.js b/lib/provider/yandex.js
index 020e167..613db8d 100644
--- a/lib/provider/yandex.js
+++ b/lib/provider/yandex.js
@@ -1,37 +1,26 @@
 const config = require('config');
-const PQueue = require('p-queue');
 const { path } = require('ramda');
 const Ax = require('../ax');
 const Db = require('../db');
 const { Offer } = require('./offer');
 const geoFilter = require('../geo_filter');
 
-const queue = new PQueue({concurrency: 1});
 const ax = Ax.create({'content-type': 'application/json'});
 
 module.exports.getOffers = async () => {
     const db = await Db.create('yandex');
 
-    const urlGetIdx = 'https://realty.yandex.ru/gate/map/offers-points-with-counter/';
-    const urlGetItems = 'https://realty.yandex.ru/gate/map/offers-gen2/';
+    const urlGetIdx = 'https://realty.yandex.ru/gate/react-page/get/';
     const query = config.get('yandex.query');
-    const listQuery = config.get('yandex.listQuery');
-
-    const rawListRes = (await ax.get(urlGetIdx, {params: Object.assign({}, query, listQuery)}));
-    let points = path(['data', 'response', 'points'], rawListRes) || [];
-    points = points.filter(({id}) => db.data.indexOf(id) === -1);
-
-    let offers = await queue.addAll(points.map((x) => () => {
-        const lat = x.lat; const lon = x.lon; const pageSize = 100;
-        return ax.get(urlGetItems, {params: Object.assign({lat, lon, pageSize}, query)});
-    }));
-    offers = offers.map((x) => path(['data', 'response', 'items', 'entities'], x) || []);
-    offers = [].concat(...offers);
-    offers = offers.filter((x) => x.roomsTotal >= 2 
-                                    && db.data.indexOf(x.offerId) === -1
-                                    && geoFilter(path(['location', 'geocoderAddress'], x))
-                )
-                .map((x) => new YandexOffer(x));
+
+    const rawListRes = await ax.get(urlGetIdx, {params: query});
+
+    const points = path(['data', 'response', 'search', 'offers', 'entities'], rawListRes) || [];
+    const offers = points.filter((x) => x.roomsTotal >= 2
+                            && db.data.indexOf(x.offerId) === -1
+                            && geoFilter(path(['location', 'geocoderAddress'], x))
+                        )
+                        .map((x) => new YandexOffer(x));
 
     console.log(`[yandex] idx: ${points.length}, offers: ${offers.length}`);
     if (offers.length > 0) {
diff --git a/package.json b/package.json
index 667cdd8..0d2dd50 100644
--- a/package.json
+++ b/package.json
@@ -1,9 +1,10 @@
 {
   "name": "cyanic",
-  "version": "1.0.5",
+  "version": "1.0.6",
   "main": "index.js",
   "license": "MIT",
   "scripts": {
+    "dev": "rm -rf db && NODE_ENV=development ./index.js",
     "lint": "./node_modules/.bin/eslint --ext .js ."
   },
   "dependencies": {
@@ -13,6 +14,7 @@
     "p-queue": "^3.0.0",
     "p-retry": "^2.0.0",
     "p-settle": "^2.1.0",
+    "qs": "^6.5.2",
     "ramda": "^0.25.0",
     "telegraf": "^3.24.1"
   },