Bitrix-D7 23.9
 
Загрузка...
Поиск...
Не найдено
importprocess.php
1<?php
9namespace Bitrix\Sale\Location\Import;
10
11use Bitrix\Main;
12use Bitrix\Main\Web\HttpClient;
13use Bitrix\Main\IO;
14
15use Bitrix\Sale\Location;
16use Bitrix\Sale\Location\DB\BlockInserter;
17use Bitrix\Sale\Location\Util\CSVReader;
18
19final class ImportProcess extends Location\Util\Process
20{
21 const DISTRIBUTOR_HOST = 'www.1c-bitrix.ru';
22 const DISTRIBUTOR_PORT = 80;
23 //const REMOTE_PATH = '/locations_data/compiled/';
24 const REMOTE_PATH = '/download/files/locations/pro/';
25 const REMOTE_SETS_PATH = 'bundles/';
26 const REMOTE_LAYOUT_FILE = 'bundles/layout.csv';
27 const REMOTE_TYPE_GROUP_FILE = 'typegroup.csv';
28 const REMOTE_TYPE_FILE = 'type.v2.csv';
29 const REMOTE_EXTERNAL_SERVICE_FILE = 'externalservice.csv';
30
31 const PACK_STANDARD = 'standard';
32 const PACK_EXTENDED = 'extended';
33
34 const LOCAL_SETS_PATH = 'bundles/';
35 const LOCAL_LOCATION_FILE = '%s.csv';
36 const LOCAL_LAYOUT_FILE = 'layout.csv';
37 const LOCAL_TYPE_GROUP_FILE = 'typegroup.csv';
38 const LOCAL_TYPE_FILE = 'type.csv';
39 const LOCAL_EXTERNAL_SERVICE_FILE = 'externalservice.csv';
40
41 const USER_FILE_DIRECTORY_SESSION_KEY = 'location_import_user_file';
42 const USER_FILE_TEMP_NAME = 'userfile.csv';
43
44 const SOURCE_REMOTE = 'remote';
45 const SOURCE_FILE = 'file';
46
47 const MAX_CODE_FETCH_BLOCK_LEN = 90;
48 const INSERTER_MTU = 99999;
49 const INSERTER_MTU_ORACLE = 9999;
50
51 const DB_TYPE_MYSQL = 'mysql';
52 const DB_TYPE_MSSQL = 'mssql';
53 const DB_TYPE_ORACLE = 'oracle';
54
55 const TREE_REBALANCE_TEMP_BLOCK_LEN = 99999;
56 const TREE_REBALANCE_TEMP_BLOCK_LEN_O = 9999;
57 const TREE_REBALANCE_TEMP_TABLE_NAME = 'b_sale_location_rebalance';
58
59 const DEBUG_MODE = false;
60
61 protected $sessionKey = 'location_import';
62 protected $rebalanceInserter = false;
63 protected $stat = array();
64 protected $hitData = array();
65 protected $useCache = true;
66
67 protected $dbConnection = null;
68 protected $dbConnType = null;
69 protected $dbHelper = null;
70
71 public function __construct($options)
72 {
73 $options = $this->prepareImportProcessOptions($options);
74
75 if($options['ONLY_DELETE_ALL'])
76 {
77 $this->addStage(array(
78 'PERCENT' => 100,
79 'CODE' => 'DELETE_ALL',
80 'CALLBACK' => 'stageDeleteAll',
81 'SUBPERCENT_CALLBACK' => 'getSubpercentForstageDeleteAll'
82 ));
83 }
84 else
85 {
86 $this->addStage(array(
87 'PERCENT' => 5,
88 'CODE' => 'DOWNLOAD_FILES',
89 'CALLBACK' => 'stageDownloadFiles',
90 'SUBPERCENT_CALLBACK' => 'getSubpercentForStageDownloadFiles'
91 ));
92
93 if($options['REQUEST']['OPTIONS']['DROP_ALL'])
94 {
95 $this->addStage(array(
96 'PERCENT' => 7,
97 'CODE' => 'DELETE_ALL',
98 'CALLBACK' => 'stageDeleteAll',
99 'SUBPERCENT_CALLBACK' => 'getSubpercentForstageDeleteAll'
100 ));
101 }
102
103 $this->addStage(array(
104 'PERCENT' => 10,
105 'CODE' => 'DROP_INDEXES',
106 'CALLBACK' => 'stageDropIndexes',
107 'SUBPERCENT_CALLBACK' => 'getSubpercentForStageDropIndexes'
108 ));
109
110 $this->addStage(array(
111 'PERCENT' => 60,
112 'STEP_SIZE' => 6000,
113 'CODE' => 'PROCESS_FILES',
114 'CALLBACK' => 'stageProcessFiles',
115 'SUBPERCENT_CALLBACK' => 'getSubpercentForStageProcessFiles'
116 ));
117
118 if($options['REQUEST']['OPTIONS']['INTEGRITY_PRESERVE'])
119 {
120 $this->addStage(array(
121 'PERCENT' => 65,
122 'STEP_SIZE' => 1,
123 'CODE' => 'INTEGRITY_PRESERVE',
124 'CALLBACK' => 'stageIntegrityPreserve'
125 ));
126 }
127
128 $this->addStage(array(
129 'PERCENT' => 90,
130 'STEP_SIZE' => 1,
131 'CODE' => 'REBALANCE_WALK_TREE',
132 'CALLBACK' => 'stageRebalanceWalkTree',
133 'SUBPERCENT_CALLBACK' => 'getSubpercentForStageRebalanceWalkTree'
134 ));
135
136 $this->addStage(array(
137 'PERCENT' => 95,
138 'STEP_SIZE' => 1,
139 'CODE' => 'REBALANCE_CLEANUP_TEMP_TABLE',
140 'CALLBACK' => 'stageRebalanceCleanupTempTable'
141 ));
142
143 $this->addStage(array(
144 'PERCENT' => 100,
145 'STEP_SIZE' => 1,
146 'CODE' => 'RESTORE_INDEXES',
147 'CALLBACK' => 'stageRestoreIndexes',
148 'SUBPERCENT_CALLBACK' => 'getSubpercentForStageRestoreIndexes'
149 ));
150 }
151
152 $this->dbConnection = Main\HttpApplication::getConnection();
153 $this->dbConnType = $this->dbConnection->getType();
154 $this->dbHelper = $this->dbConnection->getSqlHelper();
155
156 parent::__construct($options);
157 }
158
159 protected function prepareImportProcessOptions($options): array
160 {
161 if (!is_array($options))
162 {
163 $options = [];
164 }
165 $options['ONLY_DELETE_ALL'] = (bool)($options['ONLY_DELETE_ALL'] ?? false);
166 $options['LANGUAGE_ID'] = trim((string)($options['LANGUAGE_ID'] ?? LANGUAGE_ID));
167 if ($options['LANGUAGE_ID'] === '')
168 {
169 $options['LANGUAGE_ID'] = LANGUAGE_ID;
170 }
171
172 $options['REQUEST'] ??= [];
173 if (is_object($options['REQUEST']) && method_exists($options['REQUEST'], 'toArray'))
174 {
175 $options['REQUEST'] = $options['REQUEST']->toArray();
176 }
177 if (!is_array($options['REQUEST']))
178 {
179 $options['REQUEST'] = [];
180 }
181 $options['REQUEST']['OPTIONS'] ??= [];
182 if (!is_array($options['REQUEST']['OPTIONS']))
183 {
184 $options['REQUEST']['OPTIONS'] = [];
185 }
186
187 $requestOptions = &$options['REQUEST']['OPTIONS'];
188 $requestOptions['DROP_ALL'] ??= null;
189 $requestOptions['INTEGRITY_PRESERVE'] ??= null;
190 $requestOptions['EXCLUDE_COUNTRY_DISTRICT'] ??= null;
191 $requestOptions['SOURCE'] ??= null;
192 $requestOptions['TIME_LIMIT'] = (int)($requestOptions['TIME_LIMIT'] ?? 0);
193 unset($requestOptions);
194
195 $options['REQUEST']['ADDITIONAL'] ??= [];
196 if (!is_array($options['REQUEST']['ADDITIONAL']))
197 {
198 $options['REQUEST']['ADDITIONAL'] = [];
199 }
200
201 $options['LOCATION_SETS'] ??= [];
202 if (!is_array($options['LOCATION_SETS']))
203 {
204 $options['LOCATION_SETS'] = [];
205 }
206
207 return $options;
208 }
209
210 public function onBeforePerformIteration()
211 {
212 if ($this->options['ONLY_DELETE_ALL'])
213 {
214 return;
215 }
216
217 $this->data['inited'] ??= false;
218 if (!$this->data['inited'])
219 {
220 if((string)($this->data['LOCAL_PATH'] ?? '') === '')
221 {
222 [$this->data['LOCAL_PATH'], $this->data['LOCAL_PATH_CREATED']] = $this->getTemporalDirectory();
223 }
224
225 $opts = $this->options['REQUEST']['OPTIONS'];
226
227 if(!in_array($opts['SOURCE'], array(self::SOURCE_REMOTE, self::SOURCE_FILE)))
228 throw new Main\SystemException('Unknown import type');
229
230 $sets = array();
231 if($opts['SOURCE'] == self::SOURCE_REMOTE)
232 {
233 $sets = $this->normalizeQueryArray($this->options['REQUEST']['LOCATION_SETS']);
234 if(empty($sets))
235 throw new Main\SystemException('Nothing to do (no sets selected)');
236 }
237
238 $this->data['settings'] = array(
239 'sets' => $sets,
240 'options' => $opts
241 );
242
243 if($opts['SOURCE'] == self::SOURCE_REMOTE)
244 {
245 $this->data['settings']['additional'] = is_array($this->options['REQUEST']['ADDITIONAL']) ? array_flip(array_values($this->options['REQUEST']['ADDITIONAL'])) : array();
246
247 if(isset($this->data['settings']['additional']['ZIP']))
248 $this->data['settings']['additional']['ZIP_LOWER'] = $this->data['settings']['additional']['ZIP'];
249 }
250 elseif($this->checkSource(self::SOURCE_FILE))
251 {
252 $this->data['settings']['additional'] = false; // means ANY
253 }
254
255 $this->buildTypeTable();
256 $this->buildExternalSerivceTable();
257
258 $this->data['inited'] = true;
259 }
260
261 if ($this->data['inited'] && $this->data['LOCAL_PATH_CREATED'])
262 {
263 $this->touchImportTmpFiles((string)$this->data['LOCAL_PATH']);
264 }
265
266 $timeLimit = $this->data['settings']['options']['TIME_LIMIT'];
267 if ($timeLimit > 0)
268 {
269 $this->setTimeLimit($timeLimit);
270 }
271 }
272
276 private function touchImportTmpFiles(string $localPath): void
277 {
278 $iterator = new \RecursiveIteratorIterator(
279 new \RecursiveDirectoryIterator($localPath)
280 );
281 foreach ($iterator as $file) {
282
283 if ($file->isDir())
284 {
285 continue;
286 }
287
288 touch($file->getPathname());
289 }
290 }
291
293 // STAGE 1
294
295 protected function stageDownloadFiles()
296 {
297 if($this->checkSource(self::SOURCE_FILE)) // user uploaded file
298 {
299 if((string) $_SESSION[self::USER_FILE_DIRECTORY_SESSION_KEY] == '')
300 throw new Main\SystemException('User file was not uploaded properly');
301
302 $srcFilePath = $_SESSION[self::USER_FILE_DIRECTORY_SESSION_KEY].'/'.static::USER_FILE_TEMP_NAME;
303 $dstFilePath = $this->data['LOCAL_PATH'].self::getFileNameByIndex(0);
304
305 // ensure directory exists
306 $this->createDirectory($this->data['LOCAL_PATH'].'/'.self::LOCAL_SETS_PATH);
307
308 if(!@copy($srcFilePath, $dstFilePath))
309 {
310 $lastError = error_get_last();
311 throw new Main\SystemException($lastError['message']);
312 }
313
314 $this->data['files'] = array(
315 array(
316 'size' => filesize($dstFilePath),
317 'memgroup' => 'static'
318 )
319 );
320
321 $this->nextStage();
322 }
323 elseif($this->checkSource(self::SOURCE_REMOTE)) // get locations from remote server
324 {
325 if($this->getStep() == 0)
326 {
327 $this->data['files'] = array();
328
329 $this->cleanWorkDirectory();
330
331 // layout
332 $this->determineLayoutToImport();
333
334 // type groups
335 $typeGroups = $this->getRemoteTypeGroups();
336
337 // find out what groups we will include
338 $this->data['requiredGroups'] = array();
339 foreach($typeGroups as $code => $types)
340 {
341 if($code == 'LAYOUT') // layout is always included
342 continue;
343
344 foreach($types as $type)
345 {
346 if(isset($this->data['types']['allowed'][$type]))
347 {
348 $this->data['requiredGroups'][] = ToLower($code);
349 break;
350 }
351 }
352 }
353 }
354 else
355 {
356 $packPath = self::REMOTE_SETS_PATH.($this->data['settings']['options']['PACK'] == self::PACK_EXTENDED ? self::PACK_EXTENDED : self::PACK_STANDARD).'/';
357 //$packPath = self::REMOTE_SETS_PATH.'/';
358
359 if($this->getStep() == 1) // get layout (root) file
360 {
361 $this->data['files'][0] = array(
362 'size' => static::downloadFile(self::REMOTE_LAYOUT_FILE, self::getFileNameByIndex(0), false, $this->data['LOCAL_PATH']),
363 'onlyThese' => array_flip($this->data['settings']['bundles']['allpoints']),
364 'memgroup' => 'static'
365 );
366
367 $this->data['fileDownload']['currentEndPoint'] = 0;
368 $this->data['fileDownload']['currentFileOffset'] = 1;
369 }
370
371 $i =& $this->data['fileDownload']['currentEndPoint'];
372 $j =& $this->data['fileDownload']['currentFileOffset'];
373
374 while($this->checkQuota() && isset($this->data['settings']['bundles']['endpoints'][$i])) // process as many bundles as possible
375 {
376 $ep = $this->data['settings']['bundles']['endpoints'][$i];
377
378 foreach($this->data['requiredGroups'] as $code)
379 {
380 $name = self::getFileNameByIndex($j);
381 $file = $packPath.$ep.'_'.$code.'.csv';
382 try
383 {
384 $this->data['files'][$j] = array(
385 'size' => static::downloadFile($file, $name, false, $this->data['LOCAL_PATH']),
386 'memgroup' => $ep
387 );
388 $j++;
389 }
390 catch(Main\SystemException $e) // 404 or smth - just skip for now
391 {
392 }
393 }
394 $i++;
395 }
396
397 if(!isset($this->data['settings']['bundles']['endpoints'][$i])) // no more bundles to process, all files downloaded
398 {
399 unset($this->data['requiredGroups']);
400 unset($this->data['settings']['bundles']['endpoints']);
401
402 $this->nextStage();
403 return;
404 }
405 }
406
407 $this->nextStep();
408 }
409 }
410
411 protected function getSubpercentForStageDownloadFiles()
412 {
413 $pRange = $this->getCurrentPercentRange();
414
415 $currEp = (int)($this->data['fileDownload']['currentEndPoint'] ?? 0);
416
417 if (!$currEp)
418 {
419 return 0;
420 }
421
422 return round($pRange * ($currEp / count($this->data['settings']['bundles']['endpoints'])));
423 }
424
426 // STAGE 2
427
428 protected function stageDeleteAll()
429 {
430 switch($this->step)
431 {
432 case 0:
433 $this->dbConnection->query('truncate table '.Location\LocationTable::getTableName());
434 break;
435 case 1:
436 $this->dbConnection->query('truncate table '.Location\Name\LocationTable::getTableName());
437 break;
438 case 2:
439 $this->dbConnection->query('truncate table '.Location\ExternalTable::getTableName());
440 break;
441 case 3:
442 Location\GroupLocationTable::deleteAll();
443 break;
444 case 4:
445 Location\SiteLocationTable::deleteAll();
446 break;
447 }
448
449 $this->nextStep();
450
451 if($this->step >= 5)
452 $this->nextStage();
453 }
454
455 protected function getSubpercentForstageDeleteAll()
456 {
457 $pRange = $this->getCurrentPercentRange();
458 $step = $this->getStep();
459
460 $stepsCount = 5;
461
462 if($step >= $stepsCount)
463 return $pRange;
464 else
465 {
466 return round($pRange * ($step / $stepsCount));
467 }
468 }
469
471 // STAGE 2.5
472
473 protected function stageDropIndexes()
474 {
475 $indexes = array(
476 'IX_B_SALE_LOC_MARGINS',
477 'IX_B_SALE_LOC_MARGINS_REV',
478 'IX_B_SALE_LOC_PARENT',
479 'IX_B_SALE_LOC_DL',
480 'IX_B_SALE_LOC_TYPE',
481 'IX_B_SALE_LOC_NAME_NAME_U',
482 'IX_B_SALE_LOC_NAME_LI_LI',
483 'IX_B_SALE_LOC_EXT_LID_SID',
484
485 // old
486 'IXS_LOCATION_COUNTRY_ID',
487 'IXS_LOCATION_REGION_ID',
488 'IXS_LOCATION_CITY_ID',
489 'IX_B_SALE_LOCATION_1',
490 'IX_B_SALE_LOCATION_2',
491 'IX_B_SALE_LOCATION_3'
492 );
493
494 if(!isset($indexes[$this->getStep()]))
495 $this->nextStage();
496 else
497 {
498 $this->dropIndexes($indexes[$this->getStep()]);
499 $this->logMessage('Index dropped: '.$indexes[$this->getStep()]);
500 $this->nextStep();
501 }
502 }
503
504 protected function getSubpercentForStageDropIndexes()
505 {
506 $pRange = $this->getCurrentPercentRange();
507 $step = $this->getStep();
508
509 $indexCount = 14;
510
511 if($step >= $indexCount)
512 return $pRange;
513 else
514 {
515 return round($pRange * ($step / $indexCount));
516 }
517 }
518
520 // STAGE 3
521
522 protected function readBlockFromCurrentFile2()
523 {
524 $fIndex = $this->data['current']['fIndex'];
525 $fName = self::getFileNameByIndex($fIndex);
526 $onlyThese =& $this->data['files'][$fIndex]['onlyThese'];
527
528 //$this->logMessage('READ FROM File: '.$fName.' seek to '.$this->data['current']['bytesRead']);
529
530 if(!isset($this->hitData['csv']))
531 {
532 $file = $this->data['LOCAL_PATH'].$fName;
533
534 if(!file_exists($file) || !is_readable($file))
535 throw new Main\SystemException('Cannot open file '.$file.' for reading');
536
537 $this->logMessage('Chargeing File: '.$fName);
538
539 $this->hitData['csv'] = new CSVReader();
540 $this->hitData['csv']->LoadFile($file);
541 $this->hitData['csv']->AddEventCallback('AFTER_ASSOC_LINE_READ', array($this, 'provideEnFromRu'));
542 }
543
544 $block = $this->hitData['csv']->ReadBlockLowLevel($this->data['current']['bytesRead'], 100);
545
546 $this->data['current']['linesRead'] += count($block);
547
548 if(empty($block))
549 {
550 return array();
551 }
552
553 if($this->hitData['csv']->CheckFileIsLegacy())
554 {
555 $block = self::convertBlock($block);
556 }
557
558 if(is_array($onlyThese))
559 {
560 foreach($block as $i => $line)
561 {
562 if(is_array($onlyThese) && !isset($onlyThese[$line['CODE']]))
563 unset($block[$i]);
564 }
565 }
566
567 //$this->logMessage('Bytes read: '.$this->data['current']['bytesRead']);
568
569 return $block;
570 }
571
572 protected static function checkLocationCodeExists($code)
573 {
574 if ($code == '')
575 return false;
576
577 $dbConnection = Main\HttpApplication::getConnection();
578
579 $code = $dbConnection->getSqlHelper()->forSql($code);
580 $res = $dbConnection->query("select ID from ".Location\LocationTable::getTableName()." where CODE = '".$code."'")->fetch();
581
582 return $res['ID'] ?? false;
583 }
584
585 protected function importBlock(&$block)
586 {
587 if(empty($block))
588 return;
589
590 $gid = $this->getCurrentGid();
591
592 // here must decide, which languages to import
593 $langs = array_flip(
594 array_map(
595 function ($value)
596 {
597 return mb_strtoupper($value);
598 },
599 array_keys(Location\Admin\NameHelper::getLanguageList()))
600 );
601
602 foreach($block as $i => $data)
603 {
604 $code = $data['CODE'];
605
606 // this spike is only for cutting off COUNTRY_DISTRICT
607 // strongly need for the more generalized mechanism for excluding certain types
608 if (!!($this->options['REQUEST']['OPTIONS']['EXCLUDE_COUNTRY_DISTRICT']))
609 {
610 if (!isset($this->data['COUNTRY_2_DISTRICT']))
611 {
612 $this->data['COUNTRY_2_DISTRICT'] = [];
613 }
614
615 if ($data['TYPE_CODE'] == 'COUNTRY')
616 {
617 $this->data['LAST_COUNTRY'] = $data['CODE'];
618 }
619 elseif ($data['TYPE_CODE'] == 'COUNTRY_DISTRICT')
620 {
621 $this->data['COUNTRY_2_DISTRICT'][$code] = $this->data['LAST_COUNTRY'];
622 continue;
623 }
624 else
625 {
626 if (isset($this->data['COUNTRY_2_DISTRICT'][$data['PARENT_CODE']]))
627 {
628 $data['PARENT_CODE'] = $this->data['COUNTRY_2_DISTRICT'][$data['PARENT_CODE']];
629 }
630 }
631 }
632
633 // this spike is only for cutting off COUNTRY_DISTRICT
634 // strongly need for the more generalized mechanism for excluding certain types
635 if (!!($this->options['REQUEST']['OPTIONS']['EXCLUDE_COUNTRY_DISTRICT']))
636 {
637 if (!isset($this->data['COUNTRY_2_DISTRICT']))
638 {
639 $this->data['COUNTRY_2_DISTRICT'] = [];
640 }
641
642 if ($data['TYPE_CODE'] == 'COUNTRY')
643 {
644 $this->data['LAST_COUNTRY'] = $data['CODE'];
645 }
646 elseif ($data['TYPE_CODE'] == 'COUNTRY_DISTRICT')
647 {
648 $this->data['COUNTRY_2_DISTRICT'][$code] = $this->data['LAST_COUNTRY'];
649 continue;
650 }
651 else
652 {
653 if (isset($this->data['COUNTRY_2_DISTRICT'][$data['PARENT_CODE']]))
654 {
655 $data['PARENT_CODE'] = $this->data['COUNTRY_2_DISTRICT'][$data['PARENT_CODE']];
656 }
657 }
658 }
659
660 if(isset($this->data['existedlocs']['static'][$code]) || isset($this->data['existedlocs'][$gid][$code])) // already exists
661 continue;
662
663 if(!isset($this->data['types']['allowed'][$data['TYPE_CODE']])) // disallowed
664 continue;
665
666 // have to check existence first
667 if(!$this->data['TABLE_WERE_EMPTY'])
668 {
669 $existedId = $this->checkLocationCodeExists($code);
670
671 if(intval($existedId))
672 {
673 $this->data['existedlocs'][$gid][$code] = $existedId;
674 continue;
675 }
676 }
677
679 // transform parent
680 if($data['PARENT_CODE'] <> '')
681 {
682 if(isset($this->data['existedlocs']['static'][$data['PARENT_CODE']]))
683 {
684 $data['PARENT_ID'] = $this->data['existedlocs']['static'][$data['PARENT_CODE']];
685 }
686 elseif(isset($this->data['existedlocs'][$gid][$data['PARENT_CODE']]))
687 {
688 $data['PARENT_ID'] = $this->data['existedlocs'][$gid][$data['PARENT_CODE']];
689 }
690 else
691 {
692 $data['PARENT_ID'] = 0;
693 }
694 }
695 else
696 {
697 $data['PARENT_ID'] = 0;
698 }
699
700 unset($data['PARENT_CODE']);
701
703 // transform type
704 $data['TYPE_ID'] = $this->data['types']['code2id'][$data['TYPE_CODE']];
705 unset($data['TYPE_CODE']);
706
708 // add
709 $names = $data['NAME'];
710 unset($data['NAME']);
711
712 $external = $data['EXT'];
713 unset($data['EXT']);
714
715 $data['LONGITUDE'] = (float)($data['LONGITUDE'] ?? 0);
716 $data['LATITUDE'] = (float)($data['LATITUDE'] ?? 0);
717 if(!$this->checkExternalServiceAllowed('GEODATA'))
718 {
719 $data['LONGITUDE'] = 0;
720 $data['LATITUDE'] = 0;
721 }
722
723 $locationId = $this->hitData['HANDLES']['LOCATION']->insert($data);
724
725 // store for further PARENT_CODE to PARENT_ID mapping
726 //if(!strlen($this->data['types']['last']) || $this->data['types']['last'] != $data['TYPE_CODE'])
727 $this->data['existedlocs'][$gid][$data['CODE']] = $locationId;
728
730 // add names
731 if(is_array($names) && !empty($names))
732 {
733 if(is_array($langs))
734 {
735 foreach($langs as $lid => $f)
736 {
737 $lid = ToLower($lid);
738 $toAdd = static::getTranslatedName($names, $lid);
739
740 $this->hitData['HANDLES']['NAME']->insert(array(
741 'NAME' => $toAdd['NAME'],
742 'NAME_UPPER' => ToUpper($toAdd['NAME']),
743 'LANGUAGE_ID' => $lid,
744 'LOCATION_ID' => $locationId
745 ));
746 }
747 }
748 }
749
751 // add external
752 if(is_array($external) && !empty($external))
753 {
754 foreach($external as $sCode => $values)
755 {
756 if($this->checkExternalServiceAllowed($sCode))
757 {
758 $serviceId = $this->data['externalService']['code2id'][$sCode];
759
760 if(!$serviceId)
761 throw new Main\SystemException('Location import failed: external service doesnt exist');
762
763 if($sCode == 'ZIP_LOWER')
764 {
765 if($values == '')
766 continue;
767
768 $values = explode(',', $values);
769
770 if(!is_array($values))
771 continue;
772
773 $values = array_unique($values);
774 }
775
776 if(is_array($values))
777 {
778 foreach($values as $val)
779 {
780 if($val == '')
781 continue;
782
783 $this->hitData['HANDLES']['EXTERNAL']->insert(array(
784 'SERVICE_ID' => $serviceId,
785 'XML_ID' => $val,
786 'LOCATION_ID' => $locationId
787 ));
788 }
789 }
790 }
791 }
792 }
793 }
794 }
795
796 protected function getCurrentGid()
797 {
798 return $this->data['files'][$this->data['current']['fIndex']]['memgroup'];
799 }
800
801 protected function stageProcessFiles()
802 {
803 if($this->dbConnType == self::DB_TYPE_ORACLE)
804 $mtu = self::INSERTER_MTU_ORACLE;
805 else
806 $mtu = self::INSERTER_MTU;
807
808 $this->hitData['HANDLES']['LOCATION'] = new BlockInserter(array(
809 'entityName' => '\Bitrix\Sale\Location\LocationTable',
810 'exactFields' => array('CODE', 'TYPE_ID', 'PARENT_ID', 'LATITUDE', 'LONGITUDE'),
811 'parameters' => array(
812 'autoIncrementFld' => 'ID',
813 'mtu' => $mtu
814 )
815 ));
816
817 $this->hitData['HANDLES']['NAME'] = new BlockInserter(array(
818 'entityName' => '\Bitrix\Sale\Location\Name\LocationTable',
819 'exactFields' => array('NAME', 'NAME_UPPER', 'LANGUAGE_ID', 'LOCATION_ID'),
820 'parameters' => array(
821 'mtu' => $mtu
822 )
823 ));
824
825 $this->hitData['HANDLES']['EXTERNAL'] = new BlockInserter(array(
826 'entityName' => '\Bitrix\Sale\Location\ExternalTable',
827 'exactFields' => array('SERVICE_ID', 'XML_ID', 'LOCATION_ID'),
828 'parameters' => array(
829 'mtu' => $mtu
830 )
831 ));
832
833 if($this->getStep() == 0)
834 {
835 // set initial values
836 $this->data['current'] = array(
837 'fIndex' => 0,
838 'bytesRead' => 0, // current file bytes read
839 'linesRead' => 0
840 );
841
842 $this->hitData['HANDLES']['LOCATION']->resetAutoIncrementFromIndex(); // synchronize sequences, etc...
843
844 // check if we are empty
845 $this->data['TABLE_WERE_EMPTY'] = Location\LocationTable::getCountByFilter() == 0;
846
847 $this->buildStaticLocationIndex();
848 }
849
850 while($this->checkQuota())
851 {
852 $block = $this->readBlockFromCurrentFile2();
853 $this->importBlock($block);
854
855 // clean memory
856 $this->manageExistedLocationIndex(array($this->getCurrentGid()));
857
858 // or the current file is completely exhausted
859 if($this->checkFileCompletelyRead())
860 {
861 //$this->logMessage('Lines read: '.$this->data['current']['linesRead']);
862
863 // charge next file
864 unset($this->hitData['csv']);
865 $this->data['current']['fIndex']++; // next file to go
866 $this->data['current']['bytesRead'] = 0; // read counter from the beginning
867 $this->data['current']['linesRead'] = 0;
868 $this->data['current']['legacy'] = array(); // drop legacy data of the file, if were any. bye-bye
869
870 // may be that is all?
871 if($this->checkAllFilesRead())
872 {
873 unset($this->data['existedlocs']); // uff, remove that huge array at last
874
875 $this->nextStage();
876 break;
877 }
878 }
879
880 $this->nextStep();
881 }
882
883 $this->hitData['HANDLES']['LOCATION']->flush();
884 $this->hitData['HANDLES']['NAME']->flush();
885 $this->hitData['HANDLES']['EXTERNAL']->flush();
886
887 $this->logMessage('Inserted, go next: '.$this->getHitTimeString());
888
889 $this->logMemoryUsage();
890 }
891
892 protected function getSubpercentForStageProcessFiles()
893 {
894 $pRange = $this->getStagePercent($this->stage) - $this->getStagePercent($this->stage - 1);
895
896 $totalSize = 0;
897 $fileBytesRead = 0;
898
899 if(!isset($this->data['current']['fIndex']))
900 return 0;
901
902 $fIndex = $this->data['current']['fIndex'];
903
904 $i = -1;
905 foreach($this->data['files'] as $file)
906 {
907 $i++;
908
909 if($i < $fIndex)
910 $fileBytesRead += $file['size'];
911
912 $totalSize += $file['size'];
913 }
914
915 if(!$totalSize)
916 return 0;
917
918 return round($pRange * (intval($fileBytesRead + $this->data['current']['bytesRead']) / $totalSize));
919 }
920
922 // STAGE 4
923
924 protected function stageIntegrityPreserve()
925 {
926 $lay = $this->getRemoteLayout(true);
927
928 $this->restoreIndexes('IX_B_SALE_LOC_PARENT');
929
930 $res = Location\LocationTable::getList(array(
931 'select' => array(
932 'ID', 'CODE'
933 ),
934 'filter' => array(
935 '=PARENT_ID' => 0
936 )
937 ));
938 $relations = array();
939 $code2id = array();
940 while($item = $res->fetch())
941 {
942 if(isset($lay[$item['CODE']]) && ((string) $lay[$item['CODE']]['PARENT_CODE'] != '')/*except root*/)
943 $relations[$item['CODE']] = $lay[$item['CODE']]['PARENT_CODE'];
944 // relations is a match between codes from the layout file
945
946 $code2id[$item['CODE']] = $item['ID'];
947 }
948
949 $parentCode2id = $this->getLocationCodeToIdMap($relations);
950
951 foreach($code2id as $code => $id)
952 {
953 if (!isset($relations[$code]))
954 {
955 continue;
956 }
957 if ((string)($parentCode2id[$relations[$code]] ?? '') !== '') // parent really exists
958 {
959 $res = Location\LocationTable::update(
960 $id,
961 ['PARENT_ID' => $parentCode2id[$relations[$code]]]
962 );
963 if (!$res->isSuccess())
964 {
965 throw new Main\SystemException('Cannot make element become a child of its legal parent');
966 }
967 }
968 }
969
970 $this->nextStage();
971 }
972
974 // STAGE 5
975
976 protected function stageRebalanceWalkTree()
977 {
978 if(!isset($this->data['rebalance']['queue']))
979 {
980 $this->restoreIndexes('IX_B_SALE_LOC_PARENT');
981
982 $this->logMessage('initialize Queue');
983
984 $this->data['rebalance']['margin'] = -1;
985 $this->data['processed'] = 0;
986 $this->data['rebalance']['queue'] = array(array('I' => 'root', 'D' => 0));
987
988 $tableName = Location\LocationTable::getTableName();
989 $res = Main\HttpApplication::getConnection()->query("select count(*) as CNT from {$tableName}")->fetch();
990
991 $this->data['rebalance']['cnt'] = intval($res['CNT']);
992 }
993
994 $i = -1;
995 while(!empty($this->data['rebalance']['queue']) && $this->checkQuota())
996 {
997 $i++;
998
999 $node =& $this->data['rebalance']['queue'][0];
1000
1001 if(isset($node['L']))
1002 {
1003 // we have already been here
1004 array_shift($this->data['rebalance']['queue']);
1005 if($node['I'] != 'root') // we dont need for ROOT item in outgoing
1006 {
1007 $this->acceptRebalancedNode(array(
1008 'I' => $node['I'],
1009 'D' => $node['D'],
1010 'L' => $node['L'],
1011 'R' => ++$this->data['rebalance']['margin']
1012 ));
1013 }
1014 else
1015 $this->data['rebalance']['margin']++;
1016 }
1017 else
1018 {
1019 $a = $this->getCachedBundle($node['I']);
1020
1021 if(!empty($a))
1022 {
1023 // go deeper
1024 $node['L'] = ++$this->data['rebalance']['margin'];
1025
1026 foreach($a as $id)
1027 {
1028 if($this->checkNodeIsParent($id))
1029 {
1030 array_unshift($this->data['rebalance']['queue'], array('I' => $id, 'D' => $node['D'] + 1));
1031 }
1032 else // we dont need to put it to the query
1033 {
1034 $this->acceptRebalancedNode(array(
1035 'I' => $id,
1036 'D' => $node['D'] + 1,
1037 'L' => ++$this->data['rebalance']['margin'],
1038 'R' => ++$this->data['rebalance']['margin']
1039 ));
1040 }
1041 }
1042 }
1043 else
1044 {
1045 array_shift($this->data['rebalance']['queue']);
1046 $this->acceptRebalancedNode(array(
1047 'I' => $node['I'],
1048 'D' => $node['D'],
1049 'L' => ++$this->data['rebalance']['margin'],
1050 'R' => ++$this->data['rebalance']['margin']
1051 ));
1052 }
1053 }
1054 }
1055
1056 $this->logMessage('Q size is '.count($this->data['rebalance']['queue']).' already processed: '.$this->data['processed'].'/'.$this->data['rebalance']['cnt']);
1057 $this->logMemoryUsage();
1058
1059 if(empty($this->data['rebalance']['queue']))
1060 {
1061 // last flush & then merge
1062 $this->mergeRebalancedNodes();
1063
1064 $this->nextStage();
1065 return;
1066 }
1067
1068 if($this->rebalanceInserter)
1069 $this->rebalanceInserter->flush();
1070
1071 $this->nextStep();
1072 }
1073
1074 protected function getSubpercentForStageRebalanceWalkTree()
1075 {
1076 $processed = $this->data['processed'] ?? 0;
1077 $cnt = $this->data['rebalance']['cnt'] ?? 0;
1078 if (!$processed || !$cnt)
1079 return 0;
1080
1081 $pRange = $this->getCurrentPercentRange();
1082 $part = round($pRange * ($processed / $cnt));
1083
1084 return min($part, $pRange);
1085 }
1086
1088 // STAGE 6
1089
1090 protected function stageRebalanceCleanupTempTable()
1091 {
1092 $this->dropTempTable();
1093 $this->nextStage();
1094 }
1095
1097 // STAGE 7
1098
1099 protected function stageRestoreIndexes()
1100 {
1101 $indexes = array(
1102 'IX_B_SALE_LOC_MARGINS',
1103 'IX_B_SALE_LOC_MARGINS_REV',
1104 //'IX_B_SALE_LOC_PARENT', // already restored at REBALANCE_WALK_TREE stage
1105 'IX_B_SALE_LOC_DL',
1106 'IX_B_SALE_LOC_TYPE',
1107 'IX_B_SALE_LOC_NAME_NAME_U',
1108 'IX_B_SALE_LOC_NAME_LI_LI',
1109 'IX_B_SALE_LOC_EXT_LID_SID',
1110
1111 // legacy
1112 'IXS_LOCATION_COUNTRY_ID',
1113 'IXS_LOCATION_REGION_ID',
1114 'IXS_LOCATION_CITY_ID',
1115 );
1116
1117 if(isset($indexes[$this->getStep()]))
1118 {
1119 $this->restoreIndexes($indexes[$this->getStep()]);
1120 $this->logMessage('Index restored: '.$indexes[$this->getStep()]);
1121 $this->nextStep();
1122 }
1123 else
1124 {
1125 Location\LocationTable::resetLegacyPath(); // for backward compatibility
1126 $this->nextStage();
1127 }
1128 }
1129
1130 protected function getSubpercentForStageRestoreIndexes()
1131 {
1132 $pRange = $this->getCurrentPercentRange();
1133 $step = $this->getStep();
1134
1135 $stepCount = 12;
1136
1137 if($step >= $stepCount)
1138 {
1139 return $pRange;
1140 }
1141 else
1142 {
1143 return round($pRange * ($step / $stepCount));
1144 }
1145 }
1146
1148 // about stage util functions
1149
1150 protected function getLanguageId(): string
1151 {
1152 return $this->options['LANGUAGE_ID'];
1153 }
1154
1158 public function getTypes()
1159 {
1160 $result = array();
1161 $res = Location\TypeTable::getList(array(
1162 'select' => array(
1163 'CODE', 'TNAME' => 'NAME.NAME'
1164 ),
1165 'filter' => array(
1166 'NAME.LANGUAGE_ID' => $this->getLanguageId()
1167 ),
1168 'order' => array(
1169 'SORT' => 'asc',
1170 'NAME.NAME' => 'asc'
1171 )
1172 ));
1173 while($item = $res->fetch())
1174 $result[$item['CODE']] = $item['TNAME'];
1175
1176 return $result;
1177 }
1178
1179 public function getStatisticsAll()
1180 {
1181 $this->getStatistics();
1182
1183 return $this->stat;
1184 }
1185
1186 public function getStatistics($type = 'TOTAL')
1187 {
1188 if(empty($this->stat))
1189 {
1190 $types = \Bitrix\Sale\Location\Admin\TypeHelper::getTypes(array('LANGUAGE_ID' => $this->getLanguageId()));
1191
1192 $res = Location\LocationTable::getList(array(
1193 'select' => array(
1194 'CNT',
1195 'TCODE' => 'TYPE.CODE'
1196 ),
1197 'group' => array(
1198 'TYPE_ID'
1199 )
1200 ));
1201 $total = 0;
1202 $stat = array();
1203 while($item = $res->fetch())
1204 {
1205 $total += intval($item['CNT']);
1206 $stat[$item['TCODE']] = $item['CNT'];
1207 }
1208
1209 foreach($types as $code => $data)
1210 {
1211 $this->stat[$code] = array(
1212 'NAME' => $data['NAME_CURRENT'],
1213 'CODE' => $code,
1214 'CNT' => isset($stat[$code]) ? intval($stat[$code]) : 0,
1215 );
1216 }
1217
1218 $this->stat['TOTAL'] = array('CNT' => $total, 'CODE' => 'TOTAL');
1219
1220 $res = Location\GroupTable::getList(array(
1221 'runtime' => array(
1222 'CNT' => array(
1223 'data_type' => 'integer',
1224 'expression' => array(
1225 'COUNT(*)'
1226 )
1227 )
1228 ),
1229 'select' => array(
1230 'CNT'
1231 )
1232 ))->fetch();
1233
1234 $this->stat['GROUPS'] = array('CNT' => intval($res['CNT']), 'CODE' => 'GROUPS');
1235 }
1236
1237 return intval($this->stat[$type]['CNT']);
1238 }
1239
1240 public function determineLayoutToImport()
1241 {
1242 $lay = $this->getRemoteLayout(true);
1243
1244 $parentness = array();
1245 foreach($lay as $data)
1246 {
1247 $parentCode = (string)($data['PARENT_CODE'] ?? '');
1248 if ($parentCode === '')
1249 {
1250 continue;
1251 }
1252 $parentness[$parentCode] ??= 0;
1253 $parentness[$parentCode]++;
1254 }
1255
1256 $bundles = array_flip($this->data['settings']['sets']);
1257
1258 $selectedLayoutParts = array();
1259 foreach($bundles as $bundle => $void)
1260 {
1261 if(!isset($lay[$bundle]))
1262 throw new Main\SystemException('Unknown bundle passed in request');
1263
1264 // obtaining intermediate chain parts
1265 $chain = array();
1266
1267 $currentBundle = $bundle;
1268 $i = -1;
1269 while($currentBundle)
1270 {
1271 $i++;
1272
1273 if($i > 50) // smth is really bad
1274 throw new Main\SystemException('Too deep recursion got when building chains. Layout file is broken');
1275
1276 if(isset($lay[$currentBundle]))
1277 {
1278 $chain[] = $currentBundle;
1279 if($lay[$currentBundle]['PARENT_CODE'] <> '')
1280 {
1281 $currentBundle = $lay[$currentBundle]['PARENT_CODE'];
1282
1283 if(!isset($lay[$currentBundle]))
1284 {
1285 throw new Main\SystemException('Unknown parent bundle found ('.$currentBundle.'). Layout file is broken');
1286 }
1287 }
1288 else
1289 {
1290 $currentBundle = false;
1291 }
1292 }
1293 }
1294
1295 if(is_array($chain) && !empty($chain))
1296 {
1297 $chain = array_reverse($chain);
1298
1299 // find first occurance of selected bundle in the chain
1300 $subChain = array();
1301 foreach($chain as $i => $node)
1302 {
1303 if(isset($bundles[$node]))
1304 {
1305 $subChain = array_slice($chain, $i);
1306 break;
1307 }
1308 }
1309
1310 if(!empty($subChain))
1311 $selectedLayoutParts = array_merge($selectedLayoutParts, $subChain);
1312 }
1313 }
1314
1315 //$this->data['settings']['layout'] = $lay;
1316 $selectedLayoutParts = array_unique($selectedLayoutParts);
1317
1318 $this->data['settings']['bundles'] = array('endpoints' => array(), 'allpoints' => $selectedLayoutParts);
1319
1320 foreach($selectedLayoutParts as $bCode)
1321 {
1322 if(!isset($parentness[$bCode]))
1323 $this->data['settings']['bundles']['endpoints'][] = $bCode;
1324 //else
1325 // $this->data['settings']['bundles']['middlepoints'][] = $bCode;
1326 }
1327 unset($this->data['settings']['sets']);
1328 }
1329
1330 public function convertBlock($block)
1331 {
1332 $converted = array();
1333
1334 foreach($block as $line)
1335 {
1336 if($line[0] == 'S')
1337 $typeCode = 'COUNTRY';
1338 elseif($line[0] == 'R')
1339 $typeCode = 'REGION';
1340 elseif($line[0] == 'T')
1341 $typeCode = 'CITY';
1342 else
1343 throw new Main\SystemException('Unknown type found in legacy file');
1344
1345 $code = md5(implode(':', $line));
1346
1347 if($typeCode == 'REGION')
1348 $parentCode = $this->data['current']['legacy']['lastCOUNTRY'];
1349 elseif($typeCode == 'CITY')
1350 $parentCode = $this->data['current']['legacy']['lastParent'];
1351 else
1352 $parentCode = '';
1353
1354 if($typeCode != 'CITY')
1355 {
1356 $this->data['current']['legacy']['last'.$typeCode] = $code;
1357 $this->data['current']['legacy']['lastParent'] = $code;
1358 }
1359
1360 $cLine = array(
1361 'CODE' => $code,
1362 'TYPE_CODE' => $typeCode,
1363 'PARENT_CODE' => $parentCode
1364 );
1365
1366 $lang = false;
1367 $expectLang = true;
1368 $lineLen = count($line);
1369 for($k = 1; $k < $lineLen; $k++)
1370 {
1371 if($expectLang)
1372 {
1373 $lang = $line[$k];
1374 }
1375 else
1376 {
1377 $cLine['NAME'][ToUpper($lang)]['NAME'] = $line[$k];
1378 }
1379
1380 $expectLang = !$expectLang;
1381 }
1382
1383 $converted[] = $cLine;
1384 }
1385
1386 return $converted;
1387 }
1388
1389 public function checkSource($sType)
1390 {
1391 return $this->data['settings']['options']['SOURCE'] == $sType;
1392 }
1393
1394 // download layout from server
1395 public function getRemoteLayout($getFlat = false)
1396 {
1397 [$localPath, $tmpDirCreated] = $this->getTemporalDirectory();
1398
1399 static::downloadFile(self::REMOTE_LAYOUT_FILE, self::LOCAL_LAYOUT_FILE, false, $localPath);
1400
1401 $csv = new CSVReader();
1402 $csv->AddEventCallback('AFTER_ASSOC_LINE_READ', array($this, 'provideEnFromRu'));
1403 $res = $csv->ReadBlock($localPath.self::LOCAL_LAYOUT_FILE);
1404
1405 $result = array();
1406 if($getFlat)
1407 {
1408 foreach($res as $line)
1409 $result[$line['CODE']] = $line;
1410 $csv->CloseFile();
1411 return $result;
1412 }
1413
1414 $lang = $this->getLanguageId();
1415
1416 foreach($res as $line)
1417 {
1418 $line['NAME'][ToUpper($lang)] = static::getTranslatedName($line['NAME'], $lang);
1419 $result[$line['PARENT_CODE']][$line['CODE']] = $line;
1420 }
1421 $csv->CloseFile();
1422
1423 if($tmpDirCreated)
1424 {
1425 $this->deleteDirectory($localPath);
1426 }
1427
1428 return $result;
1429 }
1430
1431 // download types from server
1432 public function getRemoteTypes()
1433 {
1434 if(!$this->useCache || !isset($this->data['settings']['remote']['types']))
1435 {
1436 [$localPath, $tmpDirCreated] = $this->getTemporalDirectory();
1437
1438 static::downloadFile(self::REMOTE_TYPE_FILE, self::LOCAL_TYPE_FILE, false, $localPath);
1439
1440 $csv = new CSVReader();
1441 $csv->AddEventCallback('AFTER_ASSOC_LINE_READ', array($this, 'provideEnFromRu'));
1442 $res = $csv->ReadBlock($localPath.self::LOCAL_TYPE_FILE);
1443
1444 $result = array();
1445 foreach($res as $line)
1446 $result[$line['CODE']] = $line;
1447
1448 $this->data['settings']['remote']['types'] = $result;
1449 $csv->CloseFile();
1450 if($tmpDirCreated)
1451 {
1452 $this->deleteDirectory($localPath);
1453 }
1454 }
1455
1456 return $this->data['settings']['remote']['types'];
1457 }
1458
1459 // download external services from server
1460 public function getRemoteExternalServices()
1461 {
1462 if(!$this->useCache || !isset($this->data['settings']['remote']['external_services']))
1463 {
1464 [$localPath, $tmpDirCreated] = $this->getTemporalDirectory();
1465
1466 static::downloadFile(self::REMOTE_EXTERNAL_SERVICE_FILE, self::LOCAL_EXTERNAL_SERVICE_FILE, false, $localPath);
1467
1468 $csv = new CSVReader();
1469 $res = $csv->ReadBlock($localPath.self::LOCAL_EXTERNAL_SERVICE_FILE);
1470
1471 $result = array();
1472 foreach($res as $line)
1473 $result[$line['CODE']] = $line;
1474
1475 $this->data['settings']['remote']['external_services'] = $result;
1476 $csv->CloseFile();
1477 if($tmpDirCreated)
1478 {
1479 $this->deleteDirectory($localPath);
1480 }
1481 }
1482
1483 return $this->data['settings']['remote']['external_services'];
1484 }
1485
1486 // download type groups from server
1487 public function getRemoteTypeGroups()
1488 {
1489 if(!$this->useCache || !isset($this->data['settings']['remote']['typeGroups']))
1490 {
1491 [$localPath, $tmpDirCreated] = $this->getTemporalDirectory();
1492
1493 static::downloadFile(self::REMOTE_TYPE_GROUP_FILE, self::LOCAL_TYPE_GROUP_FILE, false, $localPath);
1494
1495 $csv = new CSVReader();
1496 $res = $csv->ReadBlock($localPath.self::LOCAL_TYPE_GROUP_FILE);
1497
1498 $result = array();
1499 foreach($res as $line)
1500 {
1501 $result[$line['CODE']] = explode(':', $line['TYPES']);
1502 }
1503
1504 $this->data['settings']['remote']['typeGroups'] = $result;
1505 $csv->CloseFile();
1506 if($tmpDirCreated)
1507 {
1508 $this->deleteDirectory($localPath);
1509 }
1510 }
1511
1512 return $this->data['settings']['remote']['typeGroups'];
1513 }
1514
1515 public function getTypeLevels($langId = LANGUAGE_ID): array
1516 {
1517 $types = $this->getRemoteTypes();
1518 $levels = array();
1519
1520 if(!isset($langId))
1521 {
1522 $langId = $this->getLanguageId();
1523 }
1524
1525 foreach($types as $type)
1526 {
1527 if($type['SELECTORLEVEL'] = intval($type['SELECTORLEVEL']))
1528 {
1529 $name = static::getTranslatedName($type['NAME'], $langId);
1530 $levels[$type['SELECTORLEVEL']]['NAMES'][] = $name['NAME'];
1531 $levels[$type['SELECTORLEVEL']]['TYPES'][] = $type['CODE'];
1532
1533 $levels[$type['SELECTORLEVEL']]['DEFAULT'] = ($type['DEFAULTSELECT'] == '1');
1534 }
1535 }
1536
1537 foreach($levels as &$group)
1538 $group['NAMES'] = implode(', ', $group['NAMES']);
1539
1540 ksort($levels, SORT_NUMERIC);
1541
1542 return $levels;
1543 }
1544
1545 public static function getSiteLanguages()
1546 {
1547 static $langs;
1548
1549 if($langs == null)
1550 {
1551 $langs = array();
1552
1553 $res = \Bitrix\Main\SiteTable::getList(array('filter' => array('ACTIVE' => 'Y'), 'select' => array('LANGUAGE_ID'), 'group' => array('LANGUAGE_ID')));
1554 while($item = $res->fetch())
1555 {
1556 $langs[ToUpper($item['LANGUAGE_ID'])] = true;
1557 }
1558
1559 $langs = array_unique(array_keys($langs)); // all active sites languages
1560 }
1561
1562 return $langs;
1563 }
1564
1565 public function getRequiredLanguages()
1566 {
1567 $required = array(ToUpper($this->getLanguageId()));
1568
1569 $langs = Location\Admin\NameHelper::getLanguageList();
1570 if(isset($langs['en']))
1571 $required[] = 'EN';
1572
1573 return array_unique(array_merge($required, static::getSiteLanguages())); // current language plus for all active sites
1574 }
1575
1576 // read type.csv and build type table
1577 protected function buildTypeTable()
1578 {
1579 $this->data['types_processed'] ??= false;
1580 if ($this->data['types_processed'])
1581 {
1582 return;
1583 }
1584
1585 // read existed
1586 $existed = static::getExistedTypes();
1587
1588 if($this->checkSource(self::SOURCE_REMOTE))
1589 {
1590 $rTypes = $this->getRemoteTypes();
1591 $this->getRemoteTypeGroups();
1592
1593 $existed = static::createTypes($rTypes, $existed);
1594
1595 if(intval($dl = $this->data['settings']['options']['DEPTH_LIMIT']))
1596 {
1597 // here we must find out what types we are allowed to read
1598
1599 $typesGroupped = $this->getTypeLevels();
1600
1601 if(!isset($typesGroupped[$dl]))
1602 throw new Main\SystemException('Unknow type level to limit');
1603
1604 $allowed = [];
1605 foreach($typesGroupped as $gId => $group)
1606 {
1607 if($gId > $dl)
1608 break;
1609
1610 foreach($group['TYPES'] as $type)
1611 $allowed[] = $type;
1612 }
1613
1614 $this->data['types']['allowed'] = $allowed;
1615 }
1616 else
1617 {
1618 $allowed = [];
1619 foreach($rTypes as $type)
1620 {
1621 $allowed[] = $type['CODE'];
1622 }
1623 $this->data['types']['allowed'] = $allowed;
1624 }
1625 }
1626 elseif($this->checkSource(self::SOURCE_FILE))
1627 {
1628 $codes = [];
1629 if(!empty($existed) && is_array($existed))
1630 {
1631 $codes = array_keys($existed);
1632 }
1633
1634 $this->data['types']['allowed'] = $codes;
1635 }
1636
1637 if (empty($this->data['types']['allowed']))
1638 {
1639 $this->data['types']['last'] = null;
1640 }
1641 else
1642 {
1643 $this->data['types']['last'] = $this->data['types']['allowed'][count($this->data['types']['allowed']) - 1];
1644 }
1645 $this->data['types']['allowed'] = array_flip($this->data['types']['allowed']);
1646
1647 $this->data['types']['code2id'] = $existed;
1648 $this->data['types_processed'] = true;
1649 }
1650
1651 protected function checkExternalServiceAllowed($code)
1652 {
1653 if($this->data['settings']['additional'] === false)
1654 return true; // ANY
1655
1656 if($code == 'ZIP_LOWER')
1657 $code = 'ZIP';
1658
1659 return isset($this->data['settings']['additional'][$code]);
1660 }
1661
1662 protected function buildExternalSerivceTable()
1663 {
1664 $this->data['external_processed'] ??= false;
1665 if ($this->data['external_processed'])
1666 {
1667 return;
1668 }
1669
1670 // read existed
1671 $existed = static::getExistedServices();
1672
1673 if($this->checkSource(self::SOURCE_REMOTE))
1674 {
1675 $external = $this->getRemoteExternalServices();
1676 foreach($external as $line)
1677 {
1678 if(!isset($existed[$line['CODE']]) && $this->checkExternalServiceAllowed($line['CODE']))
1679 {
1680 $existed[$line['CODE']] = static::createService($line);
1681 }
1682 }
1683 unset($this->data['settings']['remote']['external_services']);
1684 }
1685
1686 $this->data['externalService']['code2id'] = $existed;
1687 $this->data['external_processed'] = true;
1688 }
1689
1690 protected function buildStaticLocationIndex()
1691 {
1692 $parameters = array(
1693 'select' => array('ID', 'CODE')
1694 );
1695
1696 // get static index, it will be always in memory
1697 $parameters['filter'] = array('TYPE.CODE' => array('COUNTRY', 'COUNTRY_DISTRICT', 'REGION')); // todo: from typegroup later
1698
1699 $this->data['existedlocs'] = array('static' => array());
1700 $res = Location\LocationTable::getList($parameters);
1701 while($item = $res->fetch())
1702 $this->data['existedlocs']['static'][$item['CODE']] = $item['ID']; // get existed, "static" index
1703 }
1704
1705 protected function getLocationCodeToIdMapQuery($buffer, &$result)
1706 {
1707 $res = Location\LocationTable::getList(array('filter' => array('CODE' => $buffer), 'select' => array('ID', 'CODE')));
1708 while($item = $res->fetch())
1709 $result[$item['CODE']] = $item['ID'];
1710 }
1711
1712 protected function getLocationCodeToIdMap($codes)
1713 {
1714 if(empty($codes))
1715 return array();
1716
1717 $i = -1;
1718 $buffer = array();
1719 $result = array();
1720 foreach($codes as $code)
1721 {
1722 $i++;
1723
1724 if($i == self::MAX_CODE_FETCH_BLOCK_LEN)
1725 {
1726 $this->getLocationCodeToIdMapQuery($buffer, $result);
1727
1728 $buffer = array();
1729 $i = -1;
1730 }
1731
1732 if($code <> '')
1733 {
1734 $buffer[] = $code;
1735 }
1736 }
1737
1738 // last iteration
1739 $this->getLocationCodeToIdMapQuery($buffer, $result);
1740
1741 return $result;
1742 }
1743
1744 protected function manageExistedLocationIndex($memGroups)
1745 {
1746 $before = implode(', ', array_keys($this->data['existedlocs']));
1747
1748 $cleaned = false;
1749 foreach($this->data['existedlocs'] as $gid => $bundles)
1750 {
1751 if($gid == 'static' || in_array($gid, $memGroups))
1752 continue;
1753
1754 $cleaned = true;
1755
1756 $this->logMessage('Memory clean: REMOVING Group '.$gid);
1757 unset($this->data['existedlocs'][$gid]);
1758 }
1759
1760 if($cleaned)
1761 {
1762 $this->logMessage('BEFORE memgroups: '.$before);
1763 $this->logMessage('Clear all but '.$memGroups[0]);
1764
1765 $this->logMessage('AFTER memgroups: '.implode(', ', array_keys($this->data['existedlocs'])));
1766 }
1767 }
1768
1770 // about file and network I/O
1771
1772 protected function checkIndexExistsByName($indexName, $tableName)
1773 {
1774 $indexName = $this->dbHelper->forSql(trim($indexName));
1775 $tableName = $this->dbHelper->forSql(trim($tableName));
1776
1777 if(!mb_strlen($indexName) || !mb_strlen($tableName))
1778 return false;
1779
1780 if($this->dbConnType == self::DB_TYPE_MYSQL)
1781 $res = $this->dbConnection->query("show index from ".$tableName);
1782 elseif($this->dbConnType == self::DB_TYPE_ORACLE)
1783 $res = $this->dbConnection->query("SELECT INDEX_NAME as Key_name FROM USER_IND_COLUMNS WHERE TABLE_NAME = '".ToUpper($tableName)."'");
1784 elseif($this->dbConnType == self::DB_TYPE_MSSQL)
1785 {
1786 $res = $this->dbConnection->query("SELECT si.name Key_name
1787 FROM sysindexkeys s
1788 INNER JOIN syscolumns c ON s.id = c.id AND s.colid = c.colid
1789 INNER JOIN sysobjects o ON s.id = o.Id AND o.xtype = 'U'
1790 LEFT JOIN sysindexes si ON si.indid = s.indid AND si.id = s.id
1791 WHERE o.name = '".ToUpper($tableName)."'");
1792 }
1793
1794 while($item = $res->fetch())
1795 {
1796 if (isset($item['Key_name']) && $item['Key_name'] === $indexName)
1797 {
1798 return true;
1799 }
1800 if (isset($item['KEY_NAME']) && $item['KEY_NAME'] === $indexName)
1801 {
1802 return true;
1803 }
1804 }
1805
1806 return false;
1807 }
1808
1809 protected function dropIndexByName($indexName, $tableName)
1810 {
1811 $indexName = $this->dbHelper->forSql(trim($indexName));
1812 $tableName = $this->dbHelper->forSql(trim($tableName));
1813
1814 if(!mb_strlen($indexName) || !mb_strlen($tableName))
1815 return false;
1816
1817 if(!$this->checkIndexExistsByName($indexName, $tableName))
1818 return false;
1819
1820 if($this->dbConnType == self::DB_TYPE_MYSQL)
1821 $this->dbConnection->query("alter table {$tableName} drop index {$indexName}");
1822 elseif($this->dbConnType == self::DB_TYPE_ORACLE)
1823 $this->dbConnection->query("drop index {$indexName}");
1824 elseif($this->dbConnType == self::DB_TYPE_MSSQL)
1825 $this->dbConnection->query("drop index {$indexName} on {$tableName}");
1826
1827 return true;
1828 }
1829
1830 public static function getIndexMap()
1831 {
1832 $locationTable = Location\LocationTable::getTableName();
1833 $locationNameTable = Location\Name\LocationTable::getTableName();
1834 $locationExternalTable = Location\ExternalTable::getTableName();
1835
1836 return array(
1837 'IX_SALE_LOCATION_MARGINS' => array('TABLE' => $locationTable, 'COLUMNS' => array('LEFT_MARGIN', 'RIGHT_MARGIN')),
1838 'IX_SALE_LOCATION_MARGINS_REV' => array('TABLE' => $locationTable, 'COLUMNS' => array('RIGHT_MARGIN', 'LEFT_MARGIN')),
1839 'IX_SALE_LOCATION_PARENT' => array('TABLE' => $locationTable, 'COLUMNS' => array('PARENT_ID')),
1840 'IX_SALE_LOCATION_DL' => array('TABLE' => $locationTable, 'COLUMNS' => array('DEPTH_LEVEL')),
1841 'IX_SALE_LOCATION_TYPE' => array('TABLE' => $locationTable, 'COLUMNS' => array('TYPE_ID')),
1842 'IX_SALE_L_NAME_NAME_UPPER' => array('TABLE' => $locationNameTable, 'COLUMNS' => array('NAME_UPPER')),
1843 'IX_SALE_L_NAME_LID_LID' => array('TABLE' => $locationNameTable, 'COLUMNS' => array('LOCATION_ID', 'LANGUAGE_ID')),
1844 'IX_B_SALE_LOC_EXT_LID_SID' => array('TABLE' => $locationExternalTable, 'COLUMNS' => array('LOCATION_ID', 'SERVICE_ID')),
1845 'IX_SALE_LOCATION_TYPE_MARGIN' => array('TABLE' => $locationTable, 'COLUMNS' => array('TYPE_ID', 'LEFT_MARGIN', 'RIGHT_MARGIN')),
1846
1847 // legacy
1848 'IXS_LOCATION_COUNTRY_ID' => array('TABLE' => $locationTable, 'COLUMNS' => array('COUNTRY_ID')),
1849 'IXS_LOCATION_REGION_ID' => array('TABLE' => $locationTable, 'COLUMNS' => array('REGION_ID')),
1850 'IXS_LOCATION_CITY_ID' => array('TABLE' => $locationTable, 'COLUMNS' => array('CITY_ID')),
1851
1852 // obsolete
1853 'IX_B_SALE_LOCATION_1' => array('TABLE' => $locationTable, 'COLUMNS' => array('COUNTRY_ID'), 'DROP_ONLY' => true),
1854 'IX_B_SALE_LOCATION_2' => array('TABLE' => $locationTable, 'COLUMNS' => array('REGION_ID'), 'DROP_ONLY' => true),
1855 'IX_B_SALE_LOCATION_3' => array('TABLE' => $locationTable, 'COLUMNS' => array('CITY_ID'), 'DROP_ONLY' => true),
1856 );
1857 }
1858
1859 protected function dropIndexes($certainIndex = false)
1860 {
1861 $map = static::getIndexMap();
1862
1863 foreach($map as $index => $ixData)
1864 {
1865 if($certainIndex !== false && $certainIndex != $index)
1866 continue;
1867
1868 $this->dropIndexByName($index, $ixData['TABLE']);
1869 }
1870 }
1871
1872 public function restoreIndexes($certainIndex = false)
1873 {
1874 $map = $this->getIndexMap();
1875
1876 foreach($map as $ixName => $ixData)
1877 {
1878 if (($ixData['DROP_ONLY'] ?? null) === true)
1879 {
1880 continue;
1881 }
1882
1883 if($certainIndex !== false && $certainIndex != $ixName)
1884 continue;
1885
1886 if($this->checkIndexExistsByName($ixName, $ixData['TABLE']))
1887 return false;
1888
1889 $this->dbConnection->query('CREATE INDEX '.$ixName.' ON '.$ixData['TABLE'].' ('.implode(', ', $ixData['COLUMNS']).')');
1890 }
1891
1892 return true;
1893 }
1894
1895 private function getCachedBundle($id)
1896 {
1897 $locationTable = Location\LocationTable::getTableName();
1898
1899 $bundle = array();
1900 $res = $this->dbConnection->query("select ID from {$locationTable} where PARENT_ID = ".($id == 'root' ? '0' : intval($id)));
1901 while($item = $res->fetch())
1902 $bundle[] = $item['ID'];
1903
1904 return $bundle;
1905 }
1906
1907 private function checkNodeIsParent($id)
1908 {
1909 $locationTable = Location\LocationTable::getTableName();
1910
1911 $res = $this->dbConnection->query("select count(*) as CNT from {$locationTable} where PARENT_ID = ".($id == 'root' ? '0' : intval($id)))->fetch();
1912
1913 return intval($res['CNT']);
1914 }
1915
1916 private function mergeRebalancedNodes()
1917 {
1918 if($this->rebalanceInserter)
1919 {
1920 $this->logMessage('Finally, MERGE is in progress');
1921
1922 $this->rebalanceInserter->flush();
1923
1924 // merge temp table with location table
1925 // there should be more generalized method
1926 Location\LocationTable::mergeRelationsFromTemporalTable(self::TREE_REBALANCE_TEMP_TABLE_NAME, false, array('LEFT_MARGIN' => 'L', 'RIGHT_MARGIN' => 'R', 'DEPTH_LEVEL' => 'D', 'ID' => 'I'));
1927 }
1928 }
1929
1930 private function acceptRebalancedNode($node)
1931 {
1932 $this->createTempTable();
1933
1934 if(!$this->rebalanceInserter)
1935 {
1936 if($this->dbConnType == self::DB_TYPE_ORACLE)
1937 $mtu = self::TREE_REBALANCE_TEMP_BLOCK_LEN_O;
1938 else
1939 $mtu = self::TREE_REBALANCE_TEMP_BLOCK_LEN;
1940
1941 $this->rebalanceInserter = new BlockInserter(array(
1942 'tableName' => self::TREE_REBALANCE_TEMP_TABLE_NAME,
1943 'exactFields' => array(
1944 'I' => array('data_type' => 'integer'),
1945 'L' => array('data_type' => 'integer'),
1946 'R' => array('data_type' => 'integer'),
1947 'D' => array('data_type' => 'integer'),
1948 ),
1949 'parameters' => array(
1950 'mtu' => $mtu
1951 )
1952 ));
1953 }
1954
1955 $this->data['processed']++;
1956
1957 $this->rebalanceInserter->insert($node);
1958 }
1959
1960 private function dropTempTable()
1961 {
1962 if($this->dbConnection->isTableExists(self::TREE_REBALANCE_TEMP_TABLE_NAME))
1963 $this->dbConnection->query("drop table ".self::TREE_REBALANCE_TEMP_TABLE_NAME);
1964 }
1965
1966 private function createTempTable()
1967 {
1968 $this->data['rebalance']['tableCreated'] ??= false;
1969 if ($this->data['rebalance']['tableCreated'])
1970 {
1971 return;
1972 }
1973
1974 $tableName = self::TREE_REBALANCE_TEMP_TABLE_NAME;
1975
1976 if ($this->dbConnection->isTableExists($tableName))
1977 {
1978 $this->dbConnection->query("truncate table {$tableName}");
1979 }
1980 else
1981 {
1982
1983 if($this->dbConnType == self::DB_TYPE_ORACLE)
1984 {
1985 $this->dbConnection->query("create table {$tableName} (
1986 I NUMBER(18),
1987 L NUMBER(18),
1988 R NUMBER(18),
1989 D NUMBER(18)
1990 )");
1991 }
1992 else
1993 {
1994 $this->dbConnection->query("create table {$tableName} (
1995 I int,
1996 L int,
1997 R int,
1998 D int
1999 )");
2000 }
2001
2002 }
2003
2004 $this->data['rebalance']['tableCreated'] = true;
2005 }
2006
2007 protected function checkFileCompletelyRead()
2008 {
2009 return $this->data['current']['bytesRead'] >= $this->data['files'][$this->data['current']['fIndex']]['size'];
2010 }
2011
2012 protected function checkAllFilesRead()
2013 {
2014 return $this->data['current']['fIndex'] >= count($this->data['files']);
2015 }
2016
2017 protected function checkBufferIsFull($bufferSize)
2018 {
2019 return $bufferSize >= $this->getCurrStageStepSize();
2020 }
2021
2022 protected static function downloadFile($fileName, $storeAs, $skip404 = false, $storeTo = false)
2023 {
2024 // useless thing
2025 if(!$storeTo)
2026 $storeTo = \CTempFile::GetDirectoryName(1);
2027
2028 $storeTo .= $storeAs;
2029
2030 if(file_exists($storeTo))
2031 {
2032 if(!is_writable($storeTo))
2033 throw new Main\SystemException('Cannot remove previous '.$storeAs.' file');
2034
2035 unlink($storeTo);
2036 }
2037
2038 if(!defined('SALE_LOCATIONS_IMPORT_SOURCE_URL'))
2039 $query = 'http://'.self::DISTRIBUTOR_HOST.':'.self::DISTRIBUTOR_PORT.self::REMOTE_PATH.$fileName;
2040 else
2041 $query = 'http://'.SALE_LOCATIONS_IMPORT_SOURCE_URL.'/'.$fileName;
2042
2043 $client = new HttpClient();
2044
2045 if(!$client->download($query, $storeTo))
2046 {
2047 $eFormatted = array();
2048 foreach($client->getError() as $code => $desc)
2049 $eFormatted[] = trim($desc.' ('.$code.')');
2050
2051 throw new Main\SystemException('File download failed: '.implode(', ', $eFormatted).' ('.$query.')');
2052 }
2053
2054 $status = intval($client->getStatus());
2055
2056 if($status != 200 && file_exists($storeTo))
2057 unlink($storeTo);
2058
2059 $okay = $status == 200 || ($status == 404 && $skip404);
2060
2061 // honestly we should check for all 2xx codes, but for now this is enough
2062 if(!$okay)
2063 throw new Main\SystemException('File download failed: http error '.$status.' ('.$query.')');
2064
2065 return filesize($storeTo);
2066 }
2067
2071 protected static function cleanWorkDirectory()
2072 {
2073 }
2074
2075 protected function getFileNameByIndex($i)
2076 {
2077 return self::LOCAL_SETS_PATH.sprintf(self::LOCAL_LOCATION_FILE, $i);
2078 }
2079
2080 public function saveUserFile($inputName)
2081 {
2082 if(is_array($_FILES[$inputName]))
2083 {
2084 if($_FILES[$inputName]['error'] > 0)
2085 throw new Main\SystemException(self::explainFileUploadError($_FILES[$inputName]['error']));
2086
2087 if(!in_array($_FILES[$inputName]['type'], array(
2088 'text/plain',
2089 'text/csv',
2090 'application/vnd.ms-excel',
2091 'application/octet-stream'
2092 )))
2093 {
2094 throw new Main\SystemException('Unsupported file type');
2095 }
2096
2097 self::cleanWorkDirectory();
2098
2099 [$localPath, $tmpDirCreated] = $this->getTemporalDirectory();
2100 $fileName = $localPath.'/'.static::USER_FILE_TEMP_NAME;
2101
2102 if(!@copy($_FILES[$inputName]['tmp_name'], $fileName))
2103 {
2104 $lastError = error_get_last();
2105 throw new Main\SystemException($lastError['message']);
2106 }
2107
2108 $_SESSION[static::USER_FILE_DIRECTORY_SESSION_KEY] = $localPath;
2109 }
2110 else
2111 throw new Main\SystemException('No file were uploaded');
2112 }
2113
2114 protected static function explainFileUploadError($error)
2115 {
2116 switch ($error)
2117 {
2118 case UPLOAD_ERR_INI_SIZE:
2119 $message = 'The uploaded file exceeds the upload_max_filesize directive in php.ini';
2120 break;
2121 case UPLOAD_ERR_FORM_SIZE:
2122 $message = 'The uploaded file exceeds the MAX_FILE_SIZE directive that was specified in the HTML form';
2123 break;
2124 case UPLOAD_ERR_PARTIAL:
2125 $message = 'The uploaded file was only partially uploaded';
2126 break;
2127 case UPLOAD_ERR_NO_FILE:
2128 $message = 'No file were uploaded';
2129 break;
2130 case UPLOAD_ERR_NO_TMP_DIR:
2131 $message = 'Missing a temporary folder';
2132 break;
2133 case UPLOAD_ERR_CANT_WRITE:
2134 $message = 'Failed to write file to disk';
2135 break;
2136 case UPLOAD_ERR_EXTENSION:
2137 $message = 'File upload stopped by extension';
2138 break;
2139
2140 default:
2141 $message = 'Unknown upload error';
2142 break;
2143 }
2144 return $message;
2145 }
2146
2147 // all this mess is only to get import work on Bitrix24 (which does not provide any temporal directory in a typical meaning)
2148 protected function getTemporalDirectory(): array
2149 {
2150 $dataLocalPath = trim((string)($this->data['LOCAL_PATH'] ?? ''));
2151 $wasCreated = false;
2152 if ($dataLocalPath !== '' && \Bitrix\Main\IO\Directory::isDirectoryExists($dataLocalPath))
2153 {
2154 $localPath = $dataLocalPath;
2155 }
2156 else
2157 {
2158 $wasCreated = true;
2159 $localPath = \CTempFile::GetDirectoryName(10);
2160 if (!\Bitrix\Main\IO\Directory::isDirectoryExists($localPath))
2161 {
2162 \Bitrix\Main\IO\Directory::createDirectory($localPath);
2163 }
2164 }
2165
2166 return [
2167 $localPath,
2168 $wasCreated,
2169 ];
2170 }
2171
2172 protected static function createDirectory($path)
2173 {
2174 if(!\Bitrix\Main\IO\Directory::isDirectoryExists($path))
2175 {
2176 \Bitrix\Main\IO\Directory::createDirectory($path);
2177 }
2178 }
2179
2180 protected static function deleteDirectory($path)
2181 {
2182 if(\Bitrix\Main\IO\Directory::isDirectoryExists($path))
2183 {
2184 \Bitrix\Main\IO\Directory::deleteDirectory($path);
2185 }
2186 }
2187
2188 protected function normalizeQueryArray($value)
2189 {
2190 $result = array();
2191 if(is_array($value))
2192 {
2193 foreach($value as $v)
2194 {
2195 if($v <> '')
2196 {
2197 $result[] = $this->parseQueryCode($v);
2198 }
2199 }
2200 }
2201
2202 $result = array_unique($result);
2203 sort($result, SORT_STRING);
2204
2205 return $result;
2206 }
2207
2208 protected static function parseQueryCode($value)
2209 {
2210 $value = ToLower(trim($value));
2211
2212 if(!preg_match('#^[a-z0-9]+$#i', $value))
2213 throw new Main\SystemException('Bad request parameter');
2214
2215 return $value;
2216 }
2217
2218 public function turnOffCache()
2219 {
2220 $this->useCache = false;
2221 }
2222
2223 ########################################################
2224 ## static part is used in places like wizards, etc
2225
2226 public static function getExistedTypes()
2227 {
2228 $existed = array();
2229 $res = Location\TypeTable::getList(array('select' => array('ID', 'CODE', 'SORT')));
2230 while($item = $res->fetch())
2231 $existed[$item['CODE']] = $item['ID'];
2232
2233 return $existed;
2234 }
2235
2236 public static function getExistedServices()
2237 {
2238 $existed = array();
2239 $res = Location\ExternalServiceTable::getList(array('select' => array('ID', 'CODE')));
2240 while($item = $res->fetch())
2241 $existed[$item['CODE']] = $item['ID'];
2242
2243 return $existed;
2244 }
2245
2246 public static function createTypes($types, $existed = false)
2247 {
2248 // read existed
2249 if($existed === false)
2250 $existed = static::getExistedTypes();
2251
2252 // here we try to add type names for ALL languages
2253 $langs = Location\Admin\NameHelper::getLanguageList();
2254
2255 foreach($types as $line)
2256 {
2257 // for sure
2258 unset($line['SELECTORLEVEL']);
2259 unset($line['DEFAULTSELECT']);
2260
2261 $names = array();
2262
2263 if(!is_array($line['NAME']))
2264 $line['NAME'] = array();
2265
2266 if(is_array($langs))
2267 {
2268 foreach($langs as $lid => $f)
2269 {
2270 $names[ToUpper($lid)] = static::getTranslatedName($line['NAME'], $lid);
2271 }
2272 $line['NAME'] = $names;
2273 }
2274
2275 if(!isset($existed[$line['CODE']]))
2276 {
2277 $existed[$line['CODE']] = static::createType($line);
2278 }
2279 else
2280 {
2281 // ensure it has all appropriate translations
2282 // we can not use ::updateMultipleForOwner() here, because user may rename his types manually
2283 Location\Name\TypeTable::addAbsentForOwner($existed[$line['CODE']], $names);
2284 }
2285 }
2286
2287 return $existed;
2288 }
2289
2290 protected static function getTranslatedName($names, $languageId)
2291 {
2292 $languageIdMapped = ToUpper(Location\Admin\NameHelper::mapLanguage($languageId));
2293 $languageId = ToUpper($languageId);
2294
2295 if(is_array($names[$languageId]) && (string) $names[$languageId]['NAME'] != '')
2296 return $names[$languageId];
2297
2298 if(is_array($names[$languageIdMapped]) && (string) $names[$languageIdMapped]['NAME'] != '')
2299 return $names[$languageIdMapped];
2300
2301 return $names['EN'];
2302 }
2303
2304 public static function createType($type)
2305 {
2306 $map = Location\TypeTable::getMap($type);
2307
2308 if(is_array($type))
2309 {
2310 foreach($type as $fld => $val)
2311 {
2312 if(!isset($map[$fld]))
2313 {
2314 unset($type[$fld]);
2315 }
2316 }
2317 }
2318
2319 $res = Location\TypeTable::add($type);
2320 if(!$res->isSuccess())
2321 throw new Main\SystemException('Type creation failed: '.implode(', ', $res->getErrorMessages()));
2322
2323 return $res->getId();
2324 }
2325
2326 public static function createService($service)
2327 {
2328 $res = Location\ExternalServiceTable::add($service);
2329 if(!$res->isSuccess())
2330 throw new Main\SystemException('External service creation failed: '.implode(', ', $res->getErrorMessages()));
2331
2332 return $res->getId();
2333 }
2334
2335 public static function getTypeMap($file)
2336 {
2337 $csvReader = new CSVReader();
2338 $csvReader->LoadFile($file);
2339
2340 $types = array();
2341 $i = 0;
2342 while($type = $csvReader->FetchAssoc())
2343 {
2344 if($i) // fix for CSVReader parent class bug
2345 {
2346 unset($type['SELECTORLEVEL']);
2347 unset($type['DEFAULTSELECT']);
2348
2349 $types[$type['CODE']] = $type;
2350 }
2351
2352 $i++;
2353 }
2354
2355 return $types;
2356 }
2357
2358 public static function getServiceMap($file)
2359 {
2360 $csvReader = new CSVReader();
2361 $csvReader->LoadFile($file);
2362
2363 $services = array();
2364 while($service = $csvReader->FetchAssoc())
2365 $services[$service['CODE']] = $service;
2366
2367 return $services;
2368 }
2369
2370 // this is generally for e-shop installer
2371 public static function importFile(&$descriptior)
2372 {
2373 $timeLimit = ini_get('max_execution_time');
2374 if ($timeLimit < $descriptior['TIME_LIMIT']) set_time_limit($descriptior['TIME_LIMIT'] + 5);
2375
2376 $endTime = time() + $descriptior['TIME_LIMIT'];
2377
2378 if($descriptior['STEP'] == 'rebalance')
2379 {
2380 Location\LocationTable::resort();
2381 Location\LocationTable::resetLegacyPath();
2382 $descriptior['STEP'] = 'done';
2383 }
2384
2385 if($descriptior['STEP'] == 'import')
2386 {
2387 if(!isset($descriptior['DO_SYNC']))
2388 {
2389 $res = \Bitrix\Sale\Location\LocationTable::getList(array('select' => array('CNT')))->fetch();
2390 $descriptior['DO_SYNC'] = intval($res['CNT'] > 0);
2391 }
2392
2393 if(!isset($descriptior['TYPES']))
2394 {
2395 $descriptior['TYPE_MAP'] = static::getTypeMap($descriptior['TYPE_FILE']);
2396 $descriptior['TYPES'] = static::createTypes($descriptior['TYPE_MAP']);
2397
2398 $descriptior['SERVICE_MAP'] = static::getServiceMap($descriptior['SERVICE_FILE']);
2399 $descriptior['SERVICES'] = static::getExistedServices();
2400 }
2401
2402 $csvReader = new CSVReader();
2403 $csvReader->LoadFile($descriptior['FILE']);
2404
2405 while(time() < $endTime)
2406 {
2407 $block = $csvReader->ReadBlockLowLevel($descriptior['POS']/*changed inside*/, 10);
2408
2409 if(!count($block))
2410 break;
2411
2412 foreach($block as $item)
2413 {
2414 if($descriptior['DO_SYNC'])
2415 {
2416 $id = static::checkLocationCodeExists($item['CODE']);
2417 if($id)
2418 {
2419 $descriptior['CODES'][$item['CODE']] = $id;
2420 continue;
2421 }
2422 }
2423
2424 // type
2425 $item['TYPE_ID'] = $descriptior['TYPES'][$item['TYPE_CODE']];
2426 unset($item['TYPE_CODE']);
2427
2428 // parent id
2429 if($item['PARENT_CODE'] <> '')
2430 {
2431 if(!isset($descriptior['CODES'][$item['PARENT_CODE']]))
2432 {
2433 $descriptior['CODES'][$item['PARENT_CODE']] = static::checkLocationCodeExists($item['PARENT_CODE']);
2434 }
2435
2436 $item['PARENT_ID'] = $descriptior['CODES'][$item['PARENT_CODE']];
2437 }
2438 unset($item['PARENT_CODE']);
2439
2440 // ext
2441 if(is_array($item['EXT']))
2442 {
2443 foreach($item['EXT'] as $code => $values)
2444 {
2445 if(!empty($values))
2446 {
2447 if(!isset($descriptior['SERVICES'][$code]))
2448 {
2449 $descriptior['SERVICES'][$code] = static::createService(array(
2450 'CODE' => $code
2451 ));
2452 }
2453
2454 if($code == 'ZIP_LOWER')
2455 {
2456 if($values[0] == '')
2457 continue;
2458
2459 $values = explode(',', $values[0]);
2460
2461 if(!is_array($values))
2462 continue;
2463
2464 $values = array_unique($values);
2465 }
2466
2467 if(is_array($values))
2468 {
2469 foreach($values as $value)
2470 {
2471 if($value == '')
2472 continue;
2473
2474 $item['EXTERNAL'][] = array(
2475 'SERVICE_ID' => $descriptior['SERVICES'][$code],
2476 'XML_ID' => $value
2477 );
2478 }
2479 }
2480 }
2481 }
2482 }
2483 unset($item['EXT'], $item['ZIP_LOWER']);
2484
2485 $res = Location\LocationTable::addExtended(
2486 $item,
2487 array(
2488 'RESET_LEGACY' => false,
2489 'REBALANCE' => false
2490 )
2491 );
2492
2493 if(!$res->isSuccess())
2494 throw new Main\SystemException('Cannot create location');
2495
2496 $descriptior['CODES'][$item['CODE']] = $res->getId();
2497 }
2498 }
2499
2500 if(!count($block))
2501 {
2502 unset($descriptior['CODES']);
2503 $descriptior['STEP'] = 'rebalance';
2504 }
2505 }
2506
2507 return $descriptior['STEP'] == 'done';
2508 }
2509
2510 public function provideEnFromRu(&$data)
2511 {
2512 // restore at least "EN" translation
2513 if (!is_array($data))
2514 {
2515 return;
2516 }
2517 if (!isset($data['NAME']['RU']))
2518 {
2519 return;
2520 }
2521 if (!is_array($data['NAME']['RU']))
2522 {
2523 return;
2524 }
2525 $data['NAME']['EN'] ??= [];
2526
2527 foreach ($data['NAME']['RU'] as $k => $v)
2528 {
2529 if ((string)($data['NAME']['EN'][$k] ?? '') === '')
2530 {
2531 $data['NAME']['EN'][$k] = Location\Admin\NameHelper::translitFromUTF8($data['NAME']['RU'][$k]);
2532 }
2533 }
2534 }
2535}
Bitrix\Main\IO\Directory\isDirectoryExists
static isDirectoryExists($path)
Definition directory.php:162
Bitrix\Sale\Location\Import\ImportProcess\downloadFile
static downloadFile($fileName, $storeAs, $skip404=false, $storeTo=false)
Definition importprocess.php:2022
Bitrix\Sale\Location\Import\ImportProcess\createTypes
static createTypes($types, $existed=false)
Definition importprocess.php:2246
Bitrix\Sale\Location\Import\ImportProcess\getTranslatedName
static getTranslatedName($names, $languageId)
Definition importprocess.php:2290
Bitrix\Sale\Location\Util\Process\getStagePercent
getStagePercent($sNum=false)
Percentage.
Definition process.php:327
Bitrix\Sale\Location\Util\Process\logMessage
logMessage($message='', $addTimeStamp=true)
Definition process.php:468