Bitrix-D7 23.9
 
Загрузка...
Поиск...
Не найдено
filehash.php
1<?php
2namespace Bitrix\Clouds;
3
11
12Loc::loadMessages(__FILE__);
13
41{
47 public static function getTableName()
48 {
49 return 'b_clouds_file_hash';
50 }
51
57 public static function getMap()
58 {
59 return [
60 new IntegerField(
61 'ID',
62 [
63 'primary' => true,
64 'autocomplete' => true,
65 'title' => Loc::getMessage('FILE_HASH_ENTITY_ID_FIELD'),
66 ]
67 ),
68 new IntegerField(
69 'BUCKET_ID',
70 [
71 'required' => true,
72 'title' => Loc::getMessage('FILE_HASH_ENTITY_BUCKET_ID_FIELD'),
73 ]
74 ),
75 new StringField(
76 'FILE_PATH',
77 [
78 'required' => true,
79 'validation' => [__CLASS__, 'validateFilePath'],
80 'title' => Loc::getMessage('FILE_HASH_ENTITY_FILE_PATH_FIELD'),
81 ]
82 ),
83 new IntegerField(
84 'FILE_SIZE',
85 [
86 'title' => Loc::getMessage('FILE_HASH_ENTITY_FILE_SIZE_FIELD'),
87 ]
88 ),
89 new DatetimeField(
90 'FILE_MTIME',
91 [
92 'title' => Loc::getMessage('FILE_HASH_ENTITY_FILE_MTIME_FIELD'),
93 ]
94 ),
95 new StringField(
96 'FILE_HASH',
97 [
98 'validation' => [__CLASS__, 'validateFileHash'],
99 'title' => Loc::getMessage('FILE_HASH_ENTITY_FILE_HASH_FIELD'),
100 ]
101 ),
102 ];
103 }
104
110 public static function validateFilePath()
111 {
112 return [
113 new LengthValidator(null, 760),
114 ];
115 }
116
122 public static function validateFileHash()
123 {
124 return [
125 new LengthValidator(null, 50),
126 ];
127 }
128
138 public static function addList($bucketId, array $files)
139 {
140 $bucketId = intval($bucketId);
141 $connection = \Bitrix\Main\Application::getConnection();
142 $helper = $connection->getSqlHelper();
143 $values = [];
144 foreach ($files['file'] as $i => $file)
145 {
146 $fileSize = $files['file_size'][$i];
147 $fileMtime = \CCloudUtil::gmtTimeToDateTime($files['file_mtime'][$i]);
148 $fileHash = $files['file_hash'][$i];
149 $values [] = '('
150 . $bucketId
151 . ",'" . $helper->forSql($file) . "'"
152 . ',' . intval($fileSize)
153 . ",'" . $fileMtime->format('Y-m-d h:i:s') . "'"
154 . ",'" . $helper->forSql($fileHash) . "'"
155 . ')'
156 ;
157 }
158 $sql = '
159 INSERT INTO ' . static::getTableName() . '
160 (BUCKET_ID, FILE_PATH, FILE_SIZE, FILE_MTIME, FILE_HASH)
161 VALUES
162 ' . implode(",\n", $values) . '
163 ';
164 return $connection->query($sql);
165 }
166
178 public static function syncList($bucketId, $path, array $files, $prevLastKey)
179 {
180 $result = null;
181 $bucketId = intval($bucketId);
182 $connection = \Bitrix\Main\Application::getConnection();
183 $helper = $connection->getSqlHelper();
184
185 $index = [];
186 foreach ($files['file'] as $i => $filePath)
187 {
188 $index[$path . $filePath] = $i;
189 }
190
191 $filter = [
192 '=BUCKET_ID' => $bucketId,
193 ];
194 if ($prevLastKey)
195 {
196 $filter['>FILE_PATH'] = $path . $prevLastKey;
197 }
198 if ($files['last_key'])
199 {
200 $filter['<=FILE_PATH'] = $path . $files['last_key'];
201 }
202 $fileList = static::getList([
203 'select' => ['ID', 'FILE_PATH', 'FILE_SIZE', 'FILE_HASH'],
204 'filter' => $filter,
205 ]);
206 while ($fileInfo = $fileList->fetch())
207 {
208 if (
209 array_key_exists($fileInfo['FILE_PATH'], $index)
210 && ($files['file_size'][$index[$fileInfo['FILE_PATH']]] == $fileInfo['FILE_SIZE'])
211 && ($files['file_hash'][$index[$fileInfo['FILE_PATH']]] == $fileInfo['FILE_HASH'])
212 )
213 {
214 unset($files['file'][$index[$fileInfo['FILE_PATH']]]);
215 }
216 else
217 {
218 $deleteResult = static::delete($fileInfo['ID']);
219 }
220 }
221
222 $values = [];
223 foreach ($files['file'] as $i => $file)
224 {
225 $fileSize = $files['file_size'][$i];
226 $fileMtime = \CCloudUtil::gmtTimeToDateTime($files['file_mtime'][$i]);
227 $fileHash = $files['file_hash'][$i];
228 $values [] = '('
229 . $bucketId
230 . ",'" . $helper->forSql($path . $file) . "'"
231 . ',' . intval($fileSize)
232 . ",'" . $fileMtime->format('Y-m-d h:i:s') . "'"
233 . ",'" . $helper->forSql($fileHash) . "'"
234 . ')'
235 ;
236 }
237
238 $insertSize = 1000;
239 while ($values)
240 {
241 $sql = '
242 INSERT INTO ' . static::getTableName() . '
243 (BUCKET_ID, FILE_PATH, FILE_SIZE, FILE_MTIME, FILE_HASH)
244 VALUES
245 ' . implode(",\n", array_splice($values, 0, $insertSize)) . '
246 ';
247 $result = $connection->query($sql);
248 }
249
250 return $result;
251 }
252
263 public static function syncEnd($bucketId, $path, $prevLastKey)
264 {
265 $bucketId = intval($bucketId);
266 $connection = \Bitrix\Main\Application::getConnection();
267 $sqlHelper = $connection->getSqlHelper();
268 $delete = '
269 DELETE from ' . static::getTableName() . '
270 WHERE BUCKET_ID = ' . $bucketId . '
271 AND FILE_PATH like \'' . $sqlHelper->forSql($path) . '%\'
272 AND FILE_PATH > \'' . $sqlHelper->forSql($path . $prevLastKey) . '\'
273 ';
274 $result = $connection->query($delete);
275 return $result;
276 }
277
288 public static function addFile($bucketId, $path, array $fileInfo)
289 {
290 return static::add([
291 'BUCKET_ID' => $bucketId,
292 'FILE_PATH' => $path,
293 'FILE_SIZE' => $fileInfo['size'],
294 'FILE_MTIME' => \CCloudUtil::gmtTimeToDateTime($fileInfo['mtime']),
295 'FILE_HASH' => $fileInfo['hash'],
296 ]);
297 }
298
306 public static function getLastKey($bucketId)
307 {
308 $bucketId = intval($bucketId);
309 $connection = \Bitrix\Main\Application::getConnection();
310 $sql = 'SELECT max(FILE_PATH) LAST_KEY from ' . static::getTableName() . ' WHERE BUCKET_ID=' . $bucketId;
311 $last = $connection->query($sql)->fetch();
312 return $last && $last['LAST_KEY'] ? $last['LAST_KEY'] : '';
313 }
314
322 public static function deleteAll($bucketId)
323 {
324 $bucketId = intval($bucketId);
325 $connection = \Bitrix\Main\Application::getConnection();
326 $delete = 'DELETE from ' . static::getTableName() . ' WHERE BUCKET_ID=' . $bucketId;
327 $result = $connection->query($delete);
328 return $result;
329 }
330
339 public static function deleteByFilePath($bucketId, $filePath)
340 {
341 $bucketId = intval($bucketId);
342 $connection = \Bitrix\Main\Application::getConnection();
343 $sqlHelper = $connection->getSqlHelper();
344 $delete = '
345 DELETE from ' . static::getTableName() . '
346 WHERE BUCKET_ID = ' . $bucketId . "
347 AND FILE_PATH = '" . $sqlHelper->forSql($filePath) . "'
348 ";
349 $result = $connection->query($delete);
350 return $result;
351 }
352
363 public static function dirList($bucketId, $path, $order, $filter)
364 {
365 $connection = \Bitrix\Main\Application::getConnection();
366 $sqlHelper = $connection->getSqlHelper();
367
368 $query = \Bitrix\Clouds\FileHashTable::query();
369 $query->setSelect([
370 new ExpressionField(
371 'FILE_TYPE',
372 'if(locate(\'/\', substring(%s, length(\'' . $sqlHelper->forSql($path) . '\')+1)) > 0, \'D\', \'F\')',
373 ['FILE_PATH']
374 ),
375 new ExpressionField(
376 'NAME',
377 'substring_index(substring(%s, length(\'' . $sqlHelper->forSql($path) . '\')+1), \'/\', 1)',
378 ['FILE_PATH']
379 ),
380 new ExpressionField(
381 'SUM_FILE_SIZE',
382 'SUM(%s)',
383 ['FILE_SIZE']
384 ),
385 new ExpressionField(
386 'MAX_FILE_MTIME',
387 'MAX(%s)',
388 ['FILE_MTIME']
389 ),
390 new ExpressionField(
391 'FILE_COUNT',
392 'COUNT(1)'
393 ),
394 ]);
395
396 $filter['=BUCKET_ID'] = $bucketId;
397 $filter['%=FILE_PATH'] = $path . '%';
398 $query->setFilter($filter);
399 $query->setGroup(['FILE_TYPE', 'NAME']);
400 $query->setOrder($order);
401
402 $sql = $query->getQuery();
403
404 return $connection->query($sql);
405 }
406
417 public static function duplicateList($bucketId, $filter, $order, $limit = 0)
418 {
419 $connection = \Bitrix\Main\Application::getConnection();
420
421 $query = \Bitrix\Main\File\Internal\FileHashTable::query();
422 $query->setSelect([
423 'FILE_HASH',
424 'FILE_SIZE',
425 new ExpressionField(
426 'FILE_COUNT',
427 'COUNT(distinct %s, %s)',
428 ['FILE.SUBDIR', 'FILE.FILE_NAME']
429 ),
430 new ExpressionField(
431 'FILE_ID_MIN',
432 'MIN(%s)',
433 ['FILE_ID']
434 ),
435 new ExpressionField(
436 'FILE_ID_MAX',
437 'MAX(%s)',
438 ['FILE_ID']
439 ),
440 ]);
441
442 $filter['=FILE.HANDLER_ID'] = $bucketId;
443 $filter['>FILE_COUNT'] = 1;
444 $query->setFilter($filter);
445 $query->setGroup(['FILE_HASH', 'FILE_SIZE']);
446 $query->setOrder($order);
447 if ($limit > 0)
448 {
449 $query->setLimit($limit);
450 }
451
452 $sql = $query->getQuery();
453
454 return $connection->query($sql);
455 }
456
466 public static function getFileDuplicates($bucketId, $fileHash, $fileSize)
467 {
468 $query = \Bitrix\Main\File\Internal\FileHashTable::getList([
469 'select' => [
470 'FILE_ID',
471 ],
472 'filter' => [
473 '=FILE.HANDLER_ID' => $bucketId,
474 '=FILE_HASH' => $fileHash,
475 '=FILE_SIZE' => $fileSize,
476 ],
477 'order' => [
478 'FILE_ID' => 'ASC',
479 ],
480 ]);
481
482 $result = [];
483 while ($fileDuplicate = $query->fetch())
484 {
485 $result[] = $fileDuplicate['FILE_ID'];
486 }
487 return $result;
488 }
489
496 public static function getDuplicatesStat($bucketId)
497 {
498 $bucketId = intval($bucketId);
499 $connection = \Bitrix\Main\Application::getConnection();
500 $sql = "
501 select sum(DUP_COUNT) DUP_COUNT, sum(DUP_SIZE) DUP_SIZE
502 from (
503 select
504 b_file_hash.FILE_SIZE
505 ,b_file_hash.FILE_HASH
506 ,count(distinct b_file.SUBDIR, b_file.FILE_NAME)-1 DUP_COUNT
507 ,(count(distinct b_file.SUBDIR, b_file.FILE_NAME)-1) * b_file_hash.FILE_SIZE DUP_SIZE
508 from
509 b_file_hash
510 inner join b_file on
511 b_file.ID = b_file_hash.FILE_ID
512 where
513 b_file.HANDLER_ID = " . $bucketId . "
514 group by
515 b_file_hash.FILE_SIZE, b_file_hash.FILE_HASH
516 having
517 count(distinct b_file.SUBDIR, b_file.FILE_NAME) > 1
518 ) t
519 ";
520 return $connection->query($sql)->fetch();
521 }
522
531 public static function copyToFileHash($lastKey, $pageSize)
532 {
533 $lastKey = (int)$lastKey;
534 $pageSize = (int)$pageSize;
535 $sql = "
536 SELECT
537 b_file.ID as FILE_ID
538 ,b_clouds_file_hash.FILE_SIZE as FILE_SIZE
539 ,b_clouds_file_hash.FILE_HASH as FILE_HASH
540 FROM
541 b_file
542 INNER JOIN b_clouds_file_hash ON
543 b_clouds_file_hash.BUCKET_ID = b_file.HANDLER_ID
544 AND b_clouds_file_hash.FILE_PATH = concat('/', b_file.SUBDIR, '/', b_file.FILE_NAME)
545 LEFT JOIN b_file_duplicate ON
546 b_file_duplicate.DUPLICATE_ID = b_file.ID
547 WHERE
548 b_file.ID > " . $lastKey . "
549 AND b_file_duplicate.DUPLICATE_ID is null
550 ORDER BY b_file.ID
551 LIMIT " . $pageSize . "
552 ";
553
554 $connection = \Bitrix\Main\Application::getConnection();
555 $fileIds = $connection->query('
556 SELECT
557 min(FILE_ID) as FILE_ID_MIN
558 ,max(FILE_ID) as FILE_ID_MAX
559 ,count(FILE_ID) FILE_ID_CNT
560 FROM (' . $sql . ') t
561 ')->fetch();
562
563 if ($fileIds['FILE_ID_CNT'] > 0)
564 {
565 $connection->queryExecute('INSERT IGNORE INTO b_file_hash '.$sql);
566 }
567
568 return $fileIds;
569 }
570
578 public static function prepareDuplicates($bucketId, &$fileIds)
579 {
580 $originalId = false;
581 if ($fileIds)
582 {
583 //Exclude any id that is already a duplicate
584 $duplicates = \Bitrix\Main\File\Internal\FileDuplicateTable::getList([
585 'select' => ['DUPLICATE_ID'],
586 'filter' => [
587 '=DUPLICATE_ID' => $fileIds,
588 ],
589 ]);
590 while ($duplicate = $duplicates->fetch())
591 {
592 //Others will be excluded from the process
593 $p = array_search($duplicate['DUPLICATE_ID'], $fileIds);
594 if ($p !== false)
595 {
596 unset($fileIds[$p]);
597 }
598 }
599 }
600
601 if ($fileIds)
602 {
603 //Search throught file id for any existing originals
604 $originals = \Bitrix\Main\File\Internal\FileDuplicateTable::getList([
605 'select' => ['ORIGINAL_ID'],
606 'filter' => [
607 '=ORIGINAL_ID' => $fileIds,
608 ],
609 'order' => ['ORIGINAL_ID' => 'ASC'],
610 ]);
611 while ($original = $originals->fetch())
612 {
613 //First will be used for future deduplication
614 if ($originalId === false)
615 {
616 $originalId = $original['ORIGINAL_ID'];
617 }
618
619 //Others will be excluded from the process
620 $p = array_search($original['ORIGINAL_ID'], $fileIds);
621 if ($p !== false)
622 {
623 unset($fileIds[$p]);
624 }
625 }
626 //None originals exists yet
627 if ($originalId === false)
628 {
629 $originalId = array_shift($fileIds);
630 }
631 }
632
633 return $originalId;
634 }
635}
static duplicateList($bucketId, $filter, $order, $limit=0)
Definition filehash.php:417
static prepareDuplicates($bucketId, &$fileIds)
Definition filehash.php:578
static syncList($bucketId, $path, array $files, $prevLastKey)
Definition filehash.php:178
static syncEnd($bucketId, $path, $prevLastKey)
Definition filehash.php:263
static copyToFileHash($lastKey, $pageSize)
Definition filehash.php:531
static getDuplicatesStat($bucketId)
Definition filehash.php:496
static getFileDuplicates($bucketId, $fileHash, $fileSize)
Definition filehash.php:466
static addList($bucketId, array $files)
Definition filehash.php:138
static loadMessages($file)
Definition loc.php:64
static getMessage($code, $replace=null, $language=null)
Definition loc.php:29